Add support for repeated keys in TaskRequest.

appengine/swarming/server/task_request_test.py

 #!/usr/bin/env python
 # Copyright 2014 The LUCI Authors. All rights reserved.
 # Use of this source code is governed under the Apache License, Version 2.0
 # that can be found in the LICENSE file.
 
 import datetime
 import logging
 import random
 import sys
 import unittest
@@ skipping 57 matching lines @@
   })
 
   inputs_ref = properties.pop('inputs_ref', {
     'isolatedserver': 'https://isolateserver.appspot.com',
     'namespace': 'default-gzip',
   })
 
   properties = merge(properties, {
     'cipd_input': cipd_input,
     'command': [u'command1', u'arg1'],
-    'dimensions': {
-      u'OS': u'Windows-3.1.1',
-      u'hostname': u'localhost',
-      u'pool': u'default',
-    },
+    'dimensions_flat': [
+      u'hostname:localhost',
+      u'os:Windows-3.1.1',
+      # Test repeated keys. In this case, the bot has to be in both pools.
+      u'pool:default',
+      u'pool:testing',
+    ],
     'env': {u'foo': u'bar', u'joe': u'2'},
     'execution_timeout_secs': 30,
     'grace_period_secs': 30,
     'idempotent': False,
     'inputs_ref': inputs_ref,
     'io_timeout_secs': None,
     'has_secret_bytes': 'secret_bytes' in kwargs,
   })
-  properties['dimensions_dict'] = properties.pop('dimensions')
   now = utils.utcnow()
   args = {
     'created_ts': now,
     'name': 'Request name',
     'priority': 50,
     'properties': properties,
     'expiration_ts': now + datetime.timedelta(seconds=30),
     'tags': [u'tag:1'],
     'user': 'Jesus',
   }
@@ skipping 133 matching lines @@
           'version': u'git_revision:deadbeef',
         },
         'packages': [{
           'package_name': u'rm',
           'path': u'bin',
           'version': u'git_revision:deadbeef',
         }],
         'server': u'https://chrome-infra-packages.appspot.com'
       },
       'command': [u'command1', u'arg1'],
-      'dimensions': {
-        u'OS': u'Windows-3.1.1',
-        u'hostname': u'localhost',
-        u'pool': u'default',
-      },
+      'dimensions': [
+        (u'hostname', u'localhost'),
+        (u'os', u'Windows-3.1.1'),
+        (u'pool', u'default'),
+        (u'pool', u'testing'),
+      ],
       'env': {u'foo': u'bar', u'joe': u'2'},
       'extra_args': [],
       'execution_timeout_secs': 30,
       'grace_period_secs': 30,
       'idempotent': True,
       'inputs_ref': {
         'isolated': None,
         'isolatedserver': u'https://isolateserver.appspot.com',
         'namespace': u'default-gzip',
       },
       'io_timeout_secs': None,
       'outputs': [],
       'has_secret_bytes': True,
     }
     expected_request = {
       'authenticated': auth_testing.DEFAULT_MOCKED_IDENTITY,
       'name': u'Request name',
       'parent_task_id': unicode(parent_id),
       'priority': 49,
       'properties': expected_properties,
       # Intentionally hard code the hash value since it has to be deterministic.
       # Other unit tests should use the calculated value.
       'properties_hash':
-          '258e0894f2589a7bc2b46bd563d01f64300ae08938fb14b3d750baff94ba714e',
+          '40635327d825d2e12ba704b7db72be55dc1dcf195ff881eff558c1be2a9f7ef6',
       'pubsub_topic': None,
       'pubsub_userdata': None,
       'service_account': u'none',
       'tags': [
-        u'OS:Windows-3.1.1',
         u'hostname:localhost',
+        u'os:Windows-3.1.1',
         u'pool:default',
+        u'pool:testing',
         u'priority:49',
         u'service_account:none',
         u'tag:1',
         u'user:Jesus',
       ],
       'user': u'Jesus',
     }
     actual = request.to_dict()
     actual.pop('created_ts')
     actual.pop('expiration_ts')
@@ skipping 23 matching lines @@
           'version': u'git_revision:deadbeef',
         },
         'packages': [{
           'package_name': u'rm',
           'path': u'bin',
           'version': u'git_revision:deadbeef',
         }],
         'server': u'https://chrome-infra-packages.appspot.com'
       },
       'command': [u'command1', u'arg1'],
-      'dimensions': {
-        u'OS': u'Windows-3.1.1',
-        u'hostname': u'localhost',
-        u'pool': u'default',
-      },
+      'dimensions': [
+        (u'hostname', u'localhost'),
+        (u'os', u'Windows-3.1.1'),
+        (u'pool', u'default'),
+        (u'pool', u'testing'),
+      ],
       'env': {u'foo': u'bar', u'joe': u'2'},
       'extra_args': [],
       'execution_timeout_secs': 30,
       'grace_period_secs': 30,
       'idempotent': True,
       'inputs_ref': {
         'isolated': None,
         'isolatedserver': u'https://isolateserver.appspot.com',
         'namespace': u'default-gzip',
       },
       'io_timeout_secs': None,
       'outputs': [],
       'has_secret_bytes': True,
     }
     expected_request = {
       'authenticated': auth_testing.DEFAULT_MOCKED_IDENTITY,
       'name': u'Request name',
       'parent_task_id': unicode(parent_id),
       'priority': 49,
       'properties': expected_properties,
       # Intentionally hard code the hash value since it has to be deterministic.
       # Other unit tests should use the calculated value.
       'properties_hash':
-          '693715e9539552e2ece434e25a665d437cfe4685504a3c7bad6356f9feffee2a',
+          'cbbf1b58e73a27c15bae48019cd6678a47a4bfb637c63b47af7231e11d82109e',
       'pubsub_topic': None,
       'pubsub_userdata': None,
       'service_account': u'none',
       'tags': [
-        u'OS:Windows-3.1.1',
         u'hostname:localhost',
+        u'os:Windows-3.1.1',
         u'pool:default',
+        u'pool:testing',
         u'priority:49',
         u'service_account:none',
         u'tag:1',
         u'user:Jesus',
       ],
       'user': u'Jesus',
     }
     actual = request.to_dict()
     # expiration_ts - created_ts == scheduling_expiration_secs.
     actual.pop('created_ts')
@@ skipping 14 matching lines @@
       _gen_request(parent_task_id='1d69b9f088008810')
 
   def test_init_new_request_idempotent(self):
     request = mkreq(_gen_request(properties=dict(idempotent=True)))
     as_dict = request.to_dict()
     self.assertEqual(True, as_dict['properties']['idempotent'])
     # Intentionally hard code the hash value since it has to be deterministic.
     # Other unit tests should use the calculated value.
     # Ensure the algorithm is deterministic.
     self.assertEqual(
-        'c7445927612637c589b72dd47c4657b166a53ac6ecdfd08c167539331a7890d3',
+        'b4ee0e7018e5fdb414ede87d0da2bc1479211191c5cd6a7af552916d4aa40d25',
         as_dict['properties_hash'])
 
   def test_init_new_request_bot_service_account(self):
     request = mkreq(_gen_request(service_account_token='bot'))
     as_dict = request.to_dict()
     self.assertEqual('bot', as_dict['service_account'])
     self.assertIn(u'service_account:bot', as_dict['tags'])
 
   def test_duped(self):
     # Two TestRequest with the same properties.
@@ skipping 134 matching lines @@
           caches=[dict(name='git_chromium', path='git_cache')],
           cipd_input=dict(packages=[
             dict(package_name='foo', path='git_cache', version='latest')]))))
     mkcachereq()
     mkcachereq(dict(name='git_chromium', path='git_cache'))
     mkcachereq(
         dict(name='git_chromium', path='git_cache'),
         dict(name='build_chromium', path='out'))
 
     # Dimensions.
-    with self.assertRaises(TypeError):
-      mkreq(_gen_request(properties=dict(dimensions=[])))
     with self.assertRaises(datastore_errors.BadValueError):
-      mkreq(_gen_request(properties=dict(dimensions={})))
+      mkreq(_gen_request(properties=dict(dimensions_flat={})))
+    with self.assertRaises(datastore_errors.BadValueError):
+      mkreq(_gen_request(properties=dict(dimensions_flat=[])))
+    with self.assertRaises(datastore_errors.BadValueError):
+      mkreq(_gen_request(properties=dict(dimensions_dict={u'id': u'b'})))
     with self.assertRaises(datastore_errors.BadValueError):
       mkreq(_gen_request(
-          properties=dict(dimensions={u'id': u'b', u'a:': u'b'})))
+          properties=dict(dimensions_flat=[u'id:b', u'a :b'])))
+    with self.assertRaises(datastore_errors.BadValueError):
+      mkreq(_gen_request(
+          properties=dict(dimensions_flat=[u'id:b', u'a.:b'])))
     mkreq(_gen_request(
-        properties=dict(dimensions={u'id': u'b', u'a.': u'b'})))
+        properties=dict(dimensions_flat=[u'a.:b', u'id:b'])))
 
     # Environment.
     with self.assertRaises(TypeError):
       mkreq(_gen_request(properties=dict(env=[])))
     with self.assertRaises(TypeError):
       mkreq(_gen_request(properties=dict(env={u'a': 1})))
     mkreq(_gen_request(properties=dict(env={})))
 
     # Priority.
     with self.assertRaises(datastore_errors.BadValueError):
@@ skipping 76 matching lines @@
           'version': u'git_revision:deadbeef',
         },
         'packages': [{
           'package_name': u'rm',
           'path': u'bin',
           'version': u'git_revision:deadbeef',
         }],
         'server': u'https://chrome-infra-packages.appspot.com'
       },
       'command': [u'command1', u'arg1'],
-      'dimensions': {
-        u'OS': u'Windows-3.1.1',
-        u'hostname': u'localhost',
-        u'pool': u'default',
-      },
+      'dimensions': [
+        (u'hostname', u'localhost'),
+        (u'os', u'Windows-3.1.1'),
+        (u'pool', u'default'),
+        (u'pool', u'testing'),
+      ],
       'env': {u'foo': u'bar', u'joe': u'2'},
       'execution_timeout_secs': 30,
       'extra_args': [],
       'grace_period_secs': 30,
       'idempotent': False,
       'inputs_ref': {
         'isolated': None,
         'isolatedserver': u'https://isolateserver.appspot.com',
         'namespace': u'default-gzip',
       },
       'io_timeout_secs': None,
       'outputs': [],
       'has_secret_bytes': False,
     }
     # Differences from new_request() are:
     # - parent_task_id was reset to None.
     # - tag 'user:' was replaced
     # - user was replaced.
     expected_request = {
       'authenticated': auth_testing.DEFAULT_MOCKED_IDENTITY,
       'name': u'Request name (Retry #1)',
       'parent_task_id': None,
       'priority': 49,
       'properties': expected_properties,
       'properties_hash': None,
       'pubsub_topic': None,
       'pubsub_userdata': None,
       'service_account': u'none',
       'tags': [
-        u'OS:Windows-3.1.1',
         u'hostname:localhost',
+        u'os:Windows-3.1.1',
         u'pool:default',
+        u'pool:testing',
         u'priority:49',
         u'service_account:none',
         u'tag:1',
         u'user:mocked@example.com',
       ],
       'user': u'mocked@example.com',
     }
     actual = request.to_dict()
     # expiration_ts - created_ts == deadline_to_run.
     actual.pop('created_ts')
@@ skipping 42 matching lines @@
     with self.assertRaises(datastore_errors.BadValueError):
       task_request.SecretBytes(secret_bytes='a'*(20*1024+1)).put()
 
 
 if __name__ == '__main__':
   if '-v' in sys.argv:
     unittest.TestCase.maxDiff = None
   logging.basicConfig(
       level=logging.DEBUG if '-v' in sys.argv else logging.ERROR)
   unittest.main()