swarming: refactor cipd input

appengine/swarming/handlers_endpoints_test.py

 #!/usr/bin/env python
 # coding=utf-8
 # Copyright 2015 The LUCI Authors. All rights reserved.
 # Use of this source code is governed by the Apache v2.0 license that can be
 # found in the LICENSE file.
 
 import base64
 import datetime
 import json
 import logging
@@ skipping 19 matching lines @@
 
 from server import acl
 from server import bot_management
 from server import config
 from server import large
 from server import task_pack
 from server import task_request
 from server import task_result
 
 
+CIPD_SETTINGS = test_env_handlers.CIPD_SETTINGS
+
+
 def message_to_dict(rpc_message):
   return json.loads(protojson.encode_message(rpc_message))
 
 
 class BaseTest(test_env_handlers.AppTestBase, test_case.EndpointsTestCase):
 
   DATETIME_FORMAT = '%Y-%m-%dT%H:%M:%S.%f'
   DATETIME_NO_MICRO = '%Y-%m-%dT%H:%M:%S'
 
   def setUp(self):
@@ skipping 94 matching lines @@
     """Asserts that new generates appropriate metadata."""
     self.mock(random, 'getrandbits', lambda _: 0x88)
     now = datetime.datetime(2010, 1, 2, 3, 4, 5)
     self.mock_now(now)
     str_now = unicode(now.strftime(self.DATETIME_NO_MICRO))
     request = swarming_rpcs.NewTaskRequest(
         expiration_secs=30,
         name='job1',
         priority=200,
         properties=swarming_rpcs.TaskProperties(
+            cipd_input=swarming_rpcs.CipdInput(
+                packages=[
+                  swarming_rpcs.CipdPackage(
+                      package_name='rm', version='latest'),
+                ],
+                settings=CIPD_SETTINGS.SerializeToString(),
+            ),
             command=['rm', '-rf', '/'],
-            packages=[
-              swarming_rpcs.CipdPackage(package_name='rm', version='latest'),
-            ],
             dimensions=[
               swarming_rpcs.StringPair(key='pool', value='default'),
             ],
             env=[
               swarming_rpcs.StringPair(key='PATH', value='/'),
             ],
             execution_timeout_secs=30,
             io_timeout_secs=30),
         tags=['foo:bar'],
         user='joe@localhost',
         pubsub_topic='projects/abc/topics/def',
         pubsub_auth_token='secret that must not be shown',
         pubsub_userdata='userdata')
     expected = {
       u'request': {
         u'authenticated': u'user:user@example.com',
         u'created_ts': str_now,
         u'expiration_secs': u'30',
         u'name': u'job1',
         u'priority': u'200',
         u'properties': {
+          u'cipd_input': {
+            u'packages': [{
+              u'package_name': u'rm',
+              u'version': u'latest',
+            }],
+            u'settings': base64.b64encode(CIPD_SETTINGS.SerializeToString()),
+          },
           u'command': [u'rm', u'-rf', u'/'],
-          u'packages': [{
-            u'package_name': u'rm',
-            u'version': u'latest',
-          }],
           u'dimensions': [
             {u'key': u'pool', u'value': u'default'},
           ],
           u'env': [
             {u'key': u'PATH', u'value': u'/'},
           ],
           u'execution_timeout_secs': u'30',
           u'grace_period_secs': u'30',
           u'idempotent': False,
           u'io_timeout_secs': u'30',
@@ skipping 29 matching lines @@
     now_30 = self.mock_now(now, 30)
     str_now_30 = unicode(now_30.strftime(self.DATETIME_NO_MICRO))
     self.set_as_user()
 
     request = swarming_rpcs.NewTaskRequest(
         expiration_secs=30,
         name='job1',
         priority=200,
         properties=swarming_rpcs.TaskProperties(
             command=['python', 'run_test.py'],
+            cipd_input=swarming_rpcs.CipdInput(
+                packages=[
+                  swarming_rpcs.CipdPackage(
+                      package_name='rm',
+                      version=test_env_handlers.PINNED_PACKAGE_VERSION),
+                ],
+                settings=CIPD_SETTINGS.SerializeToString(),
+            ),
             dimensions=[
               swarming_rpcs.StringPair(key='os', value='Amiga'),
               swarming_rpcs.StringPair(key='pool', value='default'),
             ],
             execution_timeout_secs=3600,
             idempotent=True,
-            io_timeout_secs=1200,
-            packages=[
-              swarming_rpcs.CipdPackage(
-                  package_name='rm',
-                  version=test_env_handlers.PINNED_PACKAGE_VERSION),
-            ]),
+            io_timeout_secs=1200),
         tags=['foo:bar'],
         user='joe@localhost')
     expected = {
       u'request': {
         u'authenticated': u'user:user@example.com',
         u'created_ts': str_now_30,
         u'expiration_secs': u'30',
         u'name': u'job1',
         u'priority': u'200',
         u'properties': {
+          u'cipd_input': {
+            u'packages': [{
+              u'package_name': u'rm',
+              u'version': test_env_handlers.PINNED_PACKAGE_VERSION,
+            }],
+            u'settings': base64.b64encode(CIPD_SETTINGS.SerializeToString()),
+          },
           u'command': [u'python', u'run_test.py'],
           u'dimensions': [
             {u'key': u'os', u'value': u'Amiga'},
             {u'key': u'pool', u'value': u'default'},
           ],
           u'execution_timeout_secs': u'3600',
           u'grace_period_secs': u'30',
           u'idempotent': True,
           u'io_timeout_secs': u'1200',
-          u'packages': [{
-            u'package_name': u'rm',
-            u'version': test_env_handlers.PINNED_PACKAGE_VERSION,
-          }],
         },
         u'tags': [
           u'foo:bar',
           u'os:Amiga',
           u'pool:default',
           u'priority:200',
           u'user:joe@localhost',
         ],
         u'user': u'joe@localhost',
       },
@@ skipping 98 matching lines @@
     request = swarming_rpcs.TasksRequest(start=start, end=end)
     expected = {
       u'items': [
         {
           u'authenticated': u'user:user@example.com',
           u'created_ts': str_now_30,
           u'expiration_secs': u'30',
           u'name': u'job1',
           u'priority': u'200',
           u'properties': {
+            u'cipd_input': {
+              u'packages': [{
+                u'package_name': u'rm',
+                u'version': test_env_handlers.PINNED_PACKAGE_VERSION,
+              }],
+              u'settings': base64.b64encode(CIPD_SETTINGS.SerializeToString()),
+            },
             u'command': [u'python', u'run_test.py'],
             u'dimensions': [
               {u'key': u'os', u'value': u'Amiga'},
               {u'key': u'pool', u'value': u'default'},
             ],
             u'execution_timeout_secs': u'3600',
             u'grace_period_secs': u'30',
             u'idempotent': True,
             u'io_timeout_secs': u'1200',
-            u'packages': [{
-              u'package_name': u'rm',
-              u'version': test_env_handlers.PINNED_PACKAGE_VERSION,
-            }],
           },
           u'tags': [
             u'foo:bar',
             u'os:Amiga',
             u'pool:default',
             u'priority:200',
             u'user:joe@localhost',
           ],
           u'user': u'joe@localhost',
         },
         {
           u'authenticated': u'user:user@example.com',
           u'created_ts': str_now,
           u'expiration_secs': u'86400',
           u'name': u'task',
           u'priority': u'10',
           u'properties': {
+            u'cipd_input': {
+              u'packages': [{
+                u'package_name': u'rm',
+                u'version': test_env_handlers.PINNED_PACKAGE_VERSION,
+              }],
+              u'settings': base64.b64encode(CIPD_SETTINGS.SerializeToString()),
+            },
             u'command': [u'python', u'run_test.py'],
             u'dimensions': [
               {u'key': u'os', u'value': u'Amiga'},
               {u'key': u'pool', u'value': u'default'},
             ],
             u'execution_timeout_secs': u'3600',
             u'grace_period_secs': u'30',
             u'idempotent': True,
             u'io_timeout_secs': u'1200',
-            u'packages': [{
-              u'package_name': u'rm',
-              u'version': test_env_handlers.PINNED_PACKAGE_VERSION,
-            }],
           },
           u'tags': [
             u'commit:post',
             u'os:Amiga',
             u'os:Win',
             u'pool:default',
             u'priority:10',
             u'project:yay',
             u'user:joe@localhost',
           ],
@@ skipping 129 matching lines @@
           u'priority:200',
           u'user:joe@localhost',
         ],
         u'user': u'joe@localhost',
       },
       u'task_id': u'5cee488008810',
     }
     response = self.call_api('new', body=message_to_dict(request))
     self.assertEqual(expected, response.json)
 
+  def test_new_cipd_packages_with_defaults(self):
+    self.mock(random, 'getrandbits', lambda _: 0x88)
+    now = datetime.datetime(2010, 1, 2, 3, 4, 5)
+    self.mock_now(now)
+    str_now = unicode(now.strftime(self.DATETIME_NO_MICRO))
+
+    # Define settings on the server.
+    cfg = config.settings()
+    cfg.cipd.CopyFrom(CIPD_SETTINGS)
+    self.mock(config, 'settings', lambda: cfg)
+
+    request = swarming_rpcs.NewTaskRequest(
+        expiration_secs=30,
+        name='job1',
+        priority=200,
+        properties=swarming_rpcs.TaskProperties(
+            cipd_input=swarming_rpcs.CipdInput(
+                packages=[
+                  swarming_rpcs.CipdPackage(
+                      package_name='rm',
+                      version='latest'),
+                ],
+            ),
+            command=['rm', '-rf', '/'],
+            dimensions=[
+              swarming_rpcs.StringPair(key='pool', value='default'),
+            ],
+            env=[
+              swarming_rpcs.StringPair(key='PATH', value='/'),
+            ],
+            execution_timeout_secs=30,
+            io_timeout_secs=30),
+        tags=['foo:bar'],
+        user='joe@localhost',
+        pubsub_topic='projects/abc/topics/def',
+        pubsub_auth_token='secret that must not be shown',
+        pubsub_userdata='userdata')
+    expected = {
+      u'request': {
+        u'authenticated': u'user:user@example.com',
+        u'created_ts': str_now,
+        u'expiration_secs': u'30',
+        u'name': u'job1',
+        u'priority': u'200',
+        u'properties': {
+          u'cipd_input': {
+            u'packages': [{
+              u'package_name': u'rm',
+              u'version': u'latest',
+            }],
+            u'settings': base64.b64encode(CIPD_SETTINGS.SerializeToString()),
+          },
+          u'command': [u'rm', u'-rf', u'/'],
+          u'dimensions': [
+            {u'key': u'pool', u'value': u'default'},
+          ],
+          u'env': [
+            {u'key': u'PATH', u'value': u'/'},
+          ],
+          u'execution_timeout_secs': u'30',
+          u'grace_period_secs': u'30',
+          u'idempotent': False,
+          u'io_timeout_secs': u'30',
+        },
+        u'pubsub_topic': u'projects/abc/topics/def',
+        u'pubsub_userdata': u'userdata',
+        u'tags': [
+          u'foo:bar',
+          u'pool:default',
+          u'priority:200',
+          u'user:joe@localhost',
+        ],
+        u'user': u'joe@localhost',
+      },
+      u'task_id': u'5cee488008810',
+    }
+    response = self.call_api('new', body=message_to_dict(request))
+    self.assertEqual(expected, response.json)
+
   def test_list_ok(self):
     """Asserts that list requests all TaskResultSummaries."""
     first, second, str_now_120, start, end = self._gen_two_tasks()
     first_no_perf = first.copy()
     first_no_perf.pop('performance_stats')
     # Basic request.
     request = swarming_rpcs.TasksRequest(
         end=end, start=start, include_performance_stats=True)
     expected = {u'now': str_now_120, u'items': [second, first]}
     actual = self.call_api('list', body=message_to_dict(request)).json
@@ skipping 517 matching lines @@
     _, task_id = self.client_create_task_raw()
     self.set_as_bot()
     response = self.call_api('request', body={'task_id': task_id})
     expected = {
       u'authenticated': u'user:user@example.com',
       u'created_ts': unicode(now.strftime(self.DATETIME_FORMAT)),
       u'expiration_secs': unicode(24 * 60 * 60),
       u'name': u'hi',
       u'priority': u'10',
       u'properties': {
+        u'cipd_input': {
+          u'packages': [{
+            u'package_name': u'rm',
+            u'version': test_env_handlers.PINNED_PACKAGE_VERSION,
+          }],
+          u'settings': base64.b64encode(CIPD_SETTINGS.SerializeToString()),
+        },
         u'command': [u'python', u'run_test.py'],
         u'dimensions': [
           {u'key': u'os', u'value': u'Amiga'},
           {u'key': u'pool', u'value': u'default'},
         ],
         u'execution_timeout_secs': u'3600',
         u'grace_period_secs': u'30',
         u'idempotent': False,
         u'io_timeout_secs': u'1200',
-        u'packages': [{
-          u'package_name': u'rm',
-          u'version': test_env_handlers.PINNED_PACKAGE_VERSION,
-        }],
       },
       u'tags': [
         u'os:Amiga',
         u'pool:default',
         u'priority:10',
         u'user:joe@localhost',
       ],
       u'user': u'joe@localhost',
     }
     self.assertEqual(expected, response.json)
@@ skipping 295 matching lines @@
     self.assertEqual(expected, response.json)
 
 
 if __name__ == '__main__':
   if '-v' in sys.argv:
     unittest.TestCase.maxDiff = None
     logging.basicConfig(level=logging.DEBUG)
   else:
     logging.basicConfig(level=logging.CRITICAL)
   unittest.main()