Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(293)

Side by Side Diff: scripts/slave/recipe_modules/swarming/api.py

Issue 2102153004: Revert "Add CIPD package support to swarming recipe api" (Closed) Base URL: https://chromium.googlesource.com/chromium/tools/build.git@master
Patch Set: Created 4 years, 5 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
1 # Copyright 2014 The Chromium Authors. All rights reserved. 1 # Copyright 2014 The Chromium Authors. All rights reserved.
2 # Use of this source code is governed by a BSD-style license that can be 2 # Use of this source code is governed by a BSD-style license that can be
3 # found in the LICENSE file. 3 # found in the LICENSE file.
4 4
5 import datetime 5 import datetime
6 import functools 6 import functools
7 7
8 from recipe_engine.types import freeze 8 from recipe_engine.types import freeze
9 from recipe_engine import recipe_api 9 from recipe_engine import recipe_api
10 10
11 11
12 # Minimally supported version of swarming.py script (reported by --version). 12 # Minimally supported version of swarming.py script (reported by --version).
13 MINIMAL_SWARMING_VERSION = (0, 8, 6) 13 MINIMAL_SWARMING_VERSION = (0, 4, 10)
14 14
15 15
16 def parse_time(value): 16 def parse_time(value):
17 """Converts serialized time from the API to datetime.datetime.""" 17 """Converts serialized time from the API to datetime.datetime."""
18 # When microseconds are 0, the '.123456' suffix is elided. This means the 18 # When microseconds are 0, the '.123456' suffix is elided. This means the
19 # serialized format is not consistent, which confuses the hell out of python. 19 # serialized format is not consistent, which confuses the hell out of python.
20 # TODO(maruel): Remove third format once we enforce version >=0.8.2. 20 # TODO(maruel): Remove third format once we enforce version >=0.8.2.
21 for fmt in ('%Y-%m-%dT%H:%M:%S.%f', '%Y-%m-%dT%H:%M:%S', '%Y-%m-%d %H:%M:%S'): 21 for fmt in ('%Y-%m-%dT%H:%M:%S.%f', '%Y-%m-%dT%H:%M:%S', '%Y-%m-%d %H:%M:%S'):
22 try: 22 try:
23 return datetime.datetime.strptime(value, fmt) 23 return datetime.datetime.strptime(value, fmt)
(...skipping 296 matching lines...) Expand 10 before | Expand all | Expand 10 after
320 example WinXP try bot recipe may explicitly specify 'Windows-XP-SP3' 320 example WinXP try bot recipe may explicitly specify 'Windows-XP-SP3'
321 dimension. 321 dimension.
322 """ 322 """
323 return { 323 return {
324 'linux': 'Ubuntu-12.04', 324 'linux': 'Ubuntu-12.04',
325 'mac': 'Mac-10.9', 325 'mac': 'Mac-10.9',
326 'win': 'Windows-7-SP1', 326 'win': 'Windows-7-SP1',
327 }[platform] 327 }[platform]
328 328
329 def task(self, title, isolated_hash, shards=1, task_output_dir=None, 329 def task(self, title, isolated_hash, shards=1, task_output_dir=None,
330 extra_args=None, idempotent=None, cipd_packages=None): 330 extra_args=None, idempotent=None):
331 """Returns a new SwarmingTask instance to run an isolated executable on 331 """Returns a new SwarmingTask instance to run an isolated executable on
332 Swarming. 332 Swarming.
333 333
334 The return value can be customized if necessary (see SwarmingTask class 334 The return value can be customized if necessary (see SwarmingTask class
335 below). Pass it to 'trigger_task' to launch it on swarming. Later pass the 335 below). Pass it to 'trigger_task' to launch it on swarming. Later pass the
336 same instance to 'collect_task' to wait for the task to finish and fetch its 336 same instance to 'collect_task' to wait for the task to finish and fetch its
337 results. 337 results.
338 338
339 Args: 339 Args:
340 title: name of the test, used as part of a task ID. 340 title: name of the test, used as part of a task ID.
341 isolated_hash: hash of isolated test on isolate server, the test should 341 isolated_hash: hash of isolated test on isolate server, the test should
342 be already isolated there, see 'isolate' recipe module. 342 be already isolated there, see 'isolate' recipe module.
343 shards: if defined, the number of shards to use for the task. By default 343 shards: if defined, the number of shards to use for the task. By default
344 this value is either 1 or based on the title. 344 this value is either 1 or based on the title.
345 task_output_dir: if defined, the directory where task results are placed. 345 task_output_dir: if defined, the directory where task results are placed.
346 The caller is responsible for removing this folder when finished. 346 The caller is responsible for removing this folder when finished.
347 extra_args: list of command line arguments to pass to isolated tasks. 347 extra_args: list of command line arguments to pass to isolated tasks.
348 idempotent: whether this task is considered idempotent. Defaults 348 idempotent: whether this task is considered idempotent. Defaults
349 to self.default_idempotent if not specified. 349 to self.default_idempotent if not specified.
350 cipd_packages: list of 3-tuples corresponding to CIPD packages needed for
351 the task: ('path', 'package_name', 'version'), defined as follows:
352 path: Path relative to the Swarming root dir in which to install
353 the package.
354 package_name: Name of the package to install,
355 eg. "infra/tools/authutil/${platform}"
356 version: Version of the package, either a package instance ID,
357 ref, or tag key/value pair.
358 """ 350 """
359 if idempotent is None: 351 if idempotent is None:
360 idempotent = self.default_idempotent 352 idempotent = self.default_idempotent
361 return SwarmingTask( 353 return SwarmingTask(
362 title=title, 354 title=title,
363 isolated_hash=isolated_hash, 355 isolated_hash=isolated_hash,
364 dimensions=self._default_dimensions, 356 dimensions=self._default_dimensions,
365 env=self._default_env, 357 env=self._default_env,
366 priority=self.default_priority, 358 priority=self.default_priority,
367 shards=shards, 359 shards=shards,
368 buildername=self.m.properties.get('buildername'), 360 buildername=self.m.properties.get('buildername'),
369 buildnumber=self.m.properties.get('buildnumber'), 361 buildnumber=self.m.properties.get('buildnumber'),
370 user=self.default_user, 362 user=self.default_user,
371 expiration=self.default_expiration, 363 expiration=self.default_expiration,
372 io_timeout=self.default_io_timeout, 364 io_timeout=self.default_io_timeout,
373 hard_timeout=self.default_hard_timeout, 365 hard_timeout=self.default_hard_timeout,
374 idempotent=idempotent, 366 idempotent=idempotent,
375 extra_args=extra_args, 367 extra_args=extra_args,
376 collect_step=self._default_collect_step, 368 collect_step=self._default_collect_step,
377 task_output_dir=task_output_dir, 369 task_output_dir=task_output_dir)
378 cipd_packages=cipd_packages)
379 370
380 def gtest_task(self, title, isolated_hash, test_launcher_summary_output=None, 371 def gtest_task(self, title, isolated_hash, test_launcher_summary_output=None,
381 extra_args=None, **kwargs): 372 extra_args=None, **kwargs):
382 """Returns a new SwarmingTask instance to run an isolated gtest on Swarming. 373 """Returns a new SwarmingTask instance to run an isolated gtest on Swarming.
383 374
384 Swarming recipe module knows how collect and interpret JSON files with test 375 Swarming recipe module knows how collect and interpret JSON files with test
385 execution summary produced by chromium test launcher. It will combine JSON 376 execution summary produced by chromium test launcher. It will combine JSON
386 results from multiple shards and place it in path provided by 377 results from multiple shards and place it in path provided by
387 |test_launcher_summary_output| placeholder. 378 |test_launcher_summary_output| placeholder.
388 379
(...skipping 120 matching lines...) Expand 10 before | Expand all | Expand 10 after
509 assert ':' in tag, tag 500 assert ':' in tag, tag
510 args.extend(['--tag', tag]) 501 args.extend(['--tag', tag])
511 502
512 if self.verbose: 503 if self.verbose:
513 args.append('--verbose') 504 args.append('--verbose')
514 if task.idempotent: 505 if task.idempotent:
515 args.append('--idempotent') 506 args.append('--idempotent')
516 if task.user: 507 if task.user:
517 args.extend(['--user', task.user]) 508 args.extend(['--user', task.user])
518 509
519 if task.cipd_packages:
520 for path, pkg, version in task.cipd_packages:
521 args.extend(['--cipd-package', '%s:%s:%s' % (path, pkg, version)])
522
523 # What isolated command to trigger. 510 # What isolated command to trigger.
524 args.append(task.isolated_hash) 511 args.append(task.isolated_hash)
525 512
526 # Additional command line args for isolated command. 513 # Additional command line args for isolated command.
527 if task.extra_args: 514 if task.extra_args:
528 args.append('--') 515 args.append('--')
529 args.extend(task.extra_args) 516 args.extend(task.extra_args)
530 517
531 # The step can fail only on infra failures, so mark it as 'infra_step'. 518 # The step can fail only on infra failures, so mark it as 'infra_step'.
532 try: 519 try:
(...skipping 291 matching lines...) Expand 10 before | Expand all | Expand 10 after
824 def get_collect_cmd_args(self, task): 811 def get_collect_cmd_args(self, task):
825 """SwarmingTask -> argument list for 'swarming.py' command.""" 812 """SwarmingTask -> argument list for 'swarming.py' command."""
826 args = [ 813 args = [
827 'collect', 814 'collect',
828 '--swarming', self.swarming_server, 815 '--swarming', self.swarming_server,
829 '--decorate', 816 '--decorate',
830 '--print-status-updates', 817 '--print-status-updates',
831 ] 818 ]
832 if self.verbose: 819 if self.verbose:
833 args.append('--verbose') 820 args.append('--verbose')
834 script_version = self.m.swarming_client.get_script_version('swarming.py') 821 if self.m.swarming_client.get_script_version('swarming.py') < (0, 5):
835 if script_version < (0, 5): # pragma: nocover
836 args.extend(('--shards', str(task.shards))) 822 args.extend(('--shards', str(task.shards)))
837 args.append(task.task_name) 823 args.append(task.task_name)
838 else: 824 else:
839 args.extend(('--json', self.m.json.input(task.trigger_output))) 825 args.extend(('--json', self.m.json.input(task.trigger_output)))
840 return args 826 return args
841 827
842 def _gen_trigger_step_test_data(self, task): 828 def _gen_trigger_step_test_data(self, task):
843 """Generates an expected value of --dump-json in 'trigger' step. 829 """Generates an expected value of --dump-json in 'trigger' step.
844 830
845 Used when running recipes to generate test expectations. 831 Used when running recipes to generate test expectations.
(...skipping 52 matching lines...) Expand 10 before | Expand all | Expand 10 after
898 ], 884 ],
899 }) 885 })
900 886
901 887
902 class SwarmingTask(object): 888 class SwarmingTask(object):
903 """Definition of a task to run on swarming.""" 889 """Definition of a task to run on swarming."""
904 890
905 def __init__(self, title, isolated_hash, dimensions, env, priority, 891 def __init__(self, title, isolated_hash, dimensions, env, priority,
906 shards, buildername, buildnumber, expiration, user, io_timeout, 892 shards, buildername, buildnumber, expiration, user, io_timeout,
907 hard_timeout, idempotent, extra_args, collect_step, 893 hard_timeout, idempotent, extra_args, collect_step,
908 task_output_dir, cipd_packages=None): 894 task_output_dir):
909 """Configuration of a swarming task. 895 """Configuration of a swarming task.
910 896
911 Args: 897 Args:
912 title: display name of the task, hints to what task is doing. Usually 898 title: display name of the task, hints to what task is doing. Usually
913 corresponds to a name of a test executable. Doesn't have to be unique. 899 corresponds to a name of a test executable. Doesn't have to be unique.
914 isolated_hash: hash of isolated file that describes all files needed to 900 isolated_hash: hash of isolated file that describes all files needed to
915 run the task as well as command line to launch. See 'isolate' recipe 901 run the task as well as command line to launch. See 'isolate' recipe
916 module. 902 module.
917 cipd_packages: list of 3-tuples corresponding to CIPD packages needed for
918 the task: ('path', 'package_name', 'version'), defined as follows:
919 path: Path relative to the Swarming root dir in which to install
920 the package.
921 package_name: Name of the package to install,
922 eg. "infra/tools/authutil/${platform}"
923 version: Version of the package, either a package instance ID,
924 ref, or tag key/value pair.
925 collect_step: callback that will be called to collect and processes
926 results of task execution, signature is collect_step(task, **kwargs).
927 dimensions: key-value mapping with swarming dimensions that specify 903 dimensions: key-value mapping with swarming dimensions that specify
928 on what Swarming slaves task can run. One important dimension is 'os', 904 on what Swarming slaves task can run. One important dimension is 'os',
929 which defines platform flavor to run the task on. See Swarming doc. 905 which defines platform flavor to run the task on. See Swarming doc.
930 env: key-value mapping with additional environment variables to add to 906 env: key-value mapping with additional environment variables to add to
931 environment before launching the task executable. 907 environment before launching the task executable.
932 priority: integer [0, 255] that defines how urgent the task is. 908 priority: integer [0, 255] that defines how urgent the task is.
933 Lower value corresponds to higher priority. Swarming service executes 909 Lower value corresponds to higher priority. Swarming service executes
934 tasks with higher priority first. 910 tasks with higher priority first.
935 shards: how many concurrent shards to run, makes sense only for 911 shards: how many concurrent shards to run, makes sense only for
936 isolated tests based on gtest. Swarming uses GTEST_SHARD_INDEX 912 isolated tests based on gtest. Swarming uses GTEST_SHARD_INDEX
937 and GTEST_TOTAL_SHARDS environment variables to tell the executable 913 and GTEST_TOTAL_SHARDS environment variables to tell the executable
938 what shard to run. 914 what shard to run.
939 buildername: buildbot builder this task was triggered from. 915 buildername: buildbot builder this task was triggered from.
940 buildnumber: build number of a build this task was triggered from. 916 buildnumber: build number of a build this task was triggered from.
941 expiration: number of schedule until the task shouldn't even be run if it 917 expiration: number of schedule until the task shouldn't even be run if it
942 hadn't started yet. 918 hadn't started yet.
943 user: user that requested this task, if applicable. 919 user: user that requested this task, if applicable.
944 io_timeout: number of seconds that the task is allowed to not emit any 920 io_timeout: number of seconds that the task is allowed to not emit any
945 stdout bytes, after which it is forcibly killed. 921 stdout bytes, after which it is forcibly killed.
946 hard_timeout: number of seconds for which the task is allowed to run, 922 hard_timeout: number of seconds for which the task is allowed to run,
947 after which it is forcibly killed. 923 after which it is forcibly killed.
948 idempotent: True if the results from a previous task can be reused. E.g. 924 idempotent: True if the results from a previous task can be reused. E.g.
949 this task has no side-effects. 925 this task has no side-effects.
950 extra_args: list of command line arguments to pass to isolated tasks. 926 extra_args: list of command line arguments to pass to isolated tasks.
927 collect_step: callback that will be called to collect and processes
928 results of task execution, signature is collect_step(task, **kwargs).
951 task_output_dir: if defined, the directory where task results are placed 929 task_output_dir: if defined, the directory where task results are placed
952 during the collect step. 930 during the collect step.
953 """ 931 """
954 self._trigger_output = None 932 self._trigger_output = None
955 self.buildnumber = buildnumber 933 self.buildnumber = buildnumber
956 self.buildername = buildername 934 self.buildername = buildername
957 self.cipd_packages = cipd_packages 935 self.task_output_dir = task_output_dir
958 self.collect_step = collect_step 936 self.collect_step = collect_step
959 self.dimensions = dimensions.copy() 937 self.dimensions = dimensions.copy()
960 self.env = env.copy() 938 self.env = env.copy()
961 self.expiration = expiration 939 self.expiration = expiration
962 self.extra_args = tuple(extra_args or []) 940 self.extra_args = tuple(extra_args or [])
963 self.hard_timeout = hard_timeout 941 self.hard_timeout = hard_timeout
964 self.idempotent = idempotent 942 self.idempotent = idempotent
965 self.io_timeout = io_timeout 943 self.io_timeout = io_timeout
966 self.isolated_hash = isolated_hash 944 self.isolated_hash = isolated_hash
967 self.priority = priority 945 self.priority = priority
968 self.shards = shards 946 self.shards = shards
969 self.tags = set() 947 self.tags = set()
970 self.task_output_dir = task_output_dir
971 self.title = title 948 self.title = title
972 self.user = user 949 self.user = user
973 950
974 @property 951 @property
975 def task_name(self): 952 def task_name(self):
976 """Name of this task, derived from its other properties. 953 """Name of this task, derived from its other properties.
977 954
978 The task name is purely to make sense of the task and is not used in any 955 The task name is purely to make sense of the task and is not used in any
979 other way. 956 other way.
980 """ 957 """
(...skipping 10 matching lines...) Expand all
991 968
992 def get_shard_view_url(self, index): 969 def get_shard_view_url(self, index):
993 """Returns URL of HTML page with shard details or None if not available. 970 """Returns URL of HTML page with shard details or None if not available.
994 971
995 Works only after the task has been successfully triggered. 972 Works only after the task has been successfully triggered.
996 """ 973 """
997 if self._trigger_output and self._trigger_output.get('tasks'): 974 if self._trigger_output and self._trigger_output.get('tasks'):
998 for shard_dict in self._trigger_output['tasks'].itervalues(): 975 for shard_dict in self._trigger_output['tasks'].itervalues():
999 if shard_dict['shard_index'] == index: 976 if shard_dict['shard_index'] == index:
1000 return shard_dict['view_url'] 977 return shard_dict['view_url']
OLDNEW

Powered by Google App Engine
This is Rietveld 408576698