| OLD | NEW |
| 1 # Copyright 2014 The Chromium Authors. All rights reserved. | 1 # Copyright 2014 The Chromium Authors. All rights reserved. |
| 2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
| 3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
| 4 | 4 |
| 5 import datetime | 5 import datetime |
| 6 import functools | 6 import functools |
| 7 | 7 |
| 8 from recipe_engine.types import freeze | 8 from recipe_engine.types import freeze |
| 9 from recipe_engine import recipe_api | 9 from recipe_engine import recipe_api |
| 10 | 10 |
| 11 | 11 |
| 12 # Minimally supported version of swarming.py script (reported by --version). | 12 # Minimally supported version of swarming.py script (reported by --version). |
| 13 MINIMAL_SWARMING_VERSION = (0, 8, 6) | 13 MINIMAL_SWARMING_VERSION = (0, 4, 10) |
| 14 | 14 |
| 15 | 15 |
| 16 def parse_time(value): | 16 def parse_time(value): |
| 17 """Converts serialized time from the API to datetime.datetime.""" | 17 """Converts serialized time from the API to datetime.datetime.""" |
| 18 # When microseconds are 0, the '.123456' suffix is elided. This means the | 18 # When microseconds are 0, the '.123456' suffix is elided. This means the |
| 19 # serialized format is not consistent, which confuses the hell out of python. | 19 # serialized format is not consistent, which confuses the hell out of python. |
| 20 # TODO(maruel): Remove third format once we enforce version >=0.8.2. | 20 # TODO(maruel): Remove third format once we enforce version >=0.8.2. |
| 21 for fmt in ('%Y-%m-%dT%H:%M:%S.%f', '%Y-%m-%dT%H:%M:%S', '%Y-%m-%d %H:%M:%S'): | 21 for fmt in ('%Y-%m-%dT%H:%M:%S.%f', '%Y-%m-%dT%H:%M:%S', '%Y-%m-%d %H:%M:%S'): |
| 22 try: | 22 try: |
| 23 return datetime.datetime.strptime(value, fmt) | 23 return datetime.datetime.strptime(value, fmt) |
| (...skipping 296 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 320 example WinXP try bot recipe may explicitly specify 'Windows-XP-SP3' | 320 example WinXP try bot recipe may explicitly specify 'Windows-XP-SP3' |
| 321 dimension. | 321 dimension. |
| 322 """ | 322 """ |
| 323 return { | 323 return { |
| 324 'linux': 'Ubuntu-12.04', | 324 'linux': 'Ubuntu-12.04', |
| 325 'mac': 'Mac-10.9', | 325 'mac': 'Mac-10.9', |
| 326 'win': 'Windows-7-SP1', | 326 'win': 'Windows-7-SP1', |
| 327 }[platform] | 327 }[platform] |
| 328 | 328 |
| 329 def task(self, title, isolated_hash, shards=1, task_output_dir=None, | 329 def task(self, title, isolated_hash, shards=1, task_output_dir=None, |
| 330 extra_args=None, idempotent=None, cipd_packages=None): | 330 extra_args=None, idempotent=None): |
| 331 """Returns a new SwarmingTask instance to run an isolated executable on | 331 """Returns a new SwarmingTask instance to run an isolated executable on |
| 332 Swarming. | 332 Swarming. |
| 333 | 333 |
| 334 The return value can be customized if necessary (see SwarmingTask class | 334 The return value can be customized if necessary (see SwarmingTask class |
| 335 below). Pass it to 'trigger_task' to launch it on swarming. Later pass the | 335 below). Pass it to 'trigger_task' to launch it on swarming. Later pass the |
| 336 same instance to 'collect_task' to wait for the task to finish and fetch its | 336 same instance to 'collect_task' to wait for the task to finish and fetch its |
| 337 results. | 337 results. |
| 338 | 338 |
| 339 Args: | 339 Args: |
| 340 title: name of the test, used as part of a task ID. | 340 title: name of the test, used as part of a task ID. |
| 341 isolated_hash: hash of isolated test on isolate server, the test should | 341 isolated_hash: hash of isolated test on isolate server, the test should |
| 342 be already isolated there, see 'isolate' recipe module. | 342 be already isolated there, see 'isolate' recipe module. |
| 343 shards: if defined, the number of shards to use for the task. By default | 343 shards: if defined, the number of shards to use for the task. By default |
| 344 this value is either 1 or based on the title. | 344 this value is either 1 or based on the title. |
| 345 task_output_dir: if defined, the directory where task results are placed. | 345 task_output_dir: if defined, the directory where task results are placed. |
| 346 The caller is responsible for removing this folder when finished. | 346 The caller is responsible for removing this folder when finished. |
| 347 extra_args: list of command line arguments to pass to isolated tasks. | 347 extra_args: list of command line arguments to pass to isolated tasks. |
| 348 idempotent: whether this task is considered idempotent. Defaults | 348 idempotent: whether this task is considered idempotent. Defaults |
| 349 to self.default_idempotent if not specified. | 349 to self.default_idempotent if not specified. |
| 350 cipd_packages: list of 3-tuples corresponding to CIPD packages needed for | |
| 351 the task: ('path', 'package_name', 'version'), defined as follows: | |
| 352 path: Path relative to the Swarming root dir in which to install | |
| 353 the package. | |
| 354 package_name: Name of the package to install, | |
| 355 eg. "infra/tools/authutil/${platform}" | |
| 356 version: Version of the package, either a package instance ID, | |
| 357 ref, or tag key/value pair. | |
| 358 """ | 350 """ |
| 359 if idempotent is None: | 351 if idempotent is None: |
| 360 idempotent = self.default_idempotent | 352 idempotent = self.default_idempotent |
| 361 return SwarmingTask( | 353 return SwarmingTask( |
| 362 title=title, | 354 title=title, |
| 363 isolated_hash=isolated_hash, | 355 isolated_hash=isolated_hash, |
| 364 dimensions=self._default_dimensions, | 356 dimensions=self._default_dimensions, |
| 365 env=self._default_env, | 357 env=self._default_env, |
| 366 priority=self.default_priority, | 358 priority=self.default_priority, |
| 367 shards=shards, | 359 shards=shards, |
| 368 buildername=self.m.properties.get('buildername'), | 360 buildername=self.m.properties.get('buildername'), |
| 369 buildnumber=self.m.properties.get('buildnumber'), | 361 buildnumber=self.m.properties.get('buildnumber'), |
| 370 user=self.default_user, | 362 user=self.default_user, |
| 371 expiration=self.default_expiration, | 363 expiration=self.default_expiration, |
| 372 io_timeout=self.default_io_timeout, | 364 io_timeout=self.default_io_timeout, |
| 373 hard_timeout=self.default_hard_timeout, | 365 hard_timeout=self.default_hard_timeout, |
| 374 idempotent=idempotent, | 366 idempotent=idempotent, |
| 375 extra_args=extra_args, | 367 extra_args=extra_args, |
| 376 collect_step=self._default_collect_step, | 368 collect_step=self._default_collect_step, |
| 377 task_output_dir=task_output_dir, | 369 task_output_dir=task_output_dir) |
| 378 cipd_packages=cipd_packages) | |
| 379 | 370 |
| 380 def gtest_task(self, title, isolated_hash, test_launcher_summary_output=None, | 371 def gtest_task(self, title, isolated_hash, test_launcher_summary_output=None, |
| 381 extra_args=None, **kwargs): | 372 extra_args=None, **kwargs): |
| 382 """Returns a new SwarmingTask instance to run an isolated gtest on Swarming. | 373 """Returns a new SwarmingTask instance to run an isolated gtest on Swarming. |
| 383 | 374 |
| 384 Swarming recipe module knows how collect and interpret JSON files with test | 375 Swarming recipe module knows how collect and interpret JSON files with test |
| 385 execution summary produced by chromium test launcher. It will combine JSON | 376 execution summary produced by chromium test launcher. It will combine JSON |
| 386 results from multiple shards and place it in path provided by | 377 results from multiple shards and place it in path provided by |
| 387 |test_launcher_summary_output| placeholder. | 378 |test_launcher_summary_output| placeholder. |
| 388 | 379 |
| (...skipping 120 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 509 assert ':' in tag, tag | 500 assert ':' in tag, tag |
| 510 args.extend(['--tag', tag]) | 501 args.extend(['--tag', tag]) |
| 511 | 502 |
| 512 if self.verbose: | 503 if self.verbose: |
| 513 args.append('--verbose') | 504 args.append('--verbose') |
| 514 if task.idempotent: | 505 if task.idempotent: |
| 515 args.append('--idempotent') | 506 args.append('--idempotent') |
| 516 if task.user: | 507 if task.user: |
| 517 args.extend(['--user', task.user]) | 508 args.extend(['--user', task.user]) |
| 518 | 509 |
| 519 if task.cipd_packages: | |
| 520 for path, pkg, version in task.cipd_packages: | |
| 521 args.extend(['--cipd-package', '%s:%s:%s' % (path, pkg, version)]) | |
| 522 | |
| 523 # What isolated command to trigger. | 510 # What isolated command to trigger. |
| 524 args.append(task.isolated_hash) | 511 args.append(task.isolated_hash) |
| 525 | 512 |
| 526 # Additional command line args for isolated command. | 513 # Additional command line args for isolated command. |
| 527 if task.extra_args: | 514 if task.extra_args: |
| 528 args.append('--') | 515 args.append('--') |
| 529 args.extend(task.extra_args) | 516 args.extend(task.extra_args) |
| 530 | 517 |
| 531 # The step can fail only on infra failures, so mark it as 'infra_step'. | 518 # The step can fail only on infra failures, so mark it as 'infra_step'. |
| 532 try: | 519 try: |
| (...skipping 291 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 824 def get_collect_cmd_args(self, task): | 811 def get_collect_cmd_args(self, task): |
| 825 """SwarmingTask -> argument list for 'swarming.py' command.""" | 812 """SwarmingTask -> argument list for 'swarming.py' command.""" |
| 826 args = [ | 813 args = [ |
| 827 'collect', | 814 'collect', |
| 828 '--swarming', self.swarming_server, | 815 '--swarming', self.swarming_server, |
| 829 '--decorate', | 816 '--decorate', |
| 830 '--print-status-updates', | 817 '--print-status-updates', |
| 831 ] | 818 ] |
| 832 if self.verbose: | 819 if self.verbose: |
| 833 args.append('--verbose') | 820 args.append('--verbose') |
| 834 script_version = self.m.swarming_client.get_script_version('swarming.py') | 821 if self.m.swarming_client.get_script_version('swarming.py') < (0, 5): |
| 835 if script_version < (0, 5): # pragma: nocover | |
| 836 args.extend(('--shards', str(task.shards))) | 822 args.extend(('--shards', str(task.shards))) |
| 837 args.append(task.task_name) | 823 args.append(task.task_name) |
| 838 else: | 824 else: |
| 839 args.extend(('--json', self.m.json.input(task.trigger_output))) | 825 args.extend(('--json', self.m.json.input(task.trigger_output))) |
| 840 return args | 826 return args |
| 841 | 827 |
| 842 def _gen_trigger_step_test_data(self, task): | 828 def _gen_trigger_step_test_data(self, task): |
| 843 """Generates an expected value of --dump-json in 'trigger' step. | 829 """Generates an expected value of --dump-json in 'trigger' step. |
| 844 | 830 |
| 845 Used when running recipes to generate test expectations. | 831 Used when running recipes to generate test expectations. |
| (...skipping 52 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 898 ], | 884 ], |
| 899 }) | 885 }) |
| 900 | 886 |
| 901 | 887 |
| 902 class SwarmingTask(object): | 888 class SwarmingTask(object): |
| 903 """Definition of a task to run on swarming.""" | 889 """Definition of a task to run on swarming.""" |
| 904 | 890 |
| 905 def __init__(self, title, isolated_hash, dimensions, env, priority, | 891 def __init__(self, title, isolated_hash, dimensions, env, priority, |
| 906 shards, buildername, buildnumber, expiration, user, io_timeout, | 892 shards, buildername, buildnumber, expiration, user, io_timeout, |
| 907 hard_timeout, idempotent, extra_args, collect_step, | 893 hard_timeout, idempotent, extra_args, collect_step, |
| 908 task_output_dir, cipd_packages=None): | 894 task_output_dir): |
| 909 """Configuration of a swarming task. | 895 """Configuration of a swarming task. |
| 910 | 896 |
| 911 Args: | 897 Args: |
| 912 title: display name of the task, hints to what task is doing. Usually | 898 title: display name of the task, hints to what task is doing. Usually |
| 913 corresponds to a name of a test executable. Doesn't have to be unique. | 899 corresponds to a name of a test executable. Doesn't have to be unique. |
| 914 isolated_hash: hash of isolated file that describes all files needed to | 900 isolated_hash: hash of isolated file that describes all files needed to |
| 915 run the task as well as command line to launch. See 'isolate' recipe | 901 run the task as well as command line to launch. See 'isolate' recipe |
| 916 module. | 902 module. |
| 917 cipd_packages: list of 3-tuples corresponding to CIPD packages needed for | |
| 918 the task: ('path', 'package_name', 'version'), defined as follows: | |
| 919 path: Path relative to the Swarming root dir in which to install | |
| 920 the package. | |
| 921 package_name: Name of the package to install, | |
| 922 eg. "infra/tools/authutil/${platform}" | |
| 923 version: Version of the package, either a package instance ID, | |
| 924 ref, or tag key/value pair. | |
| 925 collect_step: callback that will be called to collect and processes | |
| 926 results of task execution, signature is collect_step(task, **kwargs). | |
| 927 dimensions: key-value mapping with swarming dimensions that specify | 903 dimensions: key-value mapping with swarming dimensions that specify |
| 928 on what Swarming slaves task can run. One important dimension is 'os', | 904 on what Swarming slaves task can run. One important dimension is 'os', |
| 929 which defines platform flavor to run the task on. See Swarming doc. | 905 which defines platform flavor to run the task on. See Swarming doc. |
| 930 env: key-value mapping with additional environment variables to add to | 906 env: key-value mapping with additional environment variables to add to |
| 931 environment before launching the task executable. | 907 environment before launching the task executable. |
| 932 priority: integer [0, 255] that defines how urgent the task is. | 908 priority: integer [0, 255] that defines how urgent the task is. |
| 933 Lower value corresponds to higher priority. Swarming service executes | 909 Lower value corresponds to higher priority. Swarming service executes |
| 934 tasks with higher priority first. | 910 tasks with higher priority first. |
| 935 shards: how many concurrent shards to run, makes sense only for | 911 shards: how many concurrent shards to run, makes sense only for |
| 936 isolated tests based on gtest. Swarming uses GTEST_SHARD_INDEX | 912 isolated tests based on gtest. Swarming uses GTEST_SHARD_INDEX |
| 937 and GTEST_TOTAL_SHARDS environment variables to tell the executable | 913 and GTEST_TOTAL_SHARDS environment variables to tell the executable |
| 938 what shard to run. | 914 what shard to run. |
| 939 buildername: buildbot builder this task was triggered from. | 915 buildername: buildbot builder this task was triggered from. |
| 940 buildnumber: build number of a build this task was triggered from. | 916 buildnumber: build number of a build this task was triggered from. |
| 941 expiration: number of schedule until the task shouldn't even be run if it | 917 expiration: number of schedule until the task shouldn't even be run if it |
| 942 hadn't started yet. | 918 hadn't started yet. |
| 943 user: user that requested this task, if applicable. | 919 user: user that requested this task, if applicable. |
| 944 io_timeout: number of seconds that the task is allowed to not emit any | 920 io_timeout: number of seconds that the task is allowed to not emit any |
| 945 stdout bytes, after which it is forcibly killed. | 921 stdout bytes, after which it is forcibly killed. |
| 946 hard_timeout: number of seconds for which the task is allowed to run, | 922 hard_timeout: number of seconds for which the task is allowed to run, |
| 947 after which it is forcibly killed. | 923 after which it is forcibly killed. |
| 948 idempotent: True if the results from a previous task can be reused. E.g. | 924 idempotent: True if the results from a previous task can be reused. E.g. |
| 949 this task has no side-effects. | 925 this task has no side-effects. |
| 950 extra_args: list of command line arguments to pass to isolated tasks. | 926 extra_args: list of command line arguments to pass to isolated tasks. |
| 927 collect_step: callback that will be called to collect and processes |
| 928 results of task execution, signature is collect_step(task, **kwargs). |
| 951 task_output_dir: if defined, the directory where task results are placed | 929 task_output_dir: if defined, the directory where task results are placed |
| 952 during the collect step. | 930 during the collect step. |
| 953 """ | 931 """ |
| 954 self._trigger_output = None | 932 self._trigger_output = None |
| 955 self.buildnumber = buildnumber | 933 self.buildnumber = buildnumber |
| 956 self.buildername = buildername | 934 self.buildername = buildername |
| 957 self.cipd_packages = cipd_packages | 935 self.task_output_dir = task_output_dir |
| 958 self.collect_step = collect_step | 936 self.collect_step = collect_step |
| 959 self.dimensions = dimensions.copy() | 937 self.dimensions = dimensions.copy() |
| 960 self.env = env.copy() | 938 self.env = env.copy() |
| 961 self.expiration = expiration | 939 self.expiration = expiration |
| 962 self.extra_args = tuple(extra_args or []) | 940 self.extra_args = tuple(extra_args or []) |
| 963 self.hard_timeout = hard_timeout | 941 self.hard_timeout = hard_timeout |
| 964 self.idempotent = idempotent | 942 self.idempotent = idempotent |
| 965 self.io_timeout = io_timeout | 943 self.io_timeout = io_timeout |
| 966 self.isolated_hash = isolated_hash | 944 self.isolated_hash = isolated_hash |
| 967 self.priority = priority | 945 self.priority = priority |
| 968 self.shards = shards | 946 self.shards = shards |
| 969 self.tags = set() | 947 self.tags = set() |
| 970 self.task_output_dir = task_output_dir | |
| 971 self.title = title | 948 self.title = title |
| 972 self.user = user | 949 self.user = user |
| 973 | 950 |
| 974 @property | 951 @property |
| 975 def task_name(self): | 952 def task_name(self): |
| 976 """Name of this task, derived from its other properties. | 953 """Name of this task, derived from its other properties. |
| 977 | 954 |
| 978 The task name is purely to make sense of the task and is not used in any | 955 The task name is purely to make sense of the task and is not used in any |
| 979 other way. | 956 other way. |
| 980 """ | 957 """ |
| (...skipping 10 matching lines...) Expand all Loading... |
| 991 | 968 |
| 992 def get_shard_view_url(self, index): | 969 def get_shard_view_url(self, index): |
| 993 """Returns URL of HTML page with shard details or None if not available. | 970 """Returns URL of HTML page with shard details or None if not available. |
| 994 | 971 |
| 995 Works only after the task has been successfully triggered. | 972 Works only after the task has been successfully triggered. |
| 996 """ | 973 """ |
| 997 if self._trigger_output and self._trigger_output.get('tasks'): | 974 if self._trigger_output and self._trigger_output.get('tasks'): |
| 998 for shard_dict in self._trigger_output['tasks'].itervalues(): | 975 for shard_dict in self._trigger_output['tasks'].itervalues(): |
| 999 if shard_dict['shard_index'] == index: | 976 if shard_dict['shard_index'] == index: |
| 1000 return shard_dict['view_url'] | 977 return shard_dict['view_url'] |
| OLD | NEW |