OLD | NEW |
1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
2 # Copyright 2013 The Swarming Authors. All rights reserved. | 2 # Copyright 2013 The Swarming Authors. All rights reserved. |
3 # Use of this source code is governed under the Apache License, Version 2.0 that | 3 # Use of this source code is governed under the Apache License, Version 2.0 that |
4 # can be found in the LICENSE file. | 4 # can be found in the LICENSE file. |
5 | 5 |
6 """Client tool to trigger tasks or retrieve results from a Swarming server.""" | 6 """Client tool to trigger tasks or retrieve results from a Swarming server.""" |
7 | 7 |
8 __version__ = '0.6.3' | 8 __version__ = '0.6.3' |
9 | 9 |
10 import collections | 10 import collections |
11 import datetime | 11 import datetime |
12 import json | 12 import json |
13 import logging | 13 import logging |
| 14 import optparse |
14 import os | 15 import os |
15 import re | 16 import re |
16 import shutil | 17 import shutil |
17 import StringIO | 18 import StringIO |
18 import subprocess | 19 import subprocess |
19 import sys | 20 import sys |
20 import threading | 21 import threading |
21 import time | 22 import time |
22 import urllib | 23 import urllib |
23 import urlparse | 24 import urlparse |
24 import zipfile | 25 import zipfile |
25 | 26 |
26 from third_party import colorama | 27 from third_party import colorama |
27 from third_party.depot_tools import fix_encoding | 28 from third_party.depot_tools import fix_encoding |
28 from third_party.depot_tools import subcommand | 29 from third_party.depot_tools import subcommand |
29 | 30 |
30 from utils import file_path | 31 from utils import file_path |
| 32 from utils import logging_utils |
31 from third_party.chromium import natsort | 33 from third_party.chromium import natsort |
32 from utils import net | 34 from utils import net |
33 from utils import on_error | 35 from utils import on_error |
34 from utils import threading_utils | 36 from utils import threading_utils |
35 from utils import tools | 37 from utils import tools |
36 from utils import zip_package | 38 from utils import zip_package |
37 | 39 |
38 import auth | 40 import auth |
39 import isolated_format | 41 import isolated_format |
40 import isolateserver | 42 import isolateserver |
(...skipping 772 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
813 | 815 |
814 ### Commands. | 816 ### Commands. |
815 | 817 |
816 | 818 |
817 def abort_task(_swarming, _manifest): | 819 def abort_task(_swarming, _manifest): |
818 """Given a task manifest that was triggered, aborts its execution.""" | 820 """Given a task manifest that was triggered, aborts its execution.""" |
819 # TODO(vadimsh): No supported by the server yet. | 821 # TODO(vadimsh): No supported by the server yet. |
820 | 822 |
821 | 823 |
822 def add_filter_options(parser): | 824 def add_filter_options(parser): |
823 parser.filter_group = tools.optparse.OptionGroup(parser, 'Filtering slaves') | 825 parser.filter_group = optparse.OptionGroup(parser, 'Filtering slaves') |
824 parser.filter_group.add_option( | 826 parser.filter_group.add_option( |
825 '-d', '--dimension', default=[], action='append', nargs=2, | 827 '-d', '--dimension', default=[], action='append', nargs=2, |
826 dest='dimensions', metavar='FOO bar', | 828 dest='dimensions', metavar='FOO bar', |
827 help='dimension to filter on') | 829 help='dimension to filter on') |
828 parser.add_option_group(parser.filter_group) | 830 parser.add_option_group(parser.filter_group) |
829 | 831 |
830 | 832 |
831 def add_sharding_options(parser): | 833 def add_sharding_options(parser): |
832 parser.sharding_group = tools.optparse.OptionGroup(parser, 'Sharding options') | 834 parser.sharding_group = optparse.OptionGroup(parser, 'Sharding options') |
833 parser.sharding_group.add_option( | 835 parser.sharding_group.add_option( |
834 '--shards', type='int', default=1, | 836 '--shards', type='int', default=1, |
835 help='Number of shards to trigger and collect.') | 837 help='Number of shards to trigger and collect.') |
836 parser.add_option_group(parser.sharding_group) | 838 parser.add_option_group(parser.sharding_group) |
837 | 839 |
838 | 840 |
839 def add_trigger_options(parser): | 841 def add_trigger_options(parser): |
840 """Adds all options to trigger a task on Swarming.""" | 842 """Adds all options to trigger a task on Swarming.""" |
841 isolateserver.add_isolate_server_options(parser) | 843 isolateserver.add_isolate_server_options(parser) |
842 add_filter_options(parser) | 844 add_filter_options(parser) |
843 | 845 |
844 parser.task_group = tools.optparse.OptionGroup(parser, 'Task properties') | 846 parser.task_group = optparse.OptionGroup(parser, 'Task properties') |
845 parser.task_group.add_option( | 847 parser.task_group.add_option( |
846 '-s', '--isolated', | 848 '-s', '--isolated', |
847 help='Hash of the .isolated to grab from the isolate server') | 849 help='Hash of the .isolated to grab from the isolate server') |
848 parser.task_group.add_option( | 850 parser.task_group.add_option( |
849 '-e', '--env', default=[], action='append', nargs=2, metavar='FOO bar', | 851 '-e', '--env', default=[], action='append', nargs=2, metavar='FOO bar', |
850 help='Environment variables to set') | 852 help='Environment variables to set') |
851 parser.task_group.add_option( | 853 parser.task_group.add_option( |
852 '--priority', type='int', default=100, | 854 '--priority', type='int', default=100, |
853 help='The lower value, the more important the task is') | 855 help='The lower value, the more important the task is') |
854 parser.task_group.add_option( | 856 parser.task_group.add_option( |
(...skipping 12 matching lines...) Expand all Loading... |
867 parser.task_group.add_option( | 869 parser.task_group.add_option( |
868 '--idempotent', action='store_true', default=False, | 870 '--idempotent', action='store_true', default=False, |
869 help='When set, the server will actively try to find a previous task ' | 871 help='When set, the server will actively try to find a previous task ' |
870 'with the same parameter and return this result instead if possible') | 872 'with the same parameter and return this result instead if possible') |
871 parser.task_group.add_option( | 873 parser.task_group.add_option( |
872 '--expiration', type='int', default=6*60*60, | 874 '--expiration', type='int', default=6*60*60, |
873 help='Seconds to allow the task to be pending for a bot to run before ' | 875 help='Seconds to allow the task to be pending for a bot to run before ' |
874 'this task request expires.') | 876 'this task request expires.') |
875 parser.task_group.add_option( | 877 parser.task_group.add_option( |
876 '--deadline', type='int', dest='expiration', | 878 '--deadline', type='int', dest='expiration', |
877 help=tools.optparse.SUPPRESS_HELP) | 879 help=optparse.SUPPRESS_HELP) |
878 parser.task_group.add_option( | 880 parser.task_group.add_option( |
879 '--hard-timeout', type='int', default=60*60, | 881 '--hard-timeout', type='int', default=60*60, |
880 help='Seconds to allow the task to complete.') | 882 help='Seconds to allow the task to complete.') |
881 parser.task_group.add_option( | 883 parser.task_group.add_option( |
882 '--io-timeout', type='int', default=20*60, | 884 '--io-timeout', type='int', default=20*60, |
883 help='Seconds to allow the task to be silent.') | 885 help='Seconds to allow the task to be silent.') |
884 parser.task_group.add_option( | 886 parser.task_group.add_option( |
885 '--raw-cmd', action='store_true', default=False, | 887 '--raw-cmd', action='store_true', default=False, |
886 help='When set, the command after -- is used as-is without run_isolated. ' | 888 help='When set, the command after -- is used as-is without run_isolated. ' |
887 'In this case, no .isolated file is expected.') | 889 'In this case, no .isolated file is expected.') |
(...skipping 52 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
940 '-t', '--timeout', | 942 '-t', '--timeout', |
941 type='float', | 943 type='float', |
942 default=80*60., | 944 default=80*60., |
943 help='Timeout to wait for result, set to 0 for no timeout; default: ' | 945 help='Timeout to wait for result, set to 0 for no timeout; default: ' |
944 '%default s') | 946 '%default s') |
945 parser.group_logging.add_option( | 947 parser.group_logging.add_option( |
946 '--decorate', action='store_true', help='Decorate output') | 948 '--decorate', action='store_true', help='Decorate output') |
947 parser.group_logging.add_option( | 949 parser.group_logging.add_option( |
948 '--print-status-updates', action='store_true', | 950 '--print-status-updates', action='store_true', |
949 help='Print periodic status updates') | 951 help='Print periodic status updates') |
950 parser.task_output_group = tools.optparse.OptionGroup(parser, 'Task output') | 952 parser.task_output_group = optparse.OptionGroup(parser, 'Task output') |
951 parser.task_output_group.add_option( | 953 parser.task_output_group.add_option( |
952 '--task-summary-json', | 954 '--task-summary-json', |
953 metavar='FILE', | 955 metavar='FILE', |
954 help='Dump a summary of task results to this file as json. It contains ' | 956 help='Dump a summary of task results to this file as json. It contains ' |
955 'only shards statuses as know to server directly. Any output files ' | 957 'only shards statuses as know to server directly. Any output files ' |
956 'emitted by the task can be collected by using --task-output-dir') | 958 'emitted by the task can be collected by using --task-output-dir') |
957 parser.task_output_group.add_option( | 959 parser.task_output_group.add_option( |
958 '--task-output-dir', | 960 '--task-output-dir', |
959 metavar='DIR', | 961 metavar='DIR', |
960 help='Directory to put task results into. When the task finishes, this ' | 962 help='Directory to put task results into. When the task finishes, this ' |
(...skipping 349 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1310 (options.swarming, ' '.join(t['task_id'] for t in tasks_sorted))) | 1312 (options.swarming, ' '.join(t['task_id'] for t in tasks_sorted))) |
1311 print('Or visit:') | 1313 print('Or visit:') |
1312 for t in tasks_sorted: | 1314 for t in tasks_sorted: |
1313 print(' ' + t['view_url']) | 1315 print(' ' + t['view_url']) |
1314 return int(not tasks) | 1316 return int(not tasks) |
1315 except Failure: | 1317 except Failure: |
1316 on_error.report(None) | 1318 on_error.report(None) |
1317 return 1 | 1319 return 1 |
1318 | 1320 |
1319 | 1321 |
1320 class OptionParserSwarming(tools.OptionParserWithLogging): | 1322 class OptionParserSwarming(logging_utils.OptionParserWithLogging): |
1321 def __init__(self, **kwargs): | 1323 def __init__(self, **kwargs): |
1322 tools.OptionParserWithLogging.__init__( | 1324 logging_utils.OptionParserWithLogging.__init__( |
1323 self, prog='swarming.py', **kwargs) | 1325 self, prog='swarming.py', **kwargs) |
1324 self.server_group = tools.optparse.OptionGroup(self, 'Server') | 1326 self.server_group = optparse.OptionGroup(self, 'Server') |
1325 self.server_group.add_option( | 1327 self.server_group.add_option( |
1326 '-S', '--swarming', | 1328 '-S', '--swarming', |
1327 metavar='URL', default=os.environ.get('SWARMING_SERVER', ''), | 1329 metavar='URL', default=os.environ.get('SWARMING_SERVER', ''), |
1328 help='Swarming server to use') | 1330 help='Swarming server to use') |
1329 self.add_option_group(self.server_group) | 1331 self.add_option_group(self.server_group) |
1330 auth.add_auth_options(self) | 1332 auth.add_auth_options(self) |
1331 | 1333 |
1332 def parse_args(self, *args, **kwargs): | 1334 def parse_args(self, *args, **kwargs): |
1333 options, args = tools.OptionParserWithLogging.parse_args( | 1335 options, args = logging_utils.OptionParserWithLogging.parse_args( |
1334 self, *args, **kwargs) | 1336 self, *args, **kwargs) |
1335 auth.process_auth_options(self, options) | 1337 auth.process_auth_options(self, options) |
1336 user = self._process_swarming(options) | 1338 user = self._process_swarming(options) |
1337 if hasattr(options, 'user') and not options.user: | 1339 if hasattr(options, 'user') and not options.user: |
1338 options.user = user | 1340 options.user = user |
1339 return options, args | 1341 return options, args |
1340 | 1342 |
1341 def _process_swarming(self, options): | 1343 def _process_swarming(self, options): |
1342 """Processes the --swarming option and aborts if not specified. | 1344 """Processes the --swarming option and aborts if not specified. |
1343 | 1345 |
(...skipping 16 matching lines...) Expand all Loading... |
1360 def main(args): | 1362 def main(args): |
1361 dispatcher = subcommand.CommandDispatcher(__name__) | 1363 dispatcher = subcommand.CommandDispatcher(__name__) |
1362 return dispatcher.execute(OptionParserSwarming(version=__version__), args) | 1364 return dispatcher.execute(OptionParserSwarming(version=__version__), args) |
1363 | 1365 |
1364 | 1366 |
1365 if __name__ == '__main__': | 1367 if __name__ == '__main__': |
1366 fix_encoding.fix_encoding() | 1368 fix_encoding.fix_encoding() |
1367 tools.disable_buffering() | 1369 tools.disable_buffering() |
1368 colorama.init() | 1370 colorama.init() |
1369 sys.exit(main(sys.argv[1:])) | 1371 sys.exit(main(sys.argv[1:])) |
OLD | NEW |