Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(279)

Side by Side Diff: client/swarming.py

Issue 1233303003: Migrate and merge logging_utils from swarming_bot into client/utils. (Closed) Base URL: git@github.com:luci/luci-py.git@master
Patch Set: Created 5 years, 5 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
1 #!/usr/bin/env python 1 #!/usr/bin/env python
2 # Copyright 2013 The Swarming Authors. All rights reserved. 2 # Copyright 2013 The Swarming Authors. All rights reserved.
3 # Use of this source code is governed under the Apache License, Version 2.0 that 3 # Use of this source code is governed under the Apache License, Version 2.0 that
4 # can be found in the LICENSE file. 4 # can be found in the LICENSE file.
5 5
6 """Client tool to trigger tasks or retrieve results from a Swarming server.""" 6 """Client tool to trigger tasks or retrieve results from a Swarming server."""
7 7
8 __version__ = '0.6.3' 8 __version__ = '0.6.3'
9 9
10 import collections 10 import collections
11 import datetime 11 import datetime
12 import json 12 import json
13 import logging 13 import logging
14 import optparse
14 import os 15 import os
15 import re 16 import re
16 import shutil 17 import shutil
17 import StringIO 18 import StringIO
18 import subprocess 19 import subprocess
19 import sys 20 import sys
20 import threading 21 import threading
21 import time 22 import time
22 import urllib 23 import urllib
23 import urlparse 24 import urlparse
24 import zipfile 25 import zipfile
25 26
26 from third_party import colorama 27 from third_party import colorama
27 from third_party.depot_tools import fix_encoding 28 from third_party.depot_tools import fix_encoding
28 from third_party.depot_tools import subcommand 29 from third_party.depot_tools import subcommand
29 30
30 from utils import file_path 31 from utils import file_path
32 from utils import logging_utils
31 from third_party.chromium import natsort 33 from third_party.chromium import natsort
32 from utils import net 34 from utils import net
33 from utils import on_error 35 from utils import on_error
34 from utils import threading_utils 36 from utils import threading_utils
35 from utils import tools 37 from utils import tools
36 from utils import zip_package 38 from utils import zip_package
37 39
38 import auth 40 import auth
39 import isolated_format 41 import isolated_format
40 import isolateserver 42 import isolateserver
(...skipping 772 matching lines...) Expand 10 before | Expand all | Expand 10 after
813 815
814 ### Commands. 816 ### Commands.
815 817
816 818
817 def abort_task(_swarming, _manifest): 819 def abort_task(_swarming, _manifest):
818 """Given a task manifest that was triggered, aborts its execution.""" 820 """Given a task manifest that was triggered, aborts its execution."""
819 # TODO(vadimsh): No supported by the server yet. 821 # TODO(vadimsh): No supported by the server yet.
820 822
821 823
822 def add_filter_options(parser): 824 def add_filter_options(parser):
823 parser.filter_group = tools.optparse.OptionGroup(parser, 'Filtering slaves') 825 parser.filter_group = optparse.OptionGroup(parser, 'Filtering slaves')
824 parser.filter_group.add_option( 826 parser.filter_group.add_option(
825 '-d', '--dimension', default=[], action='append', nargs=2, 827 '-d', '--dimension', default=[], action='append', nargs=2,
826 dest='dimensions', metavar='FOO bar', 828 dest='dimensions', metavar='FOO bar',
827 help='dimension to filter on') 829 help='dimension to filter on')
828 parser.add_option_group(parser.filter_group) 830 parser.add_option_group(parser.filter_group)
829 831
830 832
831 def add_sharding_options(parser): 833 def add_sharding_options(parser):
832 parser.sharding_group = tools.optparse.OptionGroup(parser, 'Sharding options') 834 parser.sharding_group = optparse.OptionGroup(parser, 'Sharding options')
833 parser.sharding_group.add_option( 835 parser.sharding_group.add_option(
834 '--shards', type='int', default=1, 836 '--shards', type='int', default=1,
835 help='Number of shards to trigger and collect.') 837 help='Number of shards to trigger and collect.')
836 parser.add_option_group(parser.sharding_group) 838 parser.add_option_group(parser.sharding_group)
837 839
838 840
839 def add_trigger_options(parser): 841 def add_trigger_options(parser):
840 """Adds all options to trigger a task on Swarming.""" 842 """Adds all options to trigger a task on Swarming."""
841 isolateserver.add_isolate_server_options(parser) 843 isolateserver.add_isolate_server_options(parser)
842 add_filter_options(parser) 844 add_filter_options(parser)
843 845
844 parser.task_group = tools.optparse.OptionGroup(parser, 'Task properties') 846 parser.task_group = optparse.OptionGroup(parser, 'Task properties')
845 parser.task_group.add_option( 847 parser.task_group.add_option(
846 '-s', '--isolated', 848 '-s', '--isolated',
847 help='Hash of the .isolated to grab from the isolate server') 849 help='Hash of the .isolated to grab from the isolate server')
848 parser.task_group.add_option( 850 parser.task_group.add_option(
849 '-e', '--env', default=[], action='append', nargs=2, metavar='FOO bar', 851 '-e', '--env', default=[], action='append', nargs=2, metavar='FOO bar',
850 help='Environment variables to set') 852 help='Environment variables to set')
851 parser.task_group.add_option( 853 parser.task_group.add_option(
852 '--priority', type='int', default=100, 854 '--priority', type='int', default=100,
853 help='The lower value, the more important the task is') 855 help='The lower value, the more important the task is')
854 parser.task_group.add_option( 856 parser.task_group.add_option(
(...skipping 12 matching lines...) Expand all
867 parser.task_group.add_option( 869 parser.task_group.add_option(
868 '--idempotent', action='store_true', default=False, 870 '--idempotent', action='store_true', default=False,
869 help='When set, the server will actively try to find a previous task ' 871 help='When set, the server will actively try to find a previous task '
870 'with the same parameter and return this result instead if possible') 872 'with the same parameter and return this result instead if possible')
871 parser.task_group.add_option( 873 parser.task_group.add_option(
872 '--expiration', type='int', default=6*60*60, 874 '--expiration', type='int', default=6*60*60,
873 help='Seconds to allow the task to be pending for a bot to run before ' 875 help='Seconds to allow the task to be pending for a bot to run before '
874 'this task request expires.') 876 'this task request expires.')
875 parser.task_group.add_option( 877 parser.task_group.add_option(
876 '--deadline', type='int', dest='expiration', 878 '--deadline', type='int', dest='expiration',
877 help=tools.optparse.SUPPRESS_HELP) 879 help=optparse.SUPPRESS_HELP)
878 parser.task_group.add_option( 880 parser.task_group.add_option(
879 '--hard-timeout', type='int', default=60*60, 881 '--hard-timeout', type='int', default=60*60,
880 help='Seconds to allow the task to complete.') 882 help='Seconds to allow the task to complete.')
881 parser.task_group.add_option( 883 parser.task_group.add_option(
882 '--io-timeout', type='int', default=20*60, 884 '--io-timeout', type='int', default=20*60,
883 help='Seconds to allow the task to be silent.') 885 help='Seconds to allow the task to be silent.')
884 parser.task_group.add_option( 886 parser.task_group.add_option(
885 '--raw-cmd', action='store_true', default=False, 887 '--raw-cmd', action='store_true', default=False,
886 help='When set, the command after -- is used as-is without run_isolated. ' 888 help='When set, the command after -- is used as-is without run_isolated. '
887 'In this case, no .isolated file is expected.') 889 'In this case, no .isolated file is expected.')
(...skipping 52 matching lines...) Expand 10 before | Expand all | Expand 10 after
940 '-t', '--timeout', 942 '-t', '--timeout',
941 type='float', 943 type='float',
942 default=80*60., 944 default=80*60.,
943 help='Timeout to wait for result, set to 0 for no timeout; default: ' 945 help='Timeout to wait for result, set to 0 for no timeout; default: '
944 '%default s') 946 '%default s')
945 parser.group_logging.add_option( 947 parser.group_logging.add_option(
946 '--decorate', action='store_true', help='Decorate output') 948 '--decorate', action='store_true', help='Decorate output')
947 parser.group_logging.add_option( 949 parser.group_logging.add_option(
948 '--print-status-updates', action='store_true', 950 '--print-status-updates', action='store_true',
949 help='Print periodic status updates') 951 help='Print periodic status updates')
950 parser.task_output_group = tools.optparse.OptionGroup(parser, 'Task output') 952 parser.task_output_group = optparse.OptionGroup(parser, 'Task output')
951 parser.task_output_group.add_option( 953 parser.task_output_group.add_option(
952 '--task-summary-json', 954 '--task-summary-json',
953 metavar='FILE', 955 metavar='FILE',
954 help='Dump a summary of task results to this file as json. It contains ' 956 help='Dump a summary of task results to this file as json. It contains '
955 'only shards statuses as know to server directly. Any output files ' 957 'only shards statuses as know to server directly. Any output files '
956 'emitted by the task can be collected by using --task-output-dir') 958 'emitted by the task can be collected by using --task-output-dir')
957 parser.task_output_group.add_option( 959 parser.task_output_group.add_option(
958 '--task-output-dir', 960 '--task-output-dir',
959 metavar='DIR', 961 metavar='DIR',
960 help='Directory to put task results into. When the task finishes, this ' 962 help='Directory to put task results into. When the task finishes, this '
(...skipping 349 matching lines...) Expand 10 before | Expand all | Expand 10 after
1310 (options.swarming, ' '.join(t['task_id'] for t in tasks_sorted))) 1312 (options.swarming, ' '.join(t['task_id'] for t in tasks_sorted)))
1311 print('Or visit:') 1313 print('Or visit:')
1312 for t in tasks_sorted: 1314 for t in tasks_sorted:
1313 print(' ' + t['view_url']) 1315 print(' ' + t['view_url'])
1314 return int(not tasks) 1316 return int(not tasks)
1315 except Failure: 1317 except Failure:
1316 on_error.report(None) 1318 on_error.report(None)
1317 return 1 1319 return 1
1318 1320
1319 1321
1320 class OptionParserSwarming(tools.OptionParserWithLogging): 1322 class OptionParserSwarming(logging_utils.OptionParserWithLogging):
1321 def __init__(self, **kwargs): 1323 def __init__(self, **kwargs):
1322 tools.OptionParserWithLogging.__init__( 1324 logging_utils.OptionParserWithLogging.__init__(
1323 self, prog='swarming.py', **kwargs) 1325 self, prog='swarming.py', **kwargs)
1324 self.server_group = tools.optparse.OptionGroup(self, 'Server') 1326 self.server_group = optparse.OptionGroup(self, 'Server')
1325 self.server_group.add_option( 1327 self.server_group.add_option(
1326 '-S', '--swarming', 1328 '-S', '--swarming',
1327 metavar='URL', default=os.environ.get('SWARMING_SERVER', ''), 1329 metavar='URL', default=os.environ.get('SWARMING_SERVER', ''),
1328 help='Swarming server to use') 1330 help='Swarming server to use')
1329 self.add_option_group(self.server_group) 1331 self.add_option_group(self.server_group)
1330 auth.add_auth_options(self) 1332 auth.add_auth_options(self)
1331 1333
1332 def parse_args(self, *args, **kwargs): 1334 def parse_args(self, *args, **kwargs):
1333 options, args = tools.OptionParserWithLogging.parse_args( 1335 options, args = logging_utils.OptionParserWithLogging.parse_args(
1334 self, *args, **kwargs) 1336 self, *args, **kwargs)
1335 auth.process_auth_options(self, options) 1337 auth.process_auth_options(self, options)
1336 user = self._process_swarming(options) 1338 user = self._process_swarming(options)
1337 if hasattr(options, 'user') and not options.user: 1339 if hasattr(options, 'user') and not options.user:
1338 options.user = user 1340 options.user = user
1339 return options, args 1341 return options, args
1340 1342
1341 def _process_swarming(self, options): 1343 def _process_swarming(self, options):
1342 """Processes the --swarming option and aborts if not specified. 1344 """Processes the --swarming option and aborts if not specified.
1343 1345
(...skipping 16 matching lines...) Expand all
1360 def main(args): 1362 def main(args):
1361 dispatcher = subcommand.CommandDispatcher(__name__) 1363 dispatcher = subcommand.CommandDispatcher(__name__)
1362 return dispatcher.execute(OptionParserSwarming(version=__version__), args) 1364 return dispatcher.execute(OptionParserSwarming(version=__version__), args)
1363 1365
1364 1366
1365 if __name__ == '__main__': 1367 if __name__ == '__main__':
1366 fix_encoding.fix_encoding() 1368 fix_encoding.fix_encoding()
1367 tools.disable_buffering() 1369 tools.disable_buffering()
1368 colorama.init() 1370 colorama.init()
1369 sys.exit(main(sys.argv[1:])) 1371 sys.exit(main(sys.argv[1:]))
OLDNEW

Powered by Google App Engine
This is Rietveld 408576698