OLD | NEW |
---|---|
1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
2 # Copyright 2015 The Chromium Authors. All rights reserved. | 2 # Copyright 2015 The Chromium Authors. All rights reserved. |
3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
5 | 5 |
6 """Toolbox to manage all the json files in this directory. | 6 """Toolbox to manage all the json files in this directory. |
7 | 7 |
8 It can reformat them in their canonical format or ensures they are well | 8 It can reformat them in their canonical format or ensures they are well |
9 formatted. | 9 formatted. |
10 """ | 10 """ |
11 | 11 |
12 import argparse | 12 import argparse |
13 import collections | |
13 import glob | 14 import glob |
14 import json | 15 import json |
15 import os | 16 import os |
16 import sys | 17 import sys |
17 | 18 |
18 | 19 |
19 THIS_DIR = os.path.dirname(os.path.abspath(__file__)) | 20 THIS_DIR = os.path.dirname(os.path.abspath(__file__)) |
21 sys.path.insert(0, os.path.join(THIS_DIR, '..', '..', 'third_party')) | |
20 | 22 |
23 import colorama | |
21 | 24 |
22 # These are not 'builders'. | 25 # These are not 'builders'. |
23 SKIP = { | 26 SKIP = { |
24 'compile_targets', 'gtest_tests', 'filter_compile_builders', | 27 'compile_targets', 'gtest_tests', 'filter_compile_builders', |
25 'non_filter_builders', 'non_filter_tests_builders', | 28 'non_filter_builders', 'non_filter_tests_builders', |
26 } | 29 } |
27 | 30 |
28 | 31 |
29 def upgrade_test(test): | 32 def upgrade_test(test): |
30 """Converts from old style string to new style dict.""" | 33 """Converts from old style string to new style dict.""" |
31 if isinstance(test, basestring): | 34 if isinstance(test, basestring): |
32 return {'test': test} | 35 return {'test': test} |
33 assert isinstance(test, dict) | 36 assert isinstance(test, dict) |
34 return test | 37 return test |
35 | 38 |
36 | 39 |
37 def main(): | 40 def main(): |
41 colorama.init() | |
38 parser = argparse.ArgumentParser(description=sys.modules[__name__].__doc__) | 42 parser = argparse.ArgumentParser(description=sys.modules[__name__].__doc__) |
39 group = parser.add_mutually_exclusive_group(required=True) | 43 group = parser.add_mutually_exclusive_group(required=True) |
40 group.add_argument( | 44 group.add_argument( |
41 '-c', '--check', action='store_true', help='Only check the files') | 45 '-c', '--check', action='store_true', help='Only check the files') |
42 group.add_argument( | 46 group.add_argument( |
47 '--remaining', action='store_true', | |
48 help='Count the number of tests not yet running on Swarming') | |
49 group.add_argument( | |
43 '-w', '--write', action='store_true', help='Rewrite the files') | 50 '-w', '--write', action='store_true', help='Rewrite the files') |
51 parser.add_argument( | |
52 'test_name', nargs='?', | |
53 help='The test name to print which configs to update; only to be used ' | |
54 'with --remaining') | |
44 args = parser.parse_args() | 55 args = parser.parse_args() |
45 | 56 |
57 # Stats when running in --remaining mode. | |
58 # (local, swarming, local_configs). | |
59 tests_location = collections.defaultdict(lambda: [0, 0, {}]) | |
Paweł Hajdan Jr.
2015/04/01 10:08:28
This tuple makes code readability confusing. Could
M-A Ruel
2015/04/01 12:34:03
Agreed, switched to a dict.
| |
60 | |
46 result = 0 | 61 result = 0 |
47 for filepath in glob.glob(os.path.join(THIS_DIR, '*.json')): | 62 for filepath in glob.glob(os.path.join(THIS_DIR, '*.json')): |
48 filename = os.path.basename(filepath) | 63 filename = os.path.basename(filepath) |
49 with open(filepath) as f: | 64 with open(filepath) as f: |
50 content = f.read() | 65 content = f.read() |
51 config = json.loads(content) | 66 config = json.loads(content) |
52 for builder, data in sorted(config.iteritems()): | 67 for builder, data in sorted(config.iteritems()): |
53 if builder in SKIP: | 68 if builder in SKIP: |
54 # Oddities. | 69 # Oddities. |
55 continue | 70 continue |
56 | 71 |
57 if not isinstance(data, dict): | 72 if not isinstance(data, dict): |
58 print('%s: %s is broken: %s' % (filename, builder, data)) | 73 print('%s: %s is broken: %s' % (filename, builder, data)) |
59 continue | 74 continue |
60 | 75 |
61 if 'gtest_tests' in data: | 76 if 'gtest_tests' in data: |
62 config[builder]['gtest_tests'] = sorted( | 77 config[builder]['gtest_tests'] = sorted( |
63 (upgrade_test(l) for l in data['gtest_tests']), | 78 (upgrade_test(l) for l in data['gtest_tests']), |
64 key=lambda x: x['test']) | 79 key=lambda x: x['test']) |
65 | 80 |
81 if args.remaining: | |
82 for test in data['gtest_tests']: | |
83 name = test['test'] | |
84 if test.get('swarming', {}).get('can_use_on_swarming_builders'): | |
85 tests_location[name][1] += 1 | |
86 else: | |
87 tests_location[name][0] += 1 | |
88 tests_location[name][2].setdefault(filename, []).append(builder) | |
89 | |
66 expected = json.dumps( | 90 expected = json.dumps( |
67 config, sort_keys=True, indent=2, separators=(',', ': ')) + '\n' | 91 config, sort_keys=True, indent=2, separators=(',', ': ')) + '\n' |
68 if content != expected: | 92 if content != expected: |
69 result = 1 | 93 result = 1 |
70 if args.write: | 94 if args.write: |
71 with open(filepath, 'wb') as f: | 95 with open(filepath, 'wb') as f: |
72 f.write(expected) | 96 f.write(expected) |
73 print('Updated %s' % filename) | 97 print('Updated %s' % filename) |
74 else: | 98 else: |
75 print('%s is not in canonical format' % filename) | 99 print('%s is not in canonical format' % filename) |
100 | |
101 if args.remaining: | |
102 if args.test_name: | |
103 if args.test_name not in tests_location: | |
104 print('Unknown test %s' % args.test_name) | |
105 return 1 | |
106 for config, builders in sorted( | |
107 tests_location[args.test_name][2].iteritems()): | |
108 print('%s:' % config) | |
109 for builder in sorted(builders): | |
110 print(' %s' % builder) | |
111 else: | |
112 l = max(map(len, tests_location)) | |
113 print('%-*s%sLocal %sSwarming' % | |
114 (l, 'Test', colorama.Fore.RED, colorama.Fore.GREEN)) | |
115 total_local = 0 | |
116 total_swarming = 0 | |
117 for name, location in sorted(tests_location.iteritems()): | |
118 if not location[1]: | |
119 c = colorama.Fore.RED | |
120 elif location[0]: | |
121 c = colorama.Fore.YELLOW | |
122 else: | |
123 c = colorama.Fore.GREEN | |
124 total_local += location[0] | |
125 total_swarming += location[1] | |
126 print('%s%-*s %4d %4d' % | |
127 (c, l, name, location[0], location[1])) | |
128 total = total_local + total_swarming | |
129 p_local = 100. * total_local / total | |
130 p_swarming = 100. * total_swarming / total | |
131 print('%s%-*s %4d (%4.1f%%) %4d (%4.1f%%)' % | |
132 (colorama.Fore.WHITE, l, 'Total:', total_local, p_local, | |
133 total_swarming, p_swarming)) | |
134 print('%-*s %4d' % (l, 'Total executions:', total)) | |
76 return result | 135 return result |
77 | 136 |
78 | 137 |
79 if __name__ == "__main__": | 138 if __name__ == "__main__": |
80 sys.exit(main()) | 139 sys.exit(main()) |
OLD | NEW |