OLD | NEW |
1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
2 # Copyright 2015 The Chromium Authors. All rights reserved. | 2 # Copyright 2015 The Chromium Authors. All rights reserved. |
3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
5 | 5 |
6 """Toolbox to manage all the json files in this directory. | 6 """Toolbox to manage all the json files in this directory. |
7 | 7 |
8 It can reformat them in their canonical format or ensures they are well | 8 It can reformat them in their canonical format or ensures they are well |
9 formatted. | 9 formatted. |
10 """ | 10 """ |
(...skipping 75 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
86 class Error(Exception): | 86 class Error(Exception): |
87 """Processing error.""" | 87 """Processing error.""" |
88 | 88 |
89 | 89 |
90 def get_isolates(): | 90 def get_isolates(): |
91 """Returns the list of all isolate files.""" | 91 """Returns the list of all isolate files.""" |
92 files = subprocess.check_output(['git', 'ls-files'], cwd=SRC_DIR).splitlines() | 92 files = subprocess.check_output(['git', 'ls-files'], cwd=SRC_DIR).splitlines() |
93 return [os.path.basename(f) for f in files if f.endswith('.isolate')] | 93 return [os.path.basename(f) for f in files if f.endswith('.isolate')] |
94 | 94 |
95 | 95 |
96 def process_builder_convert(data, filename, builder, test_name): | 96 def process_builder_convert(data, test_name): |
97 """Converts 'test_name' to run on Swarming in 'data'. | 97 """Converts 'test_name' to run on Swarming in 'data'. |
98 | 98 |
99 Returns True if 'test_name' was found. | 99 Returns True if 'test_name' was found. |
100 """ | 100 """ |
101 result = False | 101 result = False |
102 for test in data['gtest_tests']: | 102 for test in data['gtest_tests']: |
103 if test['test'] != test_name: | 103 if test['test'] != test_name: |
104 continue | 104 continue |
105 test.setdefault('swarming', {}) | 105 test.setdefault('swarming', {}) |
106 if not test['swarming'].get('can_use_on_swarming_builders'): | 106 if not test['swarming'].get('can_use_on_swarming_builders'): |
107 print('- %s: %s' % (filename, builder)) | |
108 test['swarming']['can_use_on_swarming_builders'] = True | 107 test['swarming']['can_use_on_swarming_builders'] = True |
109 result = True | 108 result = True |
110 return result | 109 return result |
111 | 110 |
112 | 111 |
113 def process_builder_remaining(data, filename, builder, tests_location): | 112 def process_builder_remaining(data, filename, builder, tests_location): |
114 """Calculates tests_location when mode is --remaining.""" | 113 """Calculates tests_location when mode is --remaining.""" |
115 for test in data['gtest_tests']: | 114 for test in data['gtest_tests']: |
116 name = test['test'] | 115 name = test['test'] |
117 if test.get('swarming', {}).get('can_use_on_swarming_builders'): | 116 if test.get('swarming', {}).get('can_use_on_swarming_builders'): |
118 tests_location[name]['count_run_on_swarming'] += 1 | 117 tests_location[name]['count_run_on_swarming'] += 1 |
119 else: | 118 else: |
120 tests_location[name]['count_run_local'] += 1 | 119 tests_location[name]['count_run_local'] += 1 |
121 tests_location[name]['local_configs'].setdefault( | 120 tests_location[name]['local_configs'].setdefault( |
122 filename, []).append(builder) | 121 filename, []).append(builder) |
123 | 122 |
124 | 123 |
125 def process_file(mode, test_name, tests_location, filepath, ninja_targets, | 124 def process_file(mode, test_name, tests_location, filepath, ninja_targets, |
126 ninja_targets_seen): | 125 ninja_targets_seen): |
127 """Processes a file. | 126 """Processes a json file describing what tests should be run for each recipe. |
128 | 127 |
129 The action depends on mode. Updates tests_location. | 128 The action depends on mode. Updates tests_location. |
130 | 129 |
131 Return False if the process exit code should be 1. | 130 Return False if the process exit code should be 1. |
132 """ | 131 """ |
133 filename = os.path.basename(filepath) | 132 filename = os.path.basename(filepath) |
134 with open(filepath) as f: | 133 with open(filepath) as f: |
135 content = f.read() | 134 content = f.read() |
136 try: | 135 try: |
137 config = json.loads(content) | 136 config = json.loads(content) |
(...skipping 18 matching lines...) Expand all Loading... |
156 for d in data['gtest_tests']: | 155 for d in data['gtest_tests']: |
157 if (d['test'] not in ninja_targets and | 156 if (d['test'] not in ninja_targets and |
158 d['test'] not in SKIP_NINJA_TO_GN_TARGETS): | 157 d['test'] not in SKIP_NINJA_TO_GN_TARGETS): |
159 raise Error('%s: %s / %s is not listed in ninja_to_gn.pyl.' % | 158 raise Error('%s: %s / %s is not listed in ninja_to_gn.pyl.' % |
160 (filename, builder, d['test'])) | 159 (filename, builder, d['test'])) |
161 elif d['test'] in ninja_targets: | 160 elif d['test'] in ninja_targets: |
162 ninja_targets_seen.add(d['test']) | 161 ninja_targets_seen.add(d['test']) |
163 | 162 |
164 config[builder]['gtest_tests'] = sorted( | 163 config[builder]['gtest_tests'] = sorted( |
165 data['gtest_tests'], key=lambda x: x['test']) | 164 data['gtest_tests'], key=lambda x: x['test']) |
166 if mode == 'remaining': | 165 |
| 166 # The trick here is that process_builder_remaining() is called before |
| 167 # process_builder_convert() so tests_location can be used to know how many |
| 168 # tests were converted. |
| 169 if mode in ('convert', 'remaining'): |
167 process_builder_remaining(data, filename, builder, tests_location) | 170 process_builder_remaining(data, filename, builder, tests_location) |
168 elif mode == 'convert': | 171 if mode == 'convert': |
169 process_builder_convert(data, filename, builder, test_name) | 172 process_builder_convert(data, test_name) |
170 | 173 |
171 expected = json.dumps( | 174 expected = json.dumps( |
172 config, sort_keys=True, indent=2, separators=(',', ': ')) + '\n' | 175 config, sort_keys=True, indent=2, separators=(',', ': ')) + '\n' |
173 if content != expected: | 176 if content != expected: |
174 if mode in ('convert', 'write'): | 177 if mode in ('convert', 'write'): |
175 with open(filepath, 'wb') as f: | 178 with open(filepath, 'wb') as f: |
176 f.write(expected) | 179 f.write(expected) |
177 if mode == 'write': | 180 if mode == 'write': |
178 print('Updated %s' % filename) | 181 print('Updated %s' % filename) |
179 else: | 182 else: |
180 print('%s is not in canonical format' % filename) | 183 print('%s is not in canonical format' % filename) |
181 print('run `testing/buildbot/manage.py -w` to fix') | 184 print('run `testing/buildbot/manage.py -w` to fix') |
182 return mode != 'check' | 185 return mode != 'check' |
183 return True | 186 return True |
184 | 187 |
185 | 188 |
186 def print_remaining(test_name,tests_location): | 189 def print_convert(test_name, tests_location): |
| 190 """Prints statistics for a test being converted for use in a CL description. |
| 191 """ |
| 192 data = tests_location[test_name] |
| 193 print('Convert %s to run exclusively on Swarming' % test_name) |
| 194 print('') |
| 195 print('%d configs already ran on Swarming' % data['count_run_on_swarming']) |
| 196 print('%d used to run locally and were converted:' % data['count_run_local']) |
| 197 for master, builders in sorted(data['local_configs'].iteritems()): |
| 198 for builder in builders: |
| 199 print('- %s: %s' % (master, builder)) |
| 200 print('') |
| 201 print('Ran:') |
| 202 print(' ./manage.py --convert %s' % test_name) |
| 203 print('') |
| 204 print('R=') |
| 205 print('BUG=98637') |
| 206 |
| 207 |
| 208 def print_remaining(test_name, tests_location): |
187 """Prints a visual summary of what tests are yet to be converted to run on | 209 """Prints a visual summary of what tests are yet to be converted to run on |
188 Swarming. | 210 Swarming. |
189 """ | 211 """ |
190 if test_name: | 212 if test_name: |
191 if test_name not in tests_location: | 213 if test_name not in tests_location: |
192 raise Error('Unknown test %s' % test_name) | 214 raise Error('Unknown test %s' % test_name) |
193 for config, builders in sorted( | 215 for config, builders in sorted( |
194 tests_location[test_name]['local_configs'].iteritems()): | 216 tests_location[test_name]['local_configs'].iteritems()): |
195 print('%s:' % config) | 217 print('%s:' % config) |
196 for builder in sorted(builders): | 218 for builder in sorted(builders): |
(...skipping 79 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
276 | 298 |
277 extra_targets = set(ninja_targets) - ninja_targets_seen | 299 extra_targets = set(ninja_targets) - ninja_targets_seen |
278 if extra_targets: | 300 if extra_targets: |
279 if len(extra_targets) > 1: | 301 if len(extra_targets) > 1: |
280 extra_targets_str = ', '.join(extra_targets) + ' are' | 302 extra_targets_str = ', '.join(extra_targets) + ' are' |
281 else: | 303 else: |
282 extra_targets_str = list(extra_targets)[0] + ' is' | 304 extra_targets_str = list(extra_targets)[0] + ' is' |
283 raise Error('%s listed in ninja_to_gn.pyl but not in any .json files' % | 305 raise Error('%s listed in ninja_to_gn.pyl but not in any .json files' % |
284 extra_targets_str) | 306 extra_targets_str) |
285 | 307 |
286 if args.mode == 'remaining': | 308 if args.mode == 'convert': |
| 309 print_convert(args.test_name, tests_location) |
| 310 elif args.mode == 'remaining': |
287 print_remaining(args.test_name, tests_location) | 311 print_remaining(args.test_name, tests_location) |
288 return result | 312 return result |
289 except Error as e: | 313 except Error as e: |
290 sys.stderr.write('%s\n' % e) | 314 sys.stderr.write('%s\n' % e) |
291 return 1 | 315 return 1 |
292 | 316 |
293 | 317 |
294 if __name__ == "__main__": | 318 if __name__ == "__main__": |
295 sys.exit(main()) | 319 sys.exit(main()) |
OLD | NEW |