OLD | NEW |
1 #!/usr/bin/python | 1 #!/usr/bin/python |
2 | 2 |
3 ''' | 3 ''' |
4 Copyright 2012 Google Inc. | 4 Copyright 2012 Google Inc. |
5 | 5 |
6 Use of this source code is governed by a BSD-style license that can be | 6 Use of this source code is governed by a BSD-style license that can be |
7 found in the LICENSE file. | 7 found in the LICENSE file. |
8 ''' | 8 ''' |
9 | 9 |
10 ''' | 10 ''' |
11 Rebaselines the given GM tests, on all bots and all configurations. | 11 Rebaselines the given GM tests, on all bots and all configurations. |
12 ''' | 12 ''' |
13 | 13 |
14 # System-level imports | 14 # System-level imports |
15 import argparse | 15 import argparse |
| 16 import json |
16 import os | 17 import os |
17 import re | 18 import re |
18 import subprocess | 19 import subprocess |
19 import sys | 20 import sys |
20 import urllib2 | 21 import urllib2 |
21 | 22 |
22 # Imports from within Skia | 23 # Imports from within Skia |
23 # | 24 # |
24 # We need to add the 'gm' directory, so that we can import gm_json.py within | 25 # We need to add the 'gm' directory, so that we can import gm_json.py within |
25 # that directory. That script allows us to parse the actual-results.json file | 26 # that directory. That script allows us to parse the actual-results.json file |
26 # written out by the GM tool. | 27 # written out by the GM tool. |
27 # Make sure that the 'gm' dir is in the PYTHONPATH, but add it at the *end* | 28 # Make sure that the 'gm' dir is in the PYTHONPATH, but add it at the *end* |
28 # so any dirs that are already in the PYTHONPATH will be preferred. | 29 # so any dirs that are already in the PYTHONPATH will be preferred. |
29 # | 30 # |
30 # This assumes that the 'gm' directory has been checked out as a sibling of | 31 # This assumes that the 'gm' directory has been checked out as a sibling of |
31 # the 'tools' directory containing this script, which will be the case if | 32 # the 'tools' directory containing this script, which will be the case if |
32 # 'trunk' was checked out as a single unit. | 33 # 'trunk' was checked out as a single unit. |
33 GM_DIRECTORY = os.path.realpath( | 34 GM_DIRECTORY = os.path.realpath( |
34 os.path.join(os.path.dirname(os.path.dirname(__file__)), 'gm')) | 35 os.path.join(os.path.dirname(os.path.dirname(__file__)), 'gm')) |
35 if GM_DIRECTORY not in sys.path: | 36 if GM_DIRECTORY not in sys.path: |
36 sys.path.append(GM_DIRECTORY) | 37 sys.path.append(GM_DIRECTORY) |
| 38 import buildbot_globals |
37 import gm_json | 39 import gm_json |
38 | 40 |
39 # Mapping of expectations/gm subdir (under | 41 MASTER_HOST_URL = 'http://%s:%s' % (buildbot_globals.Get('master_host'), |
40 # https://skia.googlecode.com/svn/trunk/expectations/gm/ ) | 42 buildbot_globals.Get('external_port')) |
41 # to builder name (see list at http://108.170.217.252:10117/builders ) | 43 ALL_BUILDERS = list(json.load(urllib2.urlopen( |
42 SUBDIR_MAPPING = { | 44 MASTER_HOST_URL + '/json/builders'))) |
43 'base-shuttle-win7-intel-float': | 45 TEST_BUILDERS = filter(lambda x: 'Trybot' not in x and 'Test' in x, |
44 'Test-Win7-ShuttleA-HD2000-x86-Release', | 46 ALL_BUILDERS) |
45 'base-shuttle-win7-intel-angle': | |
46 'Test-Win7-ShuttleA-HD2000-x86-Release-ANGLE', | |
47 'base-shuttle-win7-intel-directwrite': | |
48 'Test-Win7-ShuttleA-HD2000-x86-Release-DirectWrite', | |
49 'base-shuttle_ubuntu12_ati5770': | |
50 'Test-Ubuntu12-ShuttleA-ATI5770-x86_64-Release', | |
51 'base-macmini': | |
52 'Test-Mac10.6-MacMini4.1-GeForce320M-x86-Release', | |
53 'base-macmini-lion-float': | |
54 'Test-Mac10.7-MacMini4.1-GeForce320M-x86-Release', | |
55 'base-android-galaxy-nexus': | |
56 'Test-Android-GalaxyNexus-SGX540-Arm7-Debug', | |
57 'base-android-nexus-7': | |
58 'Test-Android-Nexus7-Tegra3-Arm7-Release', | |
59 'base-android-nexus-s': | |
60 'Test-Android-NexusS-SGX540-Arm7-Release', | |
61 'base-android-xoom': | |
62 'Test-Android-Xoom-Tegra2-Arm7-Release', | |
63 'base-android-nexus-10': | |
64 'Test-Android-Nexus10-MaliT604-Arm7-Release', | |
65 'base-android-nexus-4': | |
66 'Test-Android-Nexus4-Adreno320-Arm7-Release', | |
67 } | |
68 | |
69 | 47 |
70 class _InternalException(Exception): | 48 class _InternalException(Exception): |
71 pass | 49 pass |
72 | 50 |
73 # Object that handles exceptions, either raising them immediately or collecting | 51 # Object that handles exceptions, either raising them immediately or collecting |
74 # them to display later on. | 52 # them to display later on. |
75 class ExceptionHandler(object): | 53 class ExceptionHandler(object): |
76 | 54 |
77 # params: | 55 # params: |
78 # keep_going_on_failure: if False, report failures and quit right away; | 56 # keep_going_on_failure: if False, report failures and quit right away; |
(...skipping 127 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
206 actual_results = json_dict[gm_json.JSONKEY_ACTUALRESULTS] | 184 actual_results = json_dict[gm_json.JSONKEY_ACTUALRESULTS] |
207 if not sections: | 185 if not sections: |
208 sections = actual_results.keys() | 186 sections = actual_results.keys() |
209 for section in sections: | 187 for section in sections: |
210 section_results = actual_results[section] | 188 section_results = actual_results[section] |
211 if section_results: | 189 if section_results: |
212 results_to_return.update(section_results) | 190 results_to_return.update(section_results) |
213 return results_to_return | 191 return results_to_return |
214 | 192 |
215 # Rebaseline all tests/types we specified in the constructor, | 193 # Rebaseline all tests/types we specified in the constructor, |
216 # within this expectations/gm subdir. | 194 # within this builder's subdirectory in expectations/gm . |
217 # | 195 # |
218 # params: | 196 # params: |
219 # subdir : e.g. 'base-shuttle-win7-intel-float' | |
220 # builder : e.g. 'Test-Win7-ShuttleA-HD2000-x86-Release' | 197 # builder : e.g. 'Test-Win7-ShuttleA-HD2000-x86-Release' |
221 def RebaselineSubdir(self, subdir, builder): | 198 def RebaselineSubdir(self, builder): |
222 # Read in the actual result summary, and extract all the tests whose | 199 # Read in the actual result summary, and extract all the tests whose |
223 # results we need to update. | 200 # results we need to update. |
224 actuals_url = '/'.join([self._actuals_base_url, | 201 actuals_url = '/'.join([self._actuals_base_url, |
225 subdir, builder, subdir, | 202 builder, self._actuals_filename]) |
226 self._actuals_filename]) | |
227 # In most cases, we won't need to re-record results that are already | 203 # In most cases, we won't need to re-record results that are already |
228 # succeeding, but including the SUCCEEDED results will allow us to | 204 # succeeding, but including the SUCCEEDED results will allow us to |
229 # re-record expectations if they somehow get out of sync. | 205 # re-record expectations if they somehow get out of sync. |
230 sections = [gm_json.JSONKEY_ACTUALRESULTS_FAILED, | 206 sections = [gm_json.JSONKEY_ACTUALRESULTS_FAILED, |
231 gm_json.JSONKEY_ACTUALRESULTS_SUCCEEDED] | 207 gm_json.JSONKEY_ACTUALRESULTS_SUCCEEDED] |
232 if self._add_new: | 208 if self._add_new: |
233 sections.append(gm_json.JSONKEY_ACTUALRESULTS_NOCOMPARISON) | 209 sections.append(gm_json.JSONKEY_ACTUALRESULTS_NOCOMPARISON) |
234 results_to_update = self._GetActualResults(json_url=actuals_url, | 210 results_to_update = self._GetActualResults(json_url=actuals_url, |
235 sections=sections) | 211 sections=sections) |
236 | 212 |
237 # Read in current expectations. | 213 # Read in current expectations. |
238 expectations_input_filepath = os.path.join( | 214 expectations_input_filepath = os.path.join( |
239 self._expectations_root, subdir, self._expectations_input_filename) | 215 self._expectations_root, builder, self._expectations_input_filename) |
240 expectations_dict = gm_json.LoadFromFile(expectations_input_filepath) | 216 expectations_dict = gm_json.LoadFromFile(expectations_input_filepath) |
241 expected_results = expectations_dict[gm_json.JSONKEY_EXPECTEDRESULTS] | 217 expected_results = expectations_dict[gm_json.JSONKEY_EXPECTEDRESULTS] |
242 | 218 |
243 # Update the expectations in memory, skipping any tests/configs that | 219 # Update the expectations in memory, skipping any tests/configs that |
244 # the caller asked to exclude. | 220 # the caller asked to exclude. |
245 skipped_images = [] | 221 skipped_images = [] |
246 if results_to_update: | 222 if results_to_update: |
247 for (image_name, image_results) in results_to_update.iteritems(): | 223 for (image_name, image_results) in results_to_update.iteritems(): |
248 (test, config) = self._image_filename_re.match(image_name).groups() | 224 (test, config) = self._image_filename_re.match(image_name).groups() |
249 if self._tests: | 225 if self._tests: |
250 if test not in self._tests: | 226 if test not in self._tests: |
251 skipped_images.append(image_name) | 227 skipped_images.append(image_name) |
252 continue | 228 continue |
253 if self._configs: | 229 if self._configs: |
254 if config not in self._configs: | 230 if config not in self._configs: |
255 skipped_images.append(image_name) | 231 skipped_images.append(image_name) |
256 continue | 232 continue |
257 if not expected_results.get(image_name): | 233 if not expected_results.get(image_name): |
258 expected_results[image_name] = {} | 234 expected_results[image_name] = {} |
259 expected_results[image_name][gm_json.JSONKEY_EXPECTEDRESULTS_ALLOWEDDIGE
STS] = \ | 235 expected_results[image_name][gm_json.JSONKEY_EXPECTEDRESULTS_ALLOWEDDIGE
STS] = \ |
260 [image_results] | 236 [image_results] |
261 | 237 |
262 # Write out updated expectations. | 238 # Write out updated expectations. |
263 expectations_output_filepath = os.path.join( | 239 expectations_output_filepath = os.path.join( |
264 self._expectations_root, subdir, self._expectations_output_filename) | 240 self._expectations_root, builder, self._expectations_output_filename) |
265 gm_json.WriteToFile(expectations_dict, expectations_output_filepath) | 241 gm_json.WriteToFile(expectations_dict, expectations_output_filepath) |
266 | 242 |
267 # Mark the JSON file as plaintext, so text-style diffs can be applied. | 243 # Mark the JSON file as plaintext, so text-style diffs can be applied. |
268 # Fixes https://code.google.com/p/skia/issues/detail?id=1442 | 244 # Fixes https://code.google.com/p/skia/issues/detail?id=1442 |
269 if self._using_svn: | 245 if self._using_svn: |
270 self._Call(['svn', 'propset', '--quiet', 'svn:mime-type', | 246 self._Call(['svn', 'propset', '--quiet', 'svn:mime-type', |
271 'text/x-json', expectations_output_filepath]) | 247 'text/x-json', expectations_output_filepath]) |
272 | 248 |
273 # main... | 249 # main... |
274 | 250 |
275 parser = argparse.ArgumentParser() | 251 parser = argparse.ArgumentParser() |
276 parser.add_argument('--actuals-base-url', | 252 parser.add_argument('--actuals-base-url', |
277 help='base URL from which to read files containing JSON ' + | 253 help='base URL from which to read files containing JSON ' + |
278 'summaries of actual GM results; defaults to %(default)s', | 254 'summaries of actual GM results; defaults to %(default)s', |
279 default='http://skia-autogen.googlecode.com/svn/gm-actual') | 255 default='http://skia-autogen.googlecode.com/svn/gm-actual') |
280 parser.add_argument('--actuals-filename', | 256 parser.add_argument('--actuals-filename', |
281 help='filename (within platform-specific subdirectories ' + | 257 help='filename (within builder-specific subdirectories ' + |
282 'of ACTUALS_BASE_URL) to read a summary of results from; ' + | 258 'of ACTUALS_BASE_URL) to read a summary of results from; ' + |
283 'defaults to %(default)s', | 259 'defaults to %(default)s', |
284 default='actual-results.json') | 260 default='actual-results.json') |
285 # TODO(epoger): Add test that exercises --add-new argument. | 261 # TODO(epoger): Add test that exercises --add-new argument. |
286 parser.add_argument('--add-new', action='store_true', | 262 parser.add_argument('--add-new', action='store_true', |
287 help='in addition to the standard behavior of ' + | 263 help='in addition to the standard behavior of ' + |
288 'updating expectations for failing tests, add ' + | 264 'updating expectations for failing tests, add ' + |
289 'expectations for tests which don\'t have expectations ' + | 265 'expectations for tests which don\'t have expectations ' + |
290 'yet.') | 266 'yet.') |
| 267 parser.add_argument('--builders', metavar='BUILDER', nargs='+', |
| 268 help='which platforms to rebaseline; ' + |
| 269 'if unspecified, rebaseline all platforms, same as ' + |
| 270 '"--builders %s"' % ' '.join(sorted(TEST_BUILDERS))) |
291 # TODO(epoger): Add test that exercises --configs argument. | 271 # TODO(epoger): Add test that exercises --configs argument. |
292 parser.add_argument('--configs', metavar='CONFIG', nargs='+', | 272 parser.add_argument('--configs', metavar='CONFIG', nargs='+', |
293 help='which configurations to rebaseline, e.g. ' + | 273 help='which configurations to rebaseline, e.g. ' + |
294 '"--configs 565 8888", as a filter over the full set of ' + | 274 '"--configs 565 8888", as a filter over the full set of ' + |
295 'results in ACTUALS_FILENAME; if unspecified, rebaseline ' + | 275 'results in ACTUALS_FILENAME; if unspecified, rebaseline ' + |
296 '*all* configs that are available.') | 276 '*all* configs that are available.') |
297 parser.add_argument('--expectations-filename', | 277 parser.add_argument('--expectations-filename', |
298 help='filename (under EXPECTATIONS_ROOT) to read ' + | 278 help='filename (under EXPECTATIONS_ROOT) to read ' + |
299 'current expectations from, and to write new ' + | 279 'current expectations from, and to write new ' + |
300 'expectations into (unless a separate ' + | 280 'expectations into (unless a separate ' + |
301 'EXPECTATIONS_FILENAME_OUTPUT has been specified); ' + | 281 'EXPECTATIONS_FILENAME_OUTPUT has been specified); ' + |
302 'defaults to %(default)s', | 282 'defaults to %(default)s', |
303 default='expected-results.json') | 283 default='expected-results.json') |
304 parser.add_argument('--expectations-filename-output', | 284 parser.add_argument('--expectations-filename-output', |
305 help='filename (under EXPECTATIONS_ROOT) to write ' + | 285 help='filename (under EXPECTATIONS_ROOT) to write ' + |
306 'updated expectations into; by default, overwrites the ' + | 286 'updated expectations into; by default, overwrites the ' + |
307 'input file (EXPECTATIONS_FILENAME)', | 287 'input file (EXPECTATIONS_FILENAME)', |
308 default='') | 288 default='') |
309 parser.add_argument('--expectations-root', | 289 parser.add_argument('--expectations-root', |
310 help='root of expectations directory to update-- should ' + | 290 help='root of expectations directory to update-- should ' + |
311 'contain one or more base-* subdirectories. Defaults to ' + | 291 'contain one or more builder subdirectories. Defaults to ' + |
312 '%(default)s', | 292 '%(default)s', |
313 default=os.path.join('expectations', 'gm')) | 293 default=os.path.join('expectations', 'gm')) |
314 parser.add_argument('--keep-going-on-failure', action='store_true', | 294 parser.add_argument('--keep-going-on-failure', action='store_true', |
315 help='instead of halting at the first error encountered, ' + | 295 help='instead of halting at the first error encountered, ' + |
316 'keep going and rebaseline as many tests as possible, ' + | 296 'keep going and rebaseline as many tests as possible, ' + |
317 'and then report the full set of errors at the end') | 297 'and then report the full set of errors at the end') |
318 parser.add_argument('--subdirs', metavar='SUBDIR', nargs='+', | |
319 help='which platform subdirectories to rebaseline; ' + | |
320 'if unspecified, rebaseline all subdirs, same as ' + | |
321 '"--subdirs %s"' % ' '.join(sorted(SUBDIR_MAPPING.keys()))) | |
322 # TODO(epoger): Add test that exercises --tests argument. | 298 # TODO(epoger): Add test that exercises --tests argument. |
323 parser.add_argument('--tests', metavar='TEST', nargs='+', | 299 parser.add_argument('--tests', metavar='TEST', nargs='+', |
324 help='which tests to rebaseline, e.g. ' + | 300 help='which tests to rebaseline, e.g. ' + |
325 '"--tests aaclip bigmatrix", as a filter over the full ' + | 301 '"--tests aaclip bigmatrix", as a filter over the full ' + |
326 'set of results in ACTUALS_FILENAME; if unspecified, ' + | 302 'set of results in ACTUALS_FILENAME; if unspecified, ' + |
327 'rebaseline *all* tests that are available.') | 303 'rebaseline *all* tests that are available.') |
328 args = parser.parse_args() | 304 args = parser.parse_args() |
329 exception_handler = ExceptionHandler( | 305 exception_handler = ExceptionHandler( |
330 keep_going_on_failure=args.keep_going_on_failure) | 306 keep_going_on_failure=args.keep_going_on_failure) |
331 if args.subdirs: | 307 if args.builders: |
332 subdirs = args.subdirs | 308 builders = args.builders |
333 missing_json_is_fatal = True | 309 missing_json_is_fatal = True |
334 else: | 310 else: |
335 subdirs = sorted(SUBDIR_MAPPING.keys()) | 311 builders = sorted(TEST_BUILDERS) |
336 missing_json_is_fatal = False | 312 missing_json_is_fatal = False |
337 for subdir in subdirs: | 313 for builder in builders: |
338 if not subdir in SUBDIR_MAPPING.keys(): | 314 if not builder in TEST_BUILDERS: |
339 raise Exception(('unrecognized platform subdir "%s"; ' + | 315 raise Exception(('unrecognized builder "%s"; ' + |
340 'should be one of %s') % ( | 316 'should be one of %s') % ( |
341 subdir, SUBDIR_MAPPING.keys())) | 317 builder, TEST_BUILDERS)) |
342 builder = SUBDIR_MAPPING[subdir] | |
343 | 318 |
344 # We instantiate different Rebaseliner objects depending | 319 expectations_json_file = os.path.join(args.expectations_root, builder, |
345 # on whether we are rebaselining an expected-results.json file, or | |
346 # individual image files. Different expectations/gm subdirectories may move | |
347 # from individual image files to JSON-format expectations at different | |
348 # times, so we need to make this determination per subdirectory. | |
349 # | |
350 # See https://goto.google.com/ChecksumTransitionDetail | |
351 expectations_json_file = os.path.join(args.expectations_root, subdir, | |
352 args.expectations_filename) | 320 args.expectations_filename) |
353 if os.path.isfile(expectations_json_file): | 321 if os.path.isfile(expectations_json_file): |
354 rebaseliner = JsonRebaseliner( | 322 rebaseliner = JsonRebaseliner( |
355 expectations_root=args.expectations_root, | 323 expectations_root=args.expectations_root, |
356 expectations_input_filename=args.expectations_filename, | 324 expectations_input_filename=args.expectations_filename, |
357 expectations_output_filename=(args.expectations_filename_output or | 325 expectations_output_filename=(args.expectations_filename_output or |
358 args.expectations_filename), | 326 args.expectations_filename), |
359 tests=args.tests, configs=args.configs, | 327 tests=args.tests, configs=args.configs, |
360 actuals_base_url=args.actuals_base_url, | 328 actuals_base_url=args.actuals_base_url, |
361 actuals_filename=args.actuals_filename, | 329 actuals_filename=args.actuals_filename, |
362 exception_handler=exception_handler, | 330 exception_handler=exception_handler, |
363 add_new=args.add_new) | 331 add_new=args.add_new) |
364 try: | 332 try: |
365 rebaseliner.RebaselineSubdir(subdir=subdir, builder=builder) | 333 rebaseliner.RebaselineSubdir(builder=builder) |
366 except BaseException as e: | 334 except BaseException as e: |
367 exception_handler.RaiseExceptionOrContinue(e) | 335 exception_handler.RaiseExceptionOrContinue(e) |
368 else: | 336 else: |
369 exception_handler.RaiseExceptionOrContinue(_InternalException( | 337 exception_handler.RaiseExceptionOrContinue(_InternalException( |
370 'expectations_json_file %s not found' % expectations_json_file)) | 338 'expectations_json_file %s not found' % expectations_json_file)) |
371 | 339 |
372 exception_handler.ReportAllFailures() | 340 exception_handler.ReportAllFailures() |
OLD | NEW |