Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(177)

Side by Side Diff: webkit/tools/layout_tests/webkitpy/rebaseline.py

Issue 545145: Move the layout test scripts into a 'webkitpy' subdirectory in preparation... (Closed) Base URL: svn://chrome-svn/chrome/trunk/src/
Patch Set: try to de-confuse svn and the try bots Created 10 years, 11 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch | Annotate | Revision Log
OLDNEW
1 #!usr/bin/env python 1 #!usr/bin/env python
2 # Copyright (c) 2006-2009 The Chromium Authors. All rights reserved. 2 # Copyright (c) 2006-2009 The Chromium Authors. All rights reserved.
3 # Use of this source code is governed by a BSD-style license that can be 3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file. 4 # found in the LICENSE file.
5 5
6 """Rebaselining tool that automatically produces baselines for all platforms. 6 """Rebaselining tool that automatically produces baselines for all platforms.
7 7
8 The script does the following for each platform specified: 8 The script does the following for each platform specified:
9 1. Compile a list of tests that need rebaselining. 9 1. Compile a list of tests that need rebaselining.
10 2. Download test result archive from buildbot for the platform. 10 2. Download test result archive from buildbot for the platform.
(...skipping 33 matching lines...) Expand 10 before | Expand all | Expand 10 after
44 'win-vista': 'webkit-dbg-vista', 44 'win-vista': 'webkit-dbg-vista',
45 'win-xp': 'webkit-rel', 45 'win-xp': 'webkit-rel',
46 'mac': 'webkit-rel-mac5', 46 'mac': 'webkit-rel-mac5',
47 'linux': 'webkit-rel-linux', 47 'linux': 'webkit-rel-linux',
48 'win-canary': 'webkit-rel-webkit-org', 48 'win-canary': 'webkit-rel-webkit-org',
49 'win-vista-canary': 'webkit-dbg-vista', 49 'win-vista-canary': 'webkit-dbg-vista',
50 'win-xp-canary': 'webkit-rel-webkit-org', 50 'win-xp-canary': 'webkit-rel-webkit-org',
51 'mac-canary': 'webkit-rel-mac-webkit-org', 51 'mac-canary': 'webkit-rel-mac-webkit-org',
52 'linux-canary': 'webkit-rel-linux-webkit-org'} 52 'linux-canary': 'webkit-rel-linux-webkit-org'}
53 53
54
55 def RunShellWithReturnCode(command, print_output=False): 54 def RunShellWithReturnCode(command, print_output=False):
56 """Executes a command and returns the output and process return code. 55 """Executes a command and returns the output and process return code.
57 56
58 Args: 57 Args:
59 command: program and arguments. 58 command: program and arguments.
60 print_output: if true, print the command results to standard output. 59 print_output: if true, print the command results to standard output.
61 60
62 Returns: 61 Returns:
63 command output, return code 62 command output, return code
64 """ 63 """
65 64
66 # Use a shell for subcommands on Windows to get a PATH search. 65 # Use a shell for subcommands on Windows to get a PATH search.
67 use_shell = sys.platform.startswith('win') 66 use_shell = sys.platform.startswith('win')
68 p = subprocess.Popen(command, stdout=subprocess.PIPE, 67 p = subprocess.Popen(command, stdout=subprocess.PIPE,
69 stderr=subprocess.STDOUT, shell=use_shell) 68 stderr=subprocess.STDOUT, shell=use_shell)
70 if print_output: 69 if print_output:
71 output_array = [] 70 output_array = []
72 while True: 71 while True:
73 line = p.stdout.readline() 72 line = p.stdout.readline()
74 if not line: 73 if not line:
75 break 74 break
76 if print_output: 75 if print_output:
77 print line.strip('\n') 76 print line.strip('\n')
78 output_array.append(line) 77 output_array.append(line)
79 output = ''.join(output_array) 78 output = ''.join(output_array)
80 else: 79 else:
81 output = p.stdout.read() 80 output = p.stdout.read()
82 p.wait() 81 p.wait()
83 p.stdout.close() 82 p.stdout.close()
84 83
85 return output, p.returncode 84 return output, p.returncode
86
87 85
88 def RunShell(command, print_output=False): 86 def RunShell(command, print_output=False):
89 """Executes a command and returns the output. 87 """Executes a command and returns the output.
90 88
91 Args: 89 Args:
92 command: program and arguments. 90 command: program and arguments.
93 print_output: if true, print the command results to standard output. 91 print_output: if true, print the command results to standard output.
94 92
95 Returns: 93 Returns:
96 command output 94 command output
97 """ 95 """
98 96
99 output, return_code = RunShellWithReturnCode(command, print_output) 97 output, return_code = RunShellWithReturnCode(command, print_output)
100 return output 98 return output
101
102 99
103 def LogDashedString(text, platform, logging_level=logging.INFO): 100 def LogDashedString(text, platform, logging_level=logging.INFO):
104 """Log text message with dashes on both sides.""" 101 """Log text message with dashes on both sides."""
105 102
106 msg = text 103 msg = text
107 if platform: 104 if platform:
108 msg += ': ' + platform 105 msg += ': ' + platform
109 if len(msg) < 78: 106 if len(msg) < 78:
110 dashes = '-' * ((78 - len(msg)) / 2) 107 dashes = '-' * ((78 - len(msg)) / 2)
111 msg = '%s %s %s' % (dashes, msg, dashes) 108 msg = '%s %s %s' % (dashes, msg, dashes)
112 109
113 if logging_level == logging.ERROR: 110 if logging_level == logging.ERROR:
114 logging.error(msg) 111 logging.error(msg)
115 elif logging_level == logging.WARNING: 112 elif logging_level == logging.WARNING:
116 logging.warn(msg) 113 logging.warn(msg)
117 else: 114 else:
118 logging.info(msg) 115 logging.info(msg)
119 116
120 117
121 def SetupHtmlDirectory(html_directory): 118 def SetupHtmlDirectory(html_directory):
122 """Setup the directory to store html results. 119 """Setup the directory to store html results.
123 120
124 All html related files are stored in the "rebaseline_html" subdirectory. 121 All html related files are stored in the "rebaseline_html" subdirectory.
125 122
126 Args: 123 Args:
127 html_directory: parent directory that stores the rebaselining results. 124 html_directory: parent directory that stores the rebaselining results.
128 If None, a temp directory is created. 125 If None, a temp directory is created.
129 126
130 Returns: 127 Returns:
131 the directory that stores the html related rebaselining results. 128 the directory that stores the html related rebaselining results.
132 """ 129 """
133 130
134 if not html_directory: 131 if not html_directory:
135 html_directory = tempfile.mkdtemp() 132 html_directory = tempfile.mkdtemp()
136 elif not os.path.exists(html_directory): 133 elif not os.path.exists(html_directory):
137 os.mkdir(html_directory) 134 os.mkdir(html_directory)
138 135
139 html_directory = os.path.join(html_directory, 'rebaseline_html') 136 html_directory = os.path.join(html_directory, 'rebaseline_html')
140 logging.info('Html directory: "%s"', html_directory) 137 logging.info('Html directory: "%s"', html_directory)
141 138
142 if os.path.exists(html_directory): 139 if os.path.exists(html_directory):
143 shutil.rmtree(html_directory, True) 140 shutil.rmtree(html_directory, True)
144 logging.info('Deleted file at html directory: "%s"', html_directory) 141 logging.info('Deleted file at html directory: "%s"', html_directory)
145 142
146 if not os.path.exists(html_directory): 143 if not os.path.exists(html_directory):
147 os.mkdir(html_directory) 144 os.mkdir(html_directory)
148 return html_directory 145 return html_directory
149 146
150 147
151 def GetResultFileFullpath(html_directory, baseline_filename, platform, 148 def GetResultFileFullpath(html_directory, baseline_filename, platform,
152 result_type): 149 result_type):
153 """Get full path of the baseline result file. 150 """Get full path of the baseline result file.
154 151
155 Args: 152 Args:
156 html_directory: directory that stores the html related files. 153 html_directory: directory that stores the html related files.
157 baseline_filename: name of the baseline file. 154 baseline_filename: name of the baseline file.
158 platform: win, linux or mac 155 platform: win, linux or mac
159 result_type: type of the baseline result: '.txt', '.png'. 156 result_type: type of the baseline result: '.txt', '.png'.
160 157
161 Returns: 158 Returns:
162 Full path of the baseline file for rebaselining result comparison. 159 Full path of the baseline file for rebaselining result comparison.
163 """ 160 """
164 161
165 base, ext = os.path.splitext(baseline_filename) 162 base, ext = os.path.splitext(baseline_filename)
166 result_filename = '%s-%s-%s%s' % (base, platform, result_type, ext) 163 result_filename = '%s-%s-%s%s' % (base, platform, result_type, ext)
167 fullpath = os.path.join(html_directory, result_filename) 164 fullpath = os.path.join(html_directory, result_filename)
168 logging.debug(' Result file full path: "%s".', fullpath) 165 logging.debug(' Result file full path: "%s".', fullpath)
169 return fullpath 166 return fullpath
170 167
171 168
172 class Rebaseliner(object): 169 class Rebaseliner(object):
173 """Class to produce new baselines for a given platform.""" 170 """Class to produce new baselines for a given platform."""
174 171
175 REVISION_REGEX = r'<a href=\"(\d+)/\">' 172 REVISION_REGEX = r'<a href=\"(\d+)/\">'
176 173
177 def __init__(self, platform, options): 174 def __init__(self, platform, options):
178 self._file_dir = path_utils.GetAbsolutePath( 175 self._file_dir = path_utils.GetAbsolutePath(
179 os.path.dirname(sys.argv[0])) 176 os.path.dirname(os.path.dirname(sys.argv[0])))
180 self._platform = platform 177 self._platform = platform
181 self._options = options 178 self._options = options
182 self._rebaselining_tests = [] 179 self._rebaselining_tests = []
183 self._rebaselined_tests = [] 180 self._rebaselined_tests = []
184 181
185 # Create tests and expectations helper which is used to: 182 # Create tests and expectations helper which is used to:
186 # -. compile list of tests that need rebaselining. 183 # -. compile list of tests that need rebaselining.
187 # -. update the tests in test_expectations file after rebaseline 184 # -. update the tests in test_expectations file after rebaseline is done.
188 # is done. 185 self._test_expectations = test_expectations.TestExpectations(None,
189 self._test_expectations = \ 186 self._file_dir,
190 test_expectations.TestExpectations(None, 187 platform,
191 self._file_dir, 188 False,
192 platform, 189 False)
193 False, 190
194 False) 191 self._repo_type = self._GetRepoType()
195 192
196 self._repo_type = self._GetRepoType() 193 def Run(self, backup):
197 194 """Run rebaseline process."""
198 def Run(self, backup): 195
199 """Run rebaseline process.""" 196 LogDashedString('Compiling rebaselining tests', self._platform)
200 197 if not self._CompileRebaseliningTests():
201 LogDashedString('Compiling rebaselining tests', self._platform) 198 return True
202 if not self._CompileRebaseliningTests(): 199
200 LogDashedString('Downloading archive', self._platform)
201 archive_file = self._DownloadBuildBotArchive()
202 logging.info('')
203 if not archive_file:
204 logging.error('No archive found.')
205 return False
206
207 LogDashedString('Extracting and adding new baselines', self._platform)
208 if not self._ExtractAndAddNewBaselines(archive_file):
209 return False
210
211 LogDashedString('Updating rebaselined tests in file', self._platform)
212 self._UpdateRebaselinedTestsInFile(backup)
213 logging.info('')
214
215 if len(self._rebaselining_tests) != len(self._rebaselined_tests):
216 logging.warning('NOT ALL TESTS THAT NEED REBASELINING HAVE BEEN '
217 'REBASELINED.')
218 logging.warning(' Total tests needing rebaselining: %d',
219 len(self._rebaselining_tests))
220 logging.warning(' Total tests rebaselined: %d',
221 len(self._rebaselined_tests))
222 return False
223
224 logging.warning('All tests needing rebaselining were successfully '
225 'rebaselined.')
226
227 return True
228
229 def GetRebaseliningTests(self):
230 return self._rebaselining_tests
231
232 def _GetRepoType(self):
233 """Get the repository type that client is using."""
234
235 output, return_code = RunShellWithReturnCode(['svn', 'info'], False)
236 if return_code == 0:
237 return REPO_SVN
238
239 return REPO_UNKNOWN
240
241 def _CompileRebaseliningTests(self):
242 """Compile list of tests that need rebaselining for the platform.
243
244 Returns:
245 List of tests that need rebaselining or
246 None if there is no such test.
247 """
248
249 self._rebaselining_tests = self._test_expectations.GetRebaseliningFailures()
250 if not self._rebaselining_tests:
251 logging.warn('No tests found that need rebaselining.')
252 return None
253
254 logging.info('Total number of tests needing rebaselining for "%s": "%d"',
255 self._platform, len(self._rebaselining_tests))
256
257 test_no = 1
258 for test in self._rebaselining_tests:
259 logging.info(' %d: %s', test_no, test)
260 test_no += 1
261
262 return self._rebaselining_tests
263
264 def _GetLatestRevision(self, url):
265 """Get the latest layout test revision number from buildbot.
266
267 Args:
268 url: Url to retrieve layout test revision numbers.
269
270 Returns:
271 latest revision or
272 None on failure.
273 """
274
275 logging.debug('Url to retrieve revision: "%s"', url)
276
277 f = urllib.urlopen(url)
278 content = f.read()
279 f.close()
280
281 revisions = re.findall(self.REVISION_REGEX, content)
282 if not revisions:
283 logging.error('Failed to find revision, content: "%s"', content)
284 return None
285
286 revisions.sort(key=int)
287 logging.info('Latest revision: "%s"', revisions[len(revisions) - 1])
288 return revisions[len(revisions) - 1]
289
290 def _GetArchiveDirName(self, platform, webkit_canary):
291 """Get name of the layout test archive directory.
292
293 Returns:
294 Directory name or
295 None on failure
296 """
297
298 if webkit_canary:
299 platform += '-canary'
300
301 if platform in ARCHIVE_DIR_NAME_DICT:
302 return ARCHIVE_DIR_NAME_DICT[platform]
303 else:
304 logging.error('Cannot find platform key %s in archive directory name '
305 'dictionary', platform)
306 return None
307
308 def _GetArchiveUrl(self):
309 """Generate the url to download latest layout test archive.
310
311 Returns:
312 Url to download archive or
313 None on failure
314 """
315
316 dir_name = self._GetArchiveDirName(self._platform,
317 self._options.webkit_canary)
318 if not dir_name:
319 return None
320
321 logging.debug('Buildbot platform dir name: "%s"', dir_name)
322
323 url_base = '%s/%s/' % (self._options.archive_url, dir_name)
324 latest_revision = self._GetLatestRevision(url_base)
325 if latest_revision is None or latest_revision <= 0:
326 return None
327
328 archive_url = ('%s%s/layout-test-results.zip' % (url_base,
329 latest_revision))
330 logging.info('Archive url: "%s"', archive_url)
331 return archive_url
332
333 def _DownloadBuildBotArchive(self):
334 """Download layout test archive file from buildbot.
335
336 Returns:
337 True if download succeeded or
338 False otherwise.
339 """
340
341 url = self._GetArchiveUrl()
342 if url is None:
343 return None
344
345 fn = urllib.urlretrieve(url)[0]
346 logging.info('Archive downloaded and saved to file: "%s"', fn)
347 return fn
348
349 def _ExtractAndAddNewBaselines(self, archive_file):
350 """Extract new baselines from archive and add them to SVN repository.
351
352 Args:
353 archive_file: full path to the archive file.
354
355 Returns:
356 List of tests that have been rebaselined or
357 None on failure.
358 """
359
360 zip_file = zipfile.ZipFile(archive_file, 'r')
361 zip_namelist = zip_file.namelist()
362
363 logging.debug('zip file namelist:')
364 for name in zip_namelist:
365 logging.debug(' ' + name)
366
367 platform = path_utils.PlatformName(self._platform)
368 logging.debug('Platform dir: "%s"', platform)
369
370 test_no = 1
371 self._rebaselined_tests = []
372 for test in self._rebaselining_tests:
373 logging.info('Test %d: %s', test_no, test)
374
375 found = False
376 svn_error = False
377 test_basename = os.path.splitext(test)[0]
378 for suffix in BASELINE_SUFFIXES:
379 archive_test_name = 'layout-test-results/%s-actual%s' % (test_basename,
380 suffix)
381 logging.debug(' Archive test file name: "%s"', archive_test_name)
382 if not archive_test_name in zip_namelist:
383 logging.info(' %s file not in archive.', suffix)
384 continue
385
386 found = True
387 logging.info(' %s file found in archive.', suffix)
388
389 # Extract new baseline from archive and save it to a temp file.
390 data = zip_file.read(archive_test_name)
391 temp_fd, temp_name = tempfile.mkstemp(suffix)
392 f = os.fdopen(temp_fd, 'wb')
393 f.write(data)
394 f.close()
395
396 expected_filename = '%s-expected%s' % (test_basename, suffix)
397 expected_fullpath = os.path.join(
398 path_utils.ChromiumBaselinePath(platform), expected_filename)
399 expected_fullpath = os.path.normpath(expected_fullpath)
400 logging.debug(' Expected file full path: "%s"', expected_fullpath)
401
402 # TODO(victorw): for now, the rebaselining tool checks whether
403 # or not THIS baseline is duplicate and should be skipped.
404 # We could improve the tool to check all baselines in upper and lower
405 # levels and remove all duplicated baselines.
406 if self._IsDupBaseline(temp_name,
407 expected_fullpath,
408 test,
409 suffix,
410 self._platform):
411 os.remove(temp_name)
412 self._DeleteBaseline(expected_fullpath)
413 continue
414
415 # Create the new baseline directory if it doesn't already exist.
416 path_utils.MaybeMakeDirectory(os.path.dirname(expected_fullpath))
417
418 shutil.move(temp_name, expected_fullpath)
419
420 if not self._SvnAdd(expected_fullpath):
421 svn_error = True
422 elif suffix != '.checksum':
423 self._CreateHtmlBaselineFiles(expected_fullpath)
424
425 if not found:
426 logging.warn(' No new baselines found in archive.')
427 else:
428 if svn_error:
429 logging.warn(' Failed to add baselines to SVN.')
430 else:
431 logging.info(' Rebaseline succeeded.')
432 self._rebaselined_tests.append(test)
433
434 test_no += 1
435
436 zip_file.close()
437 os.remove(archive_file)
438
439 return self._rebaselined_tests
440
441 def _IsDupBaseline(self, new_baseline, baseline_path, test, suffix, platform):
442 """Check whether a baseline is duplicate and can fallback to same
443 baseline for another platform. For example, if a test has same baseline
444 on linux and windows, then we only store windows baseline and linux
445 baseline will fallback to the windows version.
446
447 Args:
448 expected_filename: baseline expectation file name.
449 test: test name.
450 suffix: file suffix of the expected results, including dot; e.g. '.txt'
451 or '.png'.
452 platform: baseline platform 'mac', 'win' or 'linux'.
453
454 Returns:
455 True if the baseline is unnecessary.
456 False otherwise.
457 """
458 test_filepath = os.path.join(path_utils.LayoutTestsDir(), test)
459 all_baselines = path_utils.ExpectedBaselines(test_filepath,
460 suffix,
461 platform,
462 True)
463 for (fallback_dir, fallback_file) in all_baselines:
464 if fallback_dir and fallback_file:
465 fallback_fullpath = os.path.normpath(
466 os.path.join(fallback_dir, fallback_file))
467 if fallback_fullpath.lower() != baseline_path.lower():
468 if not self._DiffBaselines(new_baseline, fallback_fullpath):
469 logging.info(' Found same baseline at %s', fallback_fullpath)
203 return True 470 return True
204 471 else:
205 LogDashedString('Downloading archive', self._platform)
206 archive_file = self._DownloadBuildBotArchive()
207 logging.info('')
208 if not archive_file:
209 logging.error('No archive found.')
210 return False 472 return False
211 473
212 LogDashedString('Extracting and adding new baselines', self._platform) 474 return False
213 if not self._ExtractAndAddNewBaselines(archive_file): 475
214 return False 476 def _DiffBaselines(self, file1, file2):
215 477 """Check whether two baselines are different.
216 LogDashedString('Updating rebaselined tests in file', self._platform) 478
217 self._UpdateRebaselinedTestsInFile(backup) 479 Args:
218 logging.info('') 480 file1, file2: full paths of the baselines to compare.
219 481
220 if len(self._rebaselining_tests) != len(self._rebaselined_tests): 482 Returns:
221 logging.warning('NOT ALL TESTS THAT NEED REBASELINING HAVE BEEN ' 483 True if two files are different or have different extensions.
222 'REBASELINED.') 484 False otherwise.
223 logging.warning(' Total tests needing rebaselining: %d', 485 """
224 len(self._rebaselining_tests)) 486
225 logging.warning(' Total tests rebaselined: %d', 487 ext1 = os.path.splitext(file1)[1].upper()
226 len(self._rebaselined_tests)) 488 ext2 = os.path.splitext(file2)[1].upper()
227 return False 489 if ext1 != ext2:
228 490 logging.warn('Files to compare have different ext. File1: %s; File2: %s',
229 logging.warning('All tests needing rebaselining were successfully ' 491 file1, file2)
230 'rebaselined.') 492 return True
231 493
232 return True 494 if ext1 == '.PNG':
233 495 return image_diff.ImageDiff(self._platform, '').DiffFiles(file1,
234 def GetRebaseliningTests(self): 496 file2)
235 return self._rebaselining_tests 497 else:
236 498 return text_diff.TestTextDiff(self._platform, '').DiffFiles(file1,
237 def _GetRepoType(self): 499 file2)
238 """Get the repository type that client is using.""" 500
239 501 def _DeleteBaseline(self, filename):
240 output, return_code = RunShellWithReturnCode(['svn', 'info'], False) 502 """Remove the file from repository and delete it from disk.
241 if return_code == 0: 503
242 return REPO_SVN 504 Args:
243 505 filename: full path of the file to delete.
244 return REPO_UNKNOWN 506 """
245 507
246 def _CompileRebaseliningTests(self): 508 if not filename or not os.path.isfile(filename):
247 """Compile list of tests that need rebaselining for the platform. 509 return
248 510
249 Returns: 511 if self._repo_type == REPO_SVN:
250 List of tests that need rebaselining or 512 parent_dir, basename = os.path.split(filename)
251 None if there is no such test. 513 original_dir = os.getcwd()
252 """ 514 os.chdir(parent_dir)
253 515 RunShell(['svn', 'delete', '--force', basename], False)
254 self._rebaselining_tests = \ 516 os.chdir(original_dir)
255 self._test_expectations.GetRebaseliningFailures() 517 else:
256 if not self._rebaselining_tests: 518 os.remove(filename)
257 logging.warn('No tests found that need rebaselining.') 519
258 return None 520 def _UpdateRebaselinedTestsInFile(self, backup):
259 521 """Update the rebaselined tests in test expectations file.
260 logging.info('Total number of tests needing rebaselining ' 522
261 'for "%s": "%d"', self._platform, 523 Args:
262 len(self._rebaselining_tests)) 524 backup: if True, backup the original test expectations file.
263 525
264 test_no = 1 526 Returns:
265 for test in self._rebaselining_tests: 527 no
266 logging.info(' %d: %s', test_no, test) 528 """
267 test_no += 1 529
268 530 if self._rebaselined_tests:
269 return self._rebaselining_tests 531 self._test_expectations.RemovePlatformFromFile(self._rebaselined_tests,
270 532 self._platform,
271 def _GetLatestRevision(self, url): 533 backup)
272 """Get the latest layout test revision number from buildbot. 534 else:
273 535 logging.info('No test was rebaselined so nothing to remove.')
274 Args: 536
275 url: Url to retrieve layout test revision numbers. 537 def _SvnAdd(self, filename):
276 538 """Add the file to SVN repository.
277 Returns: 539
278 latest revision or 540 Args:
279 None on failure. 541 filename: full path of the file to add.
280 """ 542
281 543 Returns:
282 logging.debug('Url to retrieve revision: "%s"', url) 544 True if the file already exists in SVN or is sucessfully added to SVN.
283 545 False otherwise.
284 f = urllib.urlopen(url) 546 """
285 content = f.read() 547
286 f.close() 548 if not filename:
287 549 return False
288 revisions = re.findall(self.REVISION_REGEX, content) 550
289 if not revisions: 551 parent_dir, basename = os.path.split(filename)
290 logging.error('Failed to find revision, content: "%s"', content) 552 if self._repo_type != REPO_SVN or parent_dir == filename:
291 return None 553 logging.info("No svn checkout found, skip svn add.")
292 554 return True
293 revisions.sort(key=int) 555
294 logging.info('Latest revision: "%s"', revisions[len(revisions) - 1]) 556 original_dir = os.getcwd()
295 return revisions[len(revisions) - 1] 557 os.chdir(parent_dir)
296 558 status_output = RunShell(['svn', 'status', basename], False)
297 def _GetArchiveDirName(self, platform, webkit_canary): 559 os.chdir(original_dir)
298 """Get name of the layout test archive directory. 560 output = status_output.upper()
299 561 if output.startswith('A') or output.startswith('M'):
300 Returns: 562 logging.info(' File already added to SVN: "%s"', filename)
301 Directory name or 563 return True
302 None on failure 564
303 """ 565 if output.find('IS NOT A WORKING COPY') >= 0:
304 566 logging.info(' File is not a working copy, add its parent: "%s"',
305 if webkit_canary: 567 parent_dir)
306 platform += '-canary' 568 return self._SvnAdd(parent_dir)
307 569
308 if platform in ARCHIVE_DIR_NAME_DICT: 570 os.chdir(parent_dir)
309 return ARCHIVE_DIR_NAME_DICT[platform] 571 add_output = RunShell(['svn', 'add', basename], True)
310 else: 572 os.chdir(original_dir)
311 logging.error('Cannot find platform key %s in archive ' 573 output = add_output.upper().rstrip()
312 'directory name dictionary', platform) 574 if output.startswith('A') and output.find(basename.upper()) >= 0:
313 return None 575 logging.info(' Added new file: "%s"', filename)
314 576 self._SvnPropSet(filename)
315 def _GetArchiveUrl(self): 577 return True
316 """Generate the url to download latest layout test archive. 578
317 579 if (not status_output) and (add_output.upper().find(
318 Returns: 580 'ALREADY UNDER VERSION CONTROL') >= 0):
319 Url to download archive or 581 logging.info(' File already under SVN and has no change: "%s"', filename)
320 None on failure 582 return True
321 """ 583
322 584 logging.warn(' Failed to add file to SVN: "%s"', filename)
323 dir_name = self._GetArchiveDirName(self._platform, 585 logging.warn(' Svn status output: "%s"', status_output)
324 self._options.webkit_canary) 586 logging.warn(' Svn add output: "%s"', add_output)
325 if not dir_name: 587 return False
326 return None 588
327 589 def _SvnPropSet(self, filename):
328 logging.debug('Buildbot platform dir name: "%s"', dir_name) 590 """Set the baseline property
329 591
330 url_base = '%s/%s/' % (self._options.archive_url, dir_name) 592 Args:
331 latest_revision = self._GetLatestRevision(url_base) 593 filename: full path of the file to add.
332 if latest_revision is None or latest_revision <= 0: 594
333 return None 595 Returns:
334 596 True if the file already exists in SVN or is sucessfully added to SVN.
335 archive_url = ('%s%s/layout-test-results.zip' % (url_base, 597 False otherwise.
336 latest_revision)) 598 """
337 logging.info('Archive url: "%s"', archive_url) 599 ext = os.path.splitext(filename)[1].upper()
338 return archive_url 600 if ext != '.TXT' and ext != '.PNG' and ext != '.CHECKSUM':
339 601 return
340 def _DownloadBuildBotArchive(self): 602
341 """Download layout test archive file from buildbot. 603 parent_dir, basename = os.path.split(filename)
342 604 original_dir = os.getcwd()
343 Returns: 605 os.chdir(parent_dir)
344 True if download succeeded or 606 if ext == '.PNG':
345 False otherwise. 607 cmd = [ 'svn', 'pset', 'svn:mime-type', 'image/png', basename ]
346 """ 608 else:
347 609 cmd = [ 'svn', 'pset', 'svn:eol-style', 'LF', basename ]
348 url = self._GetArchiveUrl() 610
349 if url is None: 611 logging.debug(' Set svn prop: %s', ' '.join(cmd))
350 return None 612 RunShell(cmd, False)
351 613 os.chdir(original_dir)
352 fn = urllib.urlretrieve(url)[0] 614
353 logging.info('Archive downloaded and saved to file: "%s"', fn) 615 def _CreateHtmlBaselineFiles(self, baseline_fullpath):
354 return fn 616 """Create baseline files (old, new and diff) in html directory.
355 617
356 def _ExtractAndAddNewBaselines(self, archive_file): 618 The files are used to compare the rebaselining results.
357 """Extract new baselines from archive and add them to SVN repository. 619
358 620 Args:
359 Args: 621 baseline_fullpath: full path of the expected baseline file.
360 archive_file: full path to the archive file. 622 """
361 623
362 Returns: 624 if not baseline_fullpath or not os.path.exists(baseline_fullpath):
363 List of tests that have been rebaselined or 625 return
364 None on failure. 626
365 """ 627 # Copy the new baseline to html directory for result comparison.
366 628 baseline_filename = os.path.basename(baseline_fullpath)
367 zip_file = zipfile.ZipFile(archive_file, 'r') 629 new_file = GetResultFileFullpath(self._options.html_directory,
368 zip_namelist = zip_file.namelist() 630 baseline_filename,
369 631 self._platform,
370 logging.debug('zip file namelist:') 632 'new')
371 for name in zip_namelist: 633 shutil.copyfile(baseline_fullpath, new_file)
372 logging.debug(' ' + name) 634 logging.info(' Html: copied new baseline file from "%s" to "%s".',
373 635 baseline_fullpath, new_file)
374 platform = path_utils.PlatformName(self._platform) 636
375 logging.debug('Platform dir: "%s"', platform) 637 # Get the old baseline from SVN and save to the html directory.
376 638 output = RunShell(['svn', 'cat', '-r', 'BASE', baseline_fullpath])
377 test_no = 1 639 if (not output) or (output.upper().rstrip().endswith(
378 self._rebaselined_tests = [] 640 'NO SUCH FILE OR DIRECTORY')):
379 for test in self._rebaselining_tests: 641 logging.info(' No base file: "%s"', baseline_fullpath)
380 logging.info('Test %d: %s', test_no, test) 642 return
381 643 base_file = GetResultFileFullpath(self._options.html_directory,
382 found = False 644 baseline_filename,
383 svn_error = False 645 self._platform,
384 test_basename = os.path.splitext(test)[0] 646 'old')
385 for suffix in BASELINE_SUFFIXES: 647 f = open(base_file, 'wb')
386 archive_test_name = ('layout-test-results/%s-actual%s' % 648 f.write(output)
387 (test_basename, suffix)) 649 f.close()
388 logging.debug(' Archive test file name: "%s"', 650 logging.info(' Html: created old baseline file: "%s".',
389 archive_test_name) 651 base_file)
390 if not archive_test_name in zip_namelist: 652
391 logging.info(' %s file not in archive.', suffix) 653 # Get the diff between old and new baselines and save to the html directory.
392 continue 654 if baseline_filename.upper().endswith('.TXT'):
393 655 # If the user specified a custom diff command in their svn config file,
394 found = True 656 # then it'll be used when we do svn diff, which we don't want to happen
395 logging.info(' %s file found in archive.', suffix) 657 # since we want the unified diff. Using --diff-cmd=diff doesn't always
396 658 # work, since they can have another diff executable in their path that
397 # Extract new baseline from archive and save it to a temp file. 659 # gives different line endings. So we use a bogus temp directory as the
398 data = zip_file.read(archive_test_name) 660 # config directory, which gets around these problems.
399 temp_fd, temp_name = tempfile.mkstemp(suffix) 661 if sys.platform.startswith("win"):
400 f = os.fdopen(temp_fd, 'wb') 662 parent_dir = tempfile.gettempdir()
401 f.write(data) 663 else:
402 f.close() 664 parent_dir = sys.path[0] # tempdir is not secure.
403 665 bogus_dir = os.path.join(parent_dir, "temp_svn_config")
404 expected_filename = '%s-expected%s' % (test_basename, suffix) 666 logging.debug(' Html: temp config dir: "%s".', bogus_dir)
405 expected_fullpath = os.path.join( 667 if not os.path.exists(bogus_dir):
406 path_utils.ChromiumBaselinePath(platform), 668 os.mkdir(bogus_dir)
407 expected_filename) 669 delete_bogus_dir = True
408 expected_fullpath = os.path.normpath(expected_fullpath) 670 else:
409 logging.debug(' Expected file full path: "%s"', 671 delete_bogus_dir = False
410 expected_fullpath) 672
411 673 output = RunShell(["svn", "diff", "--config-dir", bogus_dir,
412 # TODO(victorw): for now, the rebaselining tool checks whether 674 baseline_fullpath])
413 # or not THIS baseline is duplicate and should be skipped. 675 if output:
414 # We could improve the tool to check all baselines in upper 676 diff_file = GetResultFileFullpath(self._options.html_directory,
415 # and lower
416 # levels and remove all duplicated baselines.
417 if self._IsDupBaseline(temp_name,
418 expected_fullpath,
419 test,
420 suffix,
421 self._platform):
422 os.remove(temp_name)
423 self._DeleteBaseline(expected_fullpath)
424 continue
425
426 # Create the new baseline directory if it doesn't already
427 # exist.
428 path_utils.MaybeMakeDirectory(
429 os.path.dirname(expected_fullpath))
430
431 shutil.move(temp_name, expected_fullpath)
432
433 if not self._SvnAdd(expected_fullpath):
434 svn_error = True
435 elif suffix != '.checksum':
436 self._CreateHtmlBaselineFiles(expected_fullpath)
437
438 if not found:
439 logging.warn(' No new baselines found in archive.')
440 else:
441 if svn_error:
442 logging.warn(' Failed to add baselines to SVN.')
443 else:
444 logging.info(' Rebaseline succeeded.')
445 self._rebaselined_tests.append(test)
446
447 test_no += 1
448
449 zip_file.close()
450 os.remove(archive_file)
451
452 return self._rebaselined_tests
453
454 def _IsDupBaseline(self, new_baseline, baseline_path, test, suffix,
455 platform):
456 """Check whether a baseline is duplicate and can fallback to same
457 baseline for another platform. For example, if a test has same
458 baseline on linux and windows, then we only store windows
459 baseline and linux baseline will fallback to the windows version.
460
461 Args:
462 expected_filename: baseline expectation file name.
463 test: test name.
464 suffix: file suffix of the expected results, including dot;
465 e.g. '.txt' or '.png'.
466 platform: baseline platform 'mac', 'win' or 'linux'.
467
468 Returns:
469 True if the baseline is unnecessary.
470 False otherwise.
471 """
472 test_filepath = os.path.join(path_utils.LayoutTestsDir(), test)
473 all_baselines = path_utils.ExpectedBaselines(test_filepath,
474 suffix,
475 platform,
476 True)
477 for (fallback_dir, fallback_file) in all_baselines:
478 if fallback_dir and fallback_file:
479 fallback_fullpath = os.path.normpath(
480 os.path.join(fallback_dir, fallback_file))
481 if fallback_fullpath.lower() != baseline_path.lower():
482 if not self._DiffBaselines(new_baseline,
483 fallback_fullpath):
484 logging.info(' Found same baseline at %s',
485 fallback_fullpath)
486 return True
487 else:
488 return False
489
490 return False
491
492 def _DiffBaselines(self, file1, file2):
493 """Check whether two baselines are different.
494
495 Args:
496 file1, file2: full paths of the baselines to compare.
497
498 Returns:
499 True if two files are different or have different extensions.
500 False otherwise.
501 """
502
503 ext1 = os.path.splitext(file1)[1].upper()
504 ext2 = os.path.splitext(file2)[1].upper()
505 if ext1 != ext2:
506 logging.warn('Files to compare have different ext. '
507 'File1: %s; File2: %s', file1, file2)
508 return True
509
510 if ext1 == '.PNG':
511 return image_diff.ImageDiff(self._platform, '').DiffFiles(file1,
512 file2)
513 else:
514 return text_diff.TestTextDiff(self._platform, '').DiffFiles(file1,
515 file2)
516
517 def _DeleteBaseline(self, filename):
518 """Remove the file from repository and delete it from disk.
519
520 Args:
521 filename: full path of the file to delete.
522 """
523
524 if not filename or not os.path.isfile(filename):
525 return
526
527 if self._repo_type == REPO_SVN:
528 parent_dir, basename = os.path.split(filename)
529 original_dir = os.getcwd()
530 os.chdir(parent_dir)
531 RunShell(['svn', 'delete', '--force', basename], False)
532 os.chdir(original_dir)
533 else:
534 os.remove(filename)
535
536 def _UpdateRebaselinedTestsInFile(self, backup):
537 """Update the rebaselined tests in test expectations file.
538
539 Args:
540 backup: if True, backup the original test expectations file.
541
542 Returns:
543 no
544 """
545
546 if self._rebaselined_tests:
547 self._test_expectations.RemovePlatformFromFile(
548 self._rebaselined_tests, self._platform, backup)
549 else:
550 logging.info('No test was rebaselined so nothing to remove.')
551
552 def _SvnAdd(self, filename):
553 """Add the file to SVN repository.
554
555 Args:
556 filename: full path of the file to add.
557
558 Returns:
559 True if the file already exists in SVN or is sucessfully added
560 to SVN.
561 False otherwise.
562 """
563
564 if not filename:
565 return False
566
567 parent_dir, basename = os.path.split(filename)
568 if self._repo_type != REPO_SVN or parent_dir == filename:
569 logging.info("No svn checkout found, skip svn add.")
570 return True
571
572 original_dir = os.getcwd()
573 os.chdir(parent_dir)
574 status_output = RunShell(['svn', 'status', basename], False)
575 os.chdir(original_dir)
576 output = status_output.upper()
577 if output.startswith('A') or output.startswith('M'):
578 logging.info(' File already added to SVN: "%s"', filename)
579 return True
580
581 if output.find('IS NOT A WORKING COPY') >= 0:
582 logging.info(' File is not a working copy, add its parent: "%s"',
583 parent_dir)
584 return self._SvnAdd(parent_dir)
585
586 os.chdir(parent_dir)
587 add_output = RunShell(['svn', 'add', basename], True)
588 os.chdir(original_dir)
589 output = add_output.upper().rstrip()
590 if output.startswith('A') and output.find(basename.upper()) >= 0:
591 logging.info(' Added new file: "%s"', filename)
592 self._SvnPropSet(filename)
593 return True
594
595 if (not status_output) and (add_output.upper().find(
596 'ALREADY UNDER VERSION CONTROL') >= 0):
597 logging.info(' File already under SVN and has no change: "%s"',
598 filename)
599 return True
600
601 logging.warn(' Failed to add file to SVN: "%s"', filename)
602 logging.warn(' Svn status output: "%s"', status_output)
603 logging.warn(' Svn add output: "%s"', add_output)
604 return False
605
606 def _SvnPropSet(self, filename):
607 """Set the baseline property
608
609 Args:
610 filename: full path of the file to add.
611
612 Returns:
613 True if the file already exists in SVN or is sucessfully added
614 to SVN.
615 False otherwise.
616 """
617 ext = os.path.splitext(filename)[1].upper()
618 if ext != '.TXT' and ext != '.PNG' and ext != '.CHECKSUM':
619 return
620
621 parent_dir, basename = os.path.split(filename)
622 original_dir = os.getcwd()
623 os.chdir(parent_dir)
624 if ext == '.PNG':
625 cmd = ['svn', 'pset', 'svn:mime-type', 'image/png', basename]
626 else:
627 cmd = ['svn', 'pset', 'svn:eol-style', 'LF', basename]
628
629 logging.debug(' Set svn prop: %s', ' '.join(cmd))
630 RunShell(cmd, False)
631 os.chdir(original_dir)
632
633 def _CreateHtmlBaselineFiles(self, baseline_fullpath):
634 """Create baseline files (old, new and diff) in html directory.
635
636 The files are used to compare the rebaselining results.
637
638 Args:
639 baseline_fullpath: full path of the expected baseline file.
640 """
641
642 if not baseline_fullpath or not os.path.exists(baseline_fullpath):
643 return
644
645 # Copy the new baseline to html directory for result comparison.
646 baseline_filename = os.path.basename(baseline_fullpath)
647 new_file = GetResultFileFullpath(self._options.html_directory,
648 baseline_filename,
649 self._platform,
650 'new')
651 shutil.copyfile(baseline_fullpath, new_file)
652 logging.info(' Html: copied new baseline file from "%s" to "%s".',
653 baseline_fullpath, new_file)
654
655 # Get the old baseline from SVN and save to the html directory.
656 output = RunShell(['svn', 'cat', '-r', 'BASE', baseline_fullpath])
657 if (not output) or (output.upper().rstrip().endswith(
658 'NO SUCH FILE OR DIRECTORY')):
659 logging.info(' No base file: "%s"', baseline_fullpath)
660 return
661 base_file = GetResultFileFullpath(self._options.html_directory,
662 baseline_filename, 677 baseline_filename,
663 self._platform, 678 self._platform,
664 'old') 679 'diff')
665 f = open(base_file, 'wb') 680 f = open(diff_file, 'wb')
666 f.write(output) 681 f.write(output)
667 f.close() 682 f.close()
668 logging.info(' Html: created old baseline file: "%s".', 683 logging.info(' Html: created baseline diff file: "%s".',
669 base_file) 684 diff_file)
670 685
671 # Get the diff between old and new baselines and save to the html dir. 686 if delete_bogus_dir:
672 if baseline_filename.upper().endswith('.TXT'): 687 shutil.rmtree(bogus_dir, True)
673 # If the user specified a custom diff command in their svn config 688 logging.debug(' Html: removed temp config dir: "%s".', bogus_dir)
674 # file, then it'll be used when we do svn diff, which we don't want
675 # to happen since we want the unified diff. Using --diff-cmd=diff
676 # doesn't always work, since they can have another diff executable
677 # in their path that gives different line endings. So we use a
678 # bogus temp directory as the config directory, which gets
679 # around these problems.
680 if sys.platform.startswith("win"):
681 parent_dir = tempfile.gettempdir()
682 else:
683 parent_dir = sys.path[0] # tempdir is not secure.
684 bogus_dir = os.path.join(parent_dir, "temp_svn_config")
685 logging.debug(' Html: temp config dir: "%s".', bogus_dir)
686 if not os.path.exists(bogus_dir):
687 os.mkdir(bogus_dir)
688 delete_bogus_dir = True
689 else:
690 delete_bogus_dir = False
691
692 output = RunShell(["svn", "diff", "--config-dir", bogus_dir,
693 baseline_fullpath])
694 if output:
695 diff_file = GetResultFileFullpath(self._options.html_directory,
696 baseline_filename,
697 self._platform,
698 'diff')
699 f = open(diff_file, 'wb')
700 f.write(output)
701 f.close()
702 logging.info(' Html: created baseline diff file: "%s".',
703 diff_file)
704
705 if delete_bogus_dir:
706 shutil.rmtree(bogus_dir, True)
707 logging.debug(' Html: removed temp config dir: "%s".',
708 bogus_dir)
709
710 689
711 class HtmlGenerator(object): 690 class HtmlGenerator(object):
712 """Class to generate rebaselining result comparison html.""" 691 """Class to generate rebaselining result comparison html."""
713 692
714 HTML_REBASELINE = ('<html>' 693 HTML_REBASELINE = ('<html>'
715 '<head>' 694 '<head>'
716 '<style>' 695 '<style>'
717 'body {font-family: sans-serif;}' 696 'body {font-family: sans-serif;}'
718 '.mainTable {background: #666666;}' 697 '.mainTable {background: #666666;}'
719 '.mainTable td , .mainTable th {background: white;}' 698 '.mainTable td , .mainTable th {background: white;}'
720 '.detail {margin-left: 10px; margin-top: 3px;}' 699 '.detail {margin-left: 10px; margin-top: 3px;}'
721 '</style>' 700 '</style>'
722 '<title>Rebaselining Result Comparison (%(time)s)' 701 '<title>Rebaselining Result Comparison (%(time)s)</title>'
723 '</title>' 702 '</head>'
724 '</head>' 703 '<body>'
725 '<body>' 704 '<h2>Rebaselining Result Comparison (%(time)s)</h2>'
726 '<h2>Rebaselining Result Comparison (%(time)s)</h2>' 705 '%(body)s'
727 '%(body)s' 706 '</body>'
728 '</body>' 707 '</html>')
729 '</html>') 708 HTML_NO_REBASELINING_TESTS = '<p>No tests found that need rebaselining.</p>'
730 HTML_NO_REBASELINING_TESTS = ( 709 HTML_TABLE_TEST = ('<table class="mainTable" cellspacing=1 cellpadding=5>'
731 '<p>No tests found that need rebaselining.</p>') 710 '%s</table><br>')
732 HTML_TABLE_TEST = ('<table class="mainTable" cellspacing=1 cellpadding=5>' 711 HTML_TR_TEST = ('<tr>'
733 '%s</table><br>') 712 '<th style="background-color: #CDECDE; border-bottom: '
734 HTML_TR_TEST = ('<tr>' 713 '1px solid black; font-size: 18pt; font-weight: bold" '
735 '<th style="background-color: #CDECDE; border-bottom: ' 714 'colspan="5">'
736 '1px solid black; font-size: 18pt; font-weight: bold" ' 715 '<a href="%s">%s</a>'
737 'colspan="5">' 716 '</th>'
738 '<a href="%s">%s</a>' 717 '</tr>')
739 '</th>' 718 HTML_TEST_DETAIL = ('<div class="detail">'
740 '</tr>') 719 '<tr>'
741 HTML_TEST_DETAIL = ('<div class="detail">' 720 '<th width="100">Baseline</th>'
742 '<tr>' 721 '<th width="100">Platform</th>'
743 '<th width="100">Baseline</th>' 722 '<th width="200">Old</th>'
744 '<th width="100">Platform</th>' 723 '<th width="200">New</th>'
745 '<th width="200">Old</th>' 724 '<th width="150">Difference</th>'
746 '<th width="200">New</th>' 725 '</tr>'
747 '<th width="150">Difference</th>' 726 '%s'
748 '</tr>' 727 '</div>')
749 '%s' 728 HTML_TD_NOLINK = '<td align=center><a>%s</a></td>'
750 '</div>') 729 HTML_TD_LINK = '<td align=center><a href="%(uri)s">%(name)s</a></td>'
751 HTML_TD_NOLINK = '<td align=center><a>%s</a></td>' 730 HTML_TD_LINK_IMG = ('<td><a href="%(uri)s">'
752 HTML_TD_LINK = '<td align=center><a href="%(uri)s">%(name)s</a></td>' 731 '<img style="width: 200" src="%(uri)s" /></a></td>')
753 HTML_TD_LINK_IMG = ('<td><a href="%(uri)s">' 732 HTML_TR = '<tr>%s</tr>'
754 '<img style="width: 200" src="%(uri)s" /></a></td>') 733
755 HTML_TR = '<tr>%s</tr>' 734 def __init__(self, options, platforms, rebaselining_tests):
756 735 self._html_directory = options.html_directory
757 def __init__(self, options, platforms, rebaselining_tests): 736 self._platforms = platforms
758 self._html_directory = options.html_directory 737 self._rebaselining_tests = rebaselining_tests
759 self._platforms = platforms 738 self._html_file = os.path.join(options.html_directory, 'rebaseline.html')
760 self._rebaselining_tests = rebaselining_tests 739
761 self._html_file = os.path.join(options.html_directory, 740 def GenerateHtml(self):
762 'rebaseline.html') 741 """Generate html file for rebaselining result comparison."""
763 742
764 def GenerateHtml(self): 743 logging.info('Generating html file')
765 """Generate html file for rebaselining result comparison.""" 744
766 745 html_body = ''
767 logging.info('Generating html file') 746 if not self._rebaselining_tests:
768 747 html_body += self.HTML_NO_REBASELINING_TESTS
769 html_body = '' 748 else:
770 if not self._rebaselining_tests: 749 tests = list(self._rebaselining_tests)
771 html_body += self.HTML_NO_REBASELINING_TESTS 750 tests.sort()
772 else: 751
773 tests = list(self._rebaselining_tests) 752 test_no = 1
774 tests.sort() 753 for test in tests:
775 754 logging.info('Test %d: %s', test_no, test)
776 test_no = 1 755 html_body += self._GenerateHtmlForOneTest(test)
777 for test in tests: 756
778 logging.info('Test %d: %s', test_no, test) 757 html = self.HTML_REBASELINE % ({'time': time.asctime(), 'body': html_body})
779 html_body += self._GenerateHtmlForOneTest(test) 758 logging.debug(html)
780 759
781 html = self.HTML_REBASELINE % ({'time': time.asctime(), 760 f = open(self._html_file, 'w')
782 'body': html_body}) 761 f.write(html)
783 logging.debug(html) 762 f.close()
784 763
785 f = open(self._html_file, 'w') 764 logging.info('Baseline comparison html generated at "%s"',
786 f.write(html) 765 self._html_file)
787 f.close() 766
788 767 def ShowHtml(self):
789 logging.info('Baseline comparison html generated at "%s"', 768 """Launch the rebaselining html in brwoser."""
790 self._html_file) 769
791 770 logging.info('Launching html: "%s"', self._html_file)
792 def ShowHtml(self): 771
793 """Launch the rebaselining html in brwoser.""" 772 html_uri = path_utils.FilenameToUri(self._html_file)
794 773 webbrowser.open(html_uri, 1)
795 logging.info('Launching html: "%s"', self._html_file) 774
796 775 logging.info('Html launched.')
797 html_uri = path_utils.FilenameToUri(self._html_file) 776
798 webbrowser.open(html_uri, 1) 777 def _GenerateBaselineLinks(self, test_basename, suffix, platform):
799 778 """Generate links for baseline results (old, new and diff).
800 logging.info('Html launched.') 779
801 780 Args:
802 def _GenerateBaselineLinks(self, test_basename, suffix, platform): 781 test_basename: base filename of the test
803 """Generate links for baseline results (old, new and diff). 782 suffix: baseline file suffixes: '.txt', '.png'
804 783 platform: win, linux or mac
805 Args: 784
806 test_basename: base filename of the test 785 Returns:
807 suffix: baseline file suffixes: '.txt', '.png' 786 html links for showing baseline results (old, new and diff)
808 platform: win, linux or mac 787 """
809 788
810 Returns: 789 baseline_filename = '%s-expected%s' % (test_basename, suffix)
811 html links for showing baseline results (old, new and diff) 790 logging.debug(' baseline filename: "%s"', baseline_filename)
812 """ 791
813 792 new_file = GetResultFileFullpath(self._html_directory,
814 baseline_filename = '%s-expected%s' % (test_basename, suffix) 793 baseline_filename,
815 logging.debug(' baseline filename: "%s"', baseline_filename) 794 platform,
816 795 'new')
817 new_file = GetResultFileFullpath(self._html_directory, 796 logging.info(' New baseline file: "%s"', new_file)
818 baseline_filename, 797 if not os.path.exists(new_file):
819 platform, 798 logging.info(' No new baseline file: "%s"', new_file)
820 'new') 799 return ''
821 logging.info(' New baseline file: "%s"', new_file) 800
822 if not os.path.exists(new_file): 801 old_file = GetResultFileFullpath(self._html_directory,
823 logging.info(' No new baseline file: "%s"', new_file) 802 baseline_filename,
824 return '' 803 platform,
825 804 'old')
826 old_file = GetResultFileFullpath(self._html_directory, 805 logging.info(' Old baseline file: "%s"', old_file)
827 baseline_filename, 806 if suffix == '.png':
828 platform, 807 html_td_link = self.HTML_TD_LINK_IMG
829 'old') 808 else:
830 logging.info(' Old baseline file: "%s"', old_file) 809 html_td_link = self.HTML_TD_LINK
831 if suffix == '.png': 810
832 html_td_link = self.HTML_TD_LINK_IMG 811 links = ''
833 else: 812 if os.path.exists(old_file):
834 html_td_link = self.HTML_TD_LINK 813 links += html_td_link % {'uri': path_utils.FilenameToUri(old_file),
835 814 'name': baseline_filename}
836 links = '' 815 else:
837 if os.path.exists(old_file): 816 logging.info(' No old baseline file: "%s"', old_file)
838 links += html_td_link % {'uri': path_utils.FilenameToUri(old_file), 817 links += self.HTML_TD_NOLINK % ''
839 'name': baseline_filename} 818
840 else: 819 links += html_td_link % {'uri': path_utils.FilenameToUri(new_file),
841 logging.info(' No old baseline file: "%s"', old_file) 820 'name': baseline_filename}
842 links += self.HTML_TD_NOLINK % '' 821
843 822 diff_file = GetResultFileFullpath(self._html_directory,
844 links += html_td_link % {'uri': path_utils.FilenameToUri(new_file), 823 baseline_filename,
845 'name': baseline_filename} 824 platform,
846 825 'diff')
847 diff_file = GetResultFileFullpath(self._html_directory, 826 logging.info(' Baseline diff file: "%s"', diff_file)
848 baseline_filename, 827 if os.path.exists(diff_file):
849 platform, 828 links += html_td_link % {'uri': path_utils.FilenameToUri(diff_file),
850 'diff') 829 'name': 'Diff'}
851 logging.info(' Baseline diff file: "%s"', diff_file) 830 else:
852 if os.path.exists(diff_file): 831 logging.info(' No baseline diff file: "%s"', diff_file)
853 links += html_td_link % {'uri': path_utils.FilenameToUri( 832 links += self.HTML_TD_NOLINK % ''
854 diff_file), 'name': 'Diff'} 833
855 else: 834 return links
856 logging.info(' No baseline diff file: "%s"', diff_file) 835
857 links += self.HTML_TD_NOLINK % '' 836 def _GenerateHtmlForOneTest(self, test):
858 837 """Generate html for one rebaselining test.
859 return links 838
860 839 Args:
861 def _GenerateHtmlForOneTest(self, test): 840 test: layout test name
862 """Generate html for one rebaselining test. 841
863 842 Returns:
864 Args: 843 html that compares baseline results for the test.
865 test: layout test name 844 """
866 845
867 Returns: 846 test_basename = os.path.basename(os.path.splitext(test)[0])
868 html that compares baseline results for the test. 847 logging.info(' basename: "%s"', test_basename)
869 """ 848 rows = []
870 849 for suffix in BASELINE_SUFFIXES:
871 test_basename = os.path.basename(os.path.splitext(test)[0]) 850 if suffix == '.checksum':
872 logging.info(' basename: "%s"', test_basename) 851 continue
873 rows = [] 852
874 for suffix in BASELINE_SUFFIXES: 853 logging.info(' Checking %s files', suffix)
875 if suffix == '.checksum': 854 for platform in self._platforms:
876 continue 855 links = self._GenerateBaselineLinks(test_basename, suffix, platform)
877 856 if links:
878 logging.info(' Checking %s files', suffix) 857 row = self.HTML_TD_NOLINK % self._GetBaselineResultType(suffix)
879 for platform in self._platforms: 858 row += self.HTML_TD_NOLINK % platform
880 links = self._GenerateBaselineLinks(test_basename, suffix, 859 row += links
881 platform) 860 logging.debug(' html row: %s', row)
882 if links: 861
883 row = self.HTML_TD_NOLINK % self._GetBaselineResultType( 862 rows.append(self.HTML_TR % row)
884 suffix) 863
885 row += self.HTML_TD_NOLINK % platform 864 if rows:
886 row += links 865 test_path = os.path.join(path_utils.LayoutTestsDir(), test)
887 logging.debug(' html row: %s', row) 866 html = self.HTML_TR_TEST % (path_utils.FilenameToUri(test_path), test)
888 867 html += self.HTML_TEST_DETAIL % ' '.join(rows)
889 rows.append(self.HTML_TR % row) 868
890 869 logging.debug(' html for test: %s', html)
891 if rows: 870 return self.HTML_TABLE_TEST % html
892 test_path = os.path.join(path_utils.LayoutTestsDir(), test) 871
893 html = self.HTML_TR_TEST % (path_utils.FilenameToUri(test_path), 872 return ''
894 test) 873
895 html += self.HTML_TEST_DETAIL % ' '.join(rows) 874 def _GetBaselineResultType(self, suffix):
896 875 """Name of the baseline result type."""
897 logging.debug(' html for test: %s', html) 876
898 return self.HTML_TABLE_TEST % html 877 if suffix == '.png':
899 878 return 'Pixel'
900 return '' 879 elif suffix == '.txt':
901 880 return 'Render Tree'
902 def _GetBaselineResultType(self, suffix): 881 else:
903 """Name of the baseline result type.""" 882 return 'Other'
904
905 if suffix == '.png':
906 return 'Pixel'
907 elif suffix == '.txt':
908 return 'Render Tree'
909 else:
910 return 'Other'
911 883
912 884
913 def main(): 885 def main():
914 """Main function to produce new baselines.""" 886 """Main function to produce new baselines."""
915 887
916 option_parser = optparse.OptionParser() 888 option_parser = optparse.OptionParser()
917 option_parser.add_option('-v', '--verbose', 889 option_parser.add_option('-v', '--verbose',
918 action='store_true', 890 action='store_true',
919 default=False, 891 default=False,
920 help='include debug-level logging.') 892 help='include debug-level logging.')
921 893
922 option_parser.add_option('-p', '--platforms', 894 option_parser.add_option('-p', '--platforms',
923 default='mac,win,win-xp,win-vista,linux', 895 default='mac,win,win-xp,win-vista,linux',
924 help=('Comma delimited list of platforms ' 896 help=('Comma delimited list of platforms that need '
925 'that need rebaselining.')) 897 'rebaselining.'))
926 898
927 option_parser.add_option('-u', '--archive_url', 899 option_parser.add_option('-u', '--archive_url',
928 default=('http://build.chromium.org/buildbot/' 900 default=('http://build.chromium.org/buildbot/'
929 'layout_test_results'), 901 'layout_test_results'),
930 help=('Url to find the layout test result archive' 902 help=('Url to find the layout test result archive '
931 ' file.')) 903 'file.'))
932 904
933 option_parser.add_option('-w', '--webkit_canary', 905 option_parser.add_option('-w', '--webkit_canary',
934 action='store_true', 906 action='store_true',
935 default=False, 907 default=False,
936 help=('If True, pull baselines from webkit.org ' 908 help=('If True, pull baselines from webkit.org '
937 'canary bot.')) 909 'canary bot.'))
938 910
939 option_parser.add_option('-b', '--backup', 911 option_parser.add_option('-b', '--backup',
940 action='store_true', 912 action='store_true',
941 default=False, 913 default=False,
942 help=('Whether or not to backup the original test' 914 help=('Whether or not to backup the original test '
943 ' expectations file after rebaseline.')) 915 'expectations file after rebaseline.'))
944 916
945 option_parser.add_option('-d', '--html_directory', 917 option_parser.add_option('-d', '--html_directory',
946 default='', 918 default='',
947 help=('The directory that stores the results for' 919 help=('The directory that stores the results for '
948 ' rebaselining comparison.')) 920 'rebaselining comparison.'))
949 921
950 options = option_parser.parse_args()[0] 922 options = option_parser.parse_args()[0]
951 923
952 # Set up our logging format. 924 # Set up our logging format.
953 log_level = logging.INFO 925 log_level = logging.INFO
954 if options.verbose: 926 if options.verbose:
955 log_level = logging.DEBUG 927 log_level = logging.DEBUG
956 logging.basicConfig(level=log_level, 928 logging.basicConfig(level=log_level,
957 format=('%(asctime)s %(filename)s:%(lineno)-3d ' 929 format=('%(asctime)s %(filename)s:%(lineno)-3d '
958 '%(levelname)s %(message)s'), 930 '%(levelname)s %(message)s'),
959 datefmt='%y%m%d %H:%M:%S') 931 datefmt='%y%m%d %H:%M:%S')
960 932
961 # Verify 'platforms' option is valid 933 # Verify 'platforms' option is valid
962 if not options.platforms: 934 if not options.platforms:
963 logging.error('Invalid "platforms" option. --platforms must be ' 935 logging.error('Invalid "platforms" option. --platforms must be specified '
964 'specified in order to rebaseline.') 936 'in order to rebaseline.')
965 sys.exit(1) 937 sys.exit(1)
966 platforms = [p.strip().lower() for p in options.platforms.split(',')] 938 platforms = [p.strip().lower() for p in options.platforms.split(',')]
967 for platform in platforms: 939 for platform in platforms:
968 if not platform in REBASELINE_PLATFORM_ORDER: 940 if not platform in REBASELINE_PLATFORM_ORDER:
969 logging.error('Invalid platform: "%s"' % (platform)) 941 logging.error('Invalid platform: "%s"' % (platform))
970 sys.exit(1) 942 sys.exit(1)
971 943
972 # Adjust the platform order so rebaseline tool is running at the order of 944 # Adjust the platform order so rebaseline tool is running at the order of
973 # 'mac', 'win' and 'linux'. This is in same order with layout test baseline 945 # 'mac', 'win' and 'linux'. This is in same order with layout test baseline
974 # search paths. It simplifies how the rebaseline tool detects duplicate 946 # search paths. It simplifies how the rebaseline tool detects duplicate
975 # baselines. Check _IsDupBaseline method for details. 947 # baselines. Check _IsDupBaseline method for details.
976 rebaseline_platforms = [] 948 rebaseline_platforms = []
977 for platform in REBASELINE_PLATFORM_ORDER: 949 for platform in REBASELINE_PLATFORM_ORDER:
978 if platform in platforms: 950 if platform in platforms:
979 rebaseline_platforms.append(platform) 951 rebaseline_platforms.append(platform)
980 952
981 options.html_directory = SetupHtmlDirectory(options.html_directory) 953 options.html_directory = SetupHtmlDirectory(options.html_directory)
982 954
983 rebaselining_tests = set() 955 rebaselining_tests = set()
984 backup = options.backup 956 backup = options.backup
985 for platform in rebaseline_platforms: 957 for platform in rebaseline_platforms:
986 rebaseliner = Rebaseliner(platform, options) 958 rebaseliner = Rebaseliner(platform, options)
987
988 logging.info('')
989 LogDashedString('Rebaseline started', platform)
990 if rebaseliner.Run(backup):
991 # Only need to backup one original copy of test expectation file.
992 backup = False
993 LogDashedString('Rebaseline done', platform)
994 else:
995 LogDashedString('Rebaseline failed', platform, logging.ERROR)
996
997 rebaselining_tests |= set(rebaseliner.GetRebaseliningTests())
998 959
999 logging.info('') 960 logging.info('')
1000 LogDashedString('Rebaselining result comparison started', None) 961 LogDashedString('Rebaseline started', platform)
1001 html_generator = HtmlGenerator(options, 962 if rebaseliner.Run(backup):
1002 rebaseline_platforms, 963 # Only need to backup one original copy of test expectation file.
1003 rebaselining_tests) 964 backup = False
1004 html_generator.GenerateHtml() 965 LogDashedString('Rebaseline done', platform)
1005 html_generator.ShowHtml() 966 else:
1006 LogDashedString('Rebaselining result comparison done', None) 967 LogDashedString('Rebaseline failed', platform, logging.ERROR)
1007 968
1008 sys.exit(0) 969 rebaselining_tests |= set(rebaseliner.GetRebaseliningTests())
970
971 logging.info('')
972 LogDashedString('Rebaselining result comparison started', None)
973 html_generator = HtmlGenerator(options,
974 rebaseline_platforms,
975 rebaselining_tests)
976 html_generator.GenerateHtml()
977 html_generator.ShowHtml()
978 LogDashedString('Rebaselining result comparison done', None)
979
980 sys.exit(0)
1009 981
1010 if '__main__' == __name__: 982 if '__main__' == __name__:
1011 main() 983 main()
OLDNEW

Powered by Google App Engine
This is Rietveld 408576698