| OLD | NEW |
|---|
| (Empty) |
| 1 #!usr/bin/env python | |
| 2 # Copyright (c) 2006-2009 The Chromium Authors. All rights reserved. | |
| 3 # Use of this source code is governed by a BSD-style license that can be | |
| 4 # found in the LICENSE file. | |
| 5 | |
| 6 """Rebaselining tool that automatically produces baselines for all platforms. | |
| 7 | |
| 8 The script does the following for each platform specified: | |
| 9 1. Compile a list of tests that need rebaselining. | |
| 10 2. Download test result archive from buildbot for the platform. | |
| 11 3. Extract baselines from the archive file for all identified files. | |
| 12 4. Add new baselines to SVN repository. | |
| 13 5. For each test that has been rebaselined, remove this platform option from | |
| 14 the test in test_expectation.txt. If no other platforms remain after | |
| 15 removal, delete the rebaselined test from the file. | |
| 16 | |
| 17 At the end, the script generates a html that compares old and new baselines. | |
| 18 """ | |
| 19 | |
| 20 import logging | |
| 21 import optparse | |
| 22 import os | |
| 23 import re | |
| 24 import shutil | |
| 25 import subprocess | |
| 26 import sys | |
| 27 import tempfile | |
| 28 import time | |
| 29 import urllib | |
| 30 import webbrowser | |
| 31 import zipfile | |
| 32 | |
| 33 from layout_package import path_utils | |
| 34 from layout_package import test_expectations | |
| 35 from test_types import image_diff | |
| 36 from test_types import text_diff | |
| 37 | |
| 38 # Repository type constants. | |
| 39 REPO_SVN, REPO_UNKNOWN = range(2) | |
| 40 | |
| 41 BASELINE_SUFFIXES = ['.txt', '.png', '.checksum'] | |
| 42 REBASELINE_PLATFORM_ORDER = ['mac', 'win', 'win-xp', 'win-vista', 'linux'] | |
| 43 ARCHIVE_DIR_NAME_DICT = {'win': 'webkit-rel', | |
| 44 'win-vista': 'webkit-dbg-vista', | |
| 45 'win-xp': 'webkit-rel', | |
| 46 'mac': 'webkit-rel-mac5', | |
| 47 'linux': 'webkit-rel-linux', | |
| 48 'win-canary': 'webkit-rel-webkit-org', | |
| 49 'win-vista-canary': 'webkit-dbg-vista', | |
| 50 'win-xp-canary': 'webkit-rel-webkit-org', | |
| 51 'mac-canary': 'webkit-rel-mac-webkit-org', | |
| 52 'linux-canary': 'webkit-rel-linux-webkit-org'} | |
| 53 | |
| 54 | |
| 55 def RunShellWithReturnCode(command, print_output=False): | |
| 56 """Executes a command and returns the output and process return code. | |
| 57 | |
| 58 Args: | |
| 59 command: program and arguments. | |
| 60 print_output: if true, print the command results to standard output. | |
| 61 | |
| 62 Returns: | |
| 63 command output, return code | |
| 64 """ | |
| 65 | |
| 66 # Use a shell for subcommands on Windows to get a PATH search. | |
| 67 use_shell = sys.platform.startswith('win') | |
| 68 p = subprocess.Popen(command, stdout=subprocess.PIPE, | |
| 69 stderr=subprocess.STDOUT, shell=use_shell) | |
| 70 if print_output: | |
| 71 output_array = [] | |
| 72 while True: | |
| 73 line = p.stdout.readline() | |
| 74 if not line: | |
| 75 break | |
| 76 if print_output: | |
| 77 print line.strip('\n') | |
| 78 output_array.append(line) | |
| 79 output = ''.join(output_array) | |
| 80 else: | |
| 81 output = p.stdout.read() | |
| 82 p.wait() | |
| 83 p.stdout.close() | |
| 84 | |
| 85 return output, p.returncode | |
| 86 | |
| 87 | |
| 88 def RunShell(command, print_output=False): | |
| 89 """Executes a command and returns the output. | |
| 90 | |
| 91 Args: | |
| 92 command: program and arguments. | |
| 93 print_output: if true, print the command results to standard output. | |
| 94 | |
| 95 Returns: | |
| 96 command output | |
| 97 """ | |
| 98 | |
| 99 output, return_code = RunShellWithReturnCode(command, print_output) | |
| 100 return output | |
| 101 | |
| 102 | |
| 103 def LogDashedString(text, platform, logging_level=logging.INFO): | |
| 104 """Log text message with dashes on both sides.""" | |
| 105 | |
| 106 msg = text | |
| 107 if platform: | |
| 108 msg += ': ' + platform | |
| 109 if len(msg) < 78: | |
| 110 dashes = '-' * ((78 - len(msg)) / 2) | |
| 111 msg = '%s %s %s' % (dashes, msg, dashes) | |
| 112 | |
| 113 if logging_level == logging.ERROR: | |
| 114 logging.error(msg) | |
| 115 elif logging_level == logging.WARNING: | |
| 116 logging.warn(msg) | |
| 117 else: | |
| 118 logging.info(msg) | |
| 119 | |
| 120 | |
| 121 def SetupHtmlDirectory(html_directory): | |
| 122 """Setup the directory to store html results. | |
| 123 | |
| 124 All html related files are stored in the "rebaseline_html" subdirectory. | |
| 125 | |
| 126 Args: | |
| 127 html_directory: parent directory that stores the rebaselining results. | |
| 128 If None, a temp directory is created. | |
| 129 | |
| 130 Returns: | |
| 131 the directory that stores the html related rebaselining results. | |
| 132 """ | |
| 133 | |
| 134 if not html_directory: | |
| 135 html_directory = tempfile.mkdtemp() | |
| 136 elif not os.path.exists(html_directory): | |
| 137 os.mkdir(html_directory) | |
| 138 | |
| 139 html_directory = os.path.join(html_directory, 'rebaseline_html') | |
| 140 logging.info('Html directory: "%s"', html_directory) | |
| 141 | |
| 142 if os.path.exists(html_directory): | |
| 143 shutil.rmtree(html_directory, True) | |
| 144 logging.info('Deleted file at html directory: "%s"', html_directory) | |
| 145 | |
| 146 if not os.path.exists(html_directory): | |
| 147 os.mkdir(html_directory) | |
| 148 return html_directory | |
| 149 | |
| 150 | |
| 151 def GetResultFileFullpath(html_directory, baseline_filename, platform, | |
| 152 result_type): | |
| 153 """Get full path of the baseline result file. | |
| 154 | |
| 155 Args: | |
| 156 html_directory: directory that stores the html related files. | |
| 157 baseline_filename: name of the baseline file. | |
| 158 platform: win, linux or mac | |
| 159 result_type: type of the baseline result: '.txt', '.png'. | |
| 160 | |
| 161 Returns: | |
| 162 Full path of the baseline file for rebaselining result comparison. | |
| 163 """ | |
| 164 | |
| 165 base, ext = os.path.splitext(baseline_filename) | |
| 166 result_filename = '%s-%s-%s%s' % (base, platform, result_type, ext) | |
| 167 fullpath = os.path.join(html_directory, result_filename) | |
| 168 logging.debug(' Result file full path: "%s".', fullpath) | |
| 169 return fullpath | |
| 170 | |
| 171 | |
| 172 class Rebaseliner(object): | |
| 173 """Class to produce new baselines for a given platform.""" | |
| 174 | |
| 175 REVISION_REGEX = r'<a href=\"(\d+)/\">' | |
| 176 | |
| 177 def __init__(self, platform, options): | |
| 178 self._file_dir = path_utils.GetAbsolutePath( | |
| 179 os.path.dirname(sys.argv[0])) | |
| 180 self._platform = platform | |
| 181 self._options = options | |
| 182 self._rebaselining_tests = [] | |
| 183 self._rebaselined_tests = [] | |
| 184 | |
| 185 # Create tests and expectations helper which is used to: | |
| 186 # -. compile list of tests that need rebaselining. | |
| 187 # -. update the tests in test_expectations file after rebaseline | |
| 188 # is done. | |
| 189 self._test_expectations = \ | |
| 190 test_expectations.TestExpectations(None, | |
| 191 self._file_dir, | |
| 192 platform, | |
| 193 False, | |
| 194 False) | |
| 195 | |
| 196 self._repo_type = self._GetRepoType() | |
| 197 | |
| 198 def Run(self, backup): | |
| 199 """Run rebaseline process.""" | |
| 200 | |
| 201 LogDashedString('Compiling rebaselining tests', self._platform) | |
| 202 if not self._CompileRebaseliningTests(): | |
| 203 return True | |
| 204 | |
| 205 LogDashedString('Downloading archive', self._platform) | |
| 206 archive_file = self._DownloadBuildBotArchive() | |
| 207 logging.info('') | |
| 208 if not archive_file: | |
| 209 logging.error('No archive found.') | |
| 210 return False | |
| 211 | |
| 212 LogDashedString('Extracting and adding new baselines', self._platform) | |
| 213 if not self._ExtractAndAddNewBaselines(archive_file): | |
| 214 return False | |
| 215 | |
| 216 LogDashedString('Updating rebaselined tests in file', self._platform) | |
| 217 self._UpdateRebaselinedTestsInFile(backup) | |
| 218 logging.info('') | |
| 219 | |
| 220 if len(self._rebaselining_tests) != len(self._rebaselined_tests): | |
| 221 logging.warning('NOT ALL TESTS THAT NEED REBASELINING HAVE BEEN ' | |
| 222 'REBASELINED.') | |
| 223 logging.warning(' Total tests needing rebaselining: %d', | |
| 224 len(self._rebaselining_tests)) | |
| 225 logging.warning(' Total tests rebaselined: %d', | |
| 226 len(self._rebaselined_tests)) | |
| 227 return False | |
| 228 | |
| 229 logging.warning('All tests needing rebaselining were successfully ' | |
| 230 'rebaselined.') | |
| 231 | |
| 232 return True | |
| 233 | |
| 234 def GetRebaseliningTests(self): | |
| 235 return self._rebaselining_tests | |
| 236 | |
| 237 def _GetRepoType(self): | |
| 238 """Get the repository type that client is using.""" | |
| 239 | |
| 240 output, return_code = RunShellWithReturnCode(['svn', 'info'], False) | |
| 241 if return_code == 0: | |
| 242 return REPO_SVN | |
| 243 | |
| 244 return REPO_UNKNOWN | |
| 245 | |
| 246 def _CompileRebaseliningTests(self): | |
| 247 """Compile list of tests that need rebaselining for the platform. | |
| 248 | |
| 249 Returns: | |
| 250 List of tests that need rebaselining or | |
| 251 None if there is no such test. | |
| 252 """ | |
| 253 | |
| 254 self._rebaselining_tests = \ | |
| 255 self._test_expectations.GetRebaseliningFailures() | |
| 256 if not self._rebaselining_tests: | |
| 257 logging.warn('No tests found that need rebaselining.') | |
| 258 return None | |
| 259 | |
| 260 logging.info('Total number of tests needing rebaselining ' | |
| 261 'for "%s": "%d"', self._platform, | |
| 262 len(self._rebaselining_tests)) | |
| 263 | |
| 264 test_no = 1 | |
| 265 for test in self._rebaselining_tests: | |
| 266 logging.info(' %d: %s', test_no, test) | |
| 267 test_no += 1 | |
| 268 | |
| 269 return self._rebaselining_tests | |
| 270 | |
| 271 def _GetLatestRevision(self, url): | |
| 272 """Get the latest layout test revision number from buildbot. | |
| 273 | |
| 274 Args: | |
| 275 url: Url to retrieve layout test revision numbers. | |
| 276 | |
| 277 Returns: | |
| 278 latest revision or | |
| 279 None on failure. | |
| 280 """ | |
| 281 | |
| 282 logging.debug('Url to retrieve revision: "%s"', url) | |
| 283 | |
| 284 f = urllib.urlopen(url) | |
| 285 content = f.read() | |
| 286 f.close() | |
| 287 | |
| 288 revisions = re.findall(self.REVISION_REGEX, content) | |
| 289 if not revisions: | |
| 290 logging.error('Failed to find revision, content: "%s"', content) | |
| 291 return None | |
| 292 | |
| 293 revisions.sort(key=int) | |
| 294 logging.info('Latest revision: "%s"', revisions[len(revisions) - 1]) | |
| 295 return revisions[len(revisions) - 1] | |
| 296 | |
| 297 def _GetArchiveDirName(self, platform, webkit_canary): | |
| 298 """Get name of the layout test archive directory. | |
| 299 | |
| 300 Returns: | |
| 301 Directory name or | |
| 302 None on failure | |
| 303 """ | |
| 304 | |
| 305 if webkit_canary: | |
| 306 platform += '-canary' | |
| 307 | |
| 308 if platform in ARCHIVE_DIR_NAME_DICT: | |
| 309 return ARCHIVE_DIR_NAME_DICT[platform] | |
| 310 else: | |
| 311 logging.error('Cannot find platform key %s in archive ' | |
| 312 'directory name dictionary', platform) | |
| 313 return None | |
| 314 | |
| 315 def _GetArchiveUrl(self): | |
| 316 """Generate the url to download latest layout test archive. | |
| 317 | |
| 318 Returns: | |
| 319 Url to download archive or | |
| 320 None on failure | |
| 321 """ | |
| 322 | |
| 323 dir_name = self._GetArchiveDirName(self._platform, | |
| 324 self._options.webkit_canary) | |
| 325 if not dir_name: | |
| 326 return None | |
| 327 | |
| 328 logging.debug('Buildbot platform dir name: "%s"', dir_name) | |
| 329 | |
| 330 url_base = '%s/%s/' % (self._options.archive_url, dir_name) | |
| 331 latest_revision = self._GetLatestRevision(url_base) | |
| 332 if latest_revision is None or latest_revision <= 0: | |
| 333 return None | |
| 334 | |
| 335 archive_url = ('%s%s/layout-test-results.zip' % (url_base, | |
| 336 latest_revision)) | |
| 337 logging.info('Archive url: "%s"', archive_url) | |
| 338 return archive_url | |
| 339 | |
| 340 def _DownloadBuildBotArchive(self): | |
| 341 """Download layout test archive file from buildbot. | |
| 342 | |
| 343 Returns: | |
| 344 True if download succeeded or | |
| 345 False otherwise. | |
| 346 """ | |
| 347 | |
| 348 url = self._GetArchiveUrl() | |
| 349 if url is None: | |
| 350 return None | |
| 351 | |
| 352 fn = urllib.urlretrieve(url)[0] | |
| 353 logging.info('Archive downloaded and saved to file: "%s"', fn) | |
| 354 return fn | |
| 355 | |
| 356 def _ExtractAndAddNewBaselines(self, archive_file): | |
| 357 """Extract new baselines from archive and add them to SVN repository. | |
| 358 | |
| 359 Args: | |
| 360 archive_file: full path to the archive file. | |
| 361 | |
| 362 Returns: | |
| 363 List of tests that have been rebaselined or | |
| 364 None on failure. | |
| 365 """ | |
| 366 | |
| 367 zip_file = zipfile.ZipFile(archive_file, 'r') | |
| 368 zip_namelist = zip_file.namelist() | |
| 369 | |
| 370 logging.debug('zip file namelist:') | |
| 371 for name in zip_namelist: | |
| 372 logging.debug(' ' + name) | |
| 373 | |
| 374 platform = path_utils.PlatformName(self._platform) | |
| 375 logging.debug('Platform dir: "%s"', platform) | |
| 376 | |
| 377 test_no = 1 | |
| 378 self._rebaselined_tests = [] | |
| 379 for test in self._rebaselining_tests: | |
| 380 logging.info('Test %d: %s', test_no, test) | |
| 381 | |
| 382 found = False | |
| 383 svn_error = False | |
| 384 test_basename = os.path.splitext(test)[0] | |
| 385 for suffix in BASELINE_SUFFIXES: | |
| 386 archive_test_name = ('layout-test-results/%s-actual%s' % | |
| 387 (test_basename, suffix)) | |
| 388 logging.debug(' Archive test file name: "%s"', | |
| 389 archive_test_name) | |
| 390 if not archive_test_name in zip_namelist: | |
| 391 logging.info(' %s file not in archive.', suffix) | |
| 392 continue | |
| 393 | |
| 394 found = True | |
| 395 logging.info(' %s file found in archive.', suffix) | |
| 396 | |
| 397 # Extract new baseline from archive and save it to a temp file. | |
| 398 data = zip_file.read(archive_test_name) | |
| 399 temp_fd, temp_name = tempfile.mkstemp(suffix) | |
| 400 f = os.fdopen(temp_fd, 'wb') | |
| 401 f.write(data) | |
| 402 f.close() | |
| 403 | |
| 404 expected_filename = '%s-expected%s' % (test_basename, suffix) | |
| 405 expected_fullpath = os.path.join( | |
| 406 path_utils.ChromiumBaselinePath(platform), | |
| 407 expected_filename) | |
| 408 expected_fullpath = os.path.normpath(expected_fullpath) | |
| 409 logging.debug(' Expected file full path: "%s"', | |
| 410 expected_fullpath) | |
| 411 | |
| 412 # TODO(victorw): for now, the rebaselining tool checks whether | |
| 413 # or not THIS baseline is duplicate and should be skipped. | |
| 414 # We could improve the tool to check all baselines in upper | |
| 415 # and lower | |
| 416 # levels and remove all duplicated baselines. | |
| 417 if self._IsDupBaseline(temp_name, | |
| 418 expected_fullpath, | |
| 419 test, | |
| 420 suffix, | |
| 421 self._platform): | |
| 422 os.remove(temp_name) | |
| 423 self._DeleteBaseline(expected_fullpath) | |
| 424 continue | |
| 425 | |
| 426 # Create the new baseline directory if it doesn't already | |
| 427 # exist. | |
| 428 path_utils.MaybeMakeDirectory( | |
| 429 os.path.dirname(expected_fullpath)) | |
| 430 | |
| 431 shutil.move(temp_name, expected_fullpath) | |
| 432 | |
| 433 if not self._SvnAdd(expected_fullpath): | |
| 434 svn_error = True | |
| 435 elif suffix != '.checksum': | |
| 436 self._CreateHtmlBaselineFiles(expected_fullpath) | |
| 437 | |
| 438 if not found: | |
| 439 logging.warn(' No new baselines found in archive.') | |
| 440 else: | |
| 441 if svn_error: | |
| 442 logging.warn(' Failed to add baselines to SVN.') | |
| 443 else: | |
| 444 logging.info(' Rebaseline succeeded.') | |
| 445 self._rebaselined_tests.append(test) | |
| 446 | |
| 447 test_no += 1 | |
| 448 | |
| 449 zip_file.close() | |
| 450 os.remove(archive_file) | |
| 451 | |
| 452 return self._rebaselined_tests | |
| 453 | |
| 454 def _IsDupBaseline(self, new_baseline, baseline_path, test, suffix, | |
| 455 platform): | |
| 456 """Check whether a baseline is duplicate and can fallback to same | |
| 457 baseline for another platform. For example, if a test has same | |
| 458 baseline on linux and windows, then we only store windows | |
| 459 baseline and linux baseline will fallback to the windows version. | |
| 460 | |
| 461 Args: | |
| 462 expected_filename: baseline expectation file name. | |
| 463 test: test name. | |
| 464 suffix: file suffix of the expected results, including dot; | |
| 465 e.g. '.txt' or '.png'. | |
| 466 platform: baseline platform 'mac', 'win' or 'linux'. | |
| 467 | |
| 468 Returns: | |
| 469 True if the baseline is unnecessary. | |
| 470 False otherwise. | |
| 471 """ | |
| 472 test_filepath = os.path.join(path_utils.LayoutTestsDir(), test) | |
| 473 all_baselines = path_utils.ExpectedBaselines(test_filepath, | |
| 474 suffix, | |
| 475 platform, | |
| 476 True) | |
| 477 for (fallback_dir, fallback_file) in all_baselines: | |
| 478 if fallback_dir and fallback_file: | |
| 479 fallback_fullpath = os.path.normpath( | |
| 480 os.path.join(fallback_dir, fallback_file)) | |
| 481 if fallback_fullpath.lower() != baseline_path.lower(): | |
| 482 if not self._DiffBaselines(new_baseline, | |
| 483 fallback_fullpath): | |
| 484 logging.info(' Found same baseline at %s', | |
| 485 fallback_fullpath) | |
| 486 return True | |
| 487 else: | |
| 488 return False | |
| 489 | |
| 490 return False | |
| 491 | |
| 492 def _DiffBaselines(self, file1, file2): | |
| 493 """Check whether two baselines are different. | |
| 494 | |
| 495 Args: | |
| 496 file1, file2: full paths of the baselines to compare. | |
| 497 | |
| 498 Returns: | |
| 499 True if two files are different or have different extensions. | |
| 500 False otherwise. | |
| 501 """ | |
| 502 | |
| 503 ext1 = os.path.splitext(file1)[1].upper() | |
| 504 ext2 = os.path.splitext(file2)[1].upper() | |
| 505 if ext1 != ext2: | |
| 506 logging.warn('Files to compare have different ext. ' | |
| 507 'File1: %s; File2: %s', file1, file2) | |
| 508 return True | |
| 509 | |
| 510 if ext1 == '.PNG': | |
| 511 return image_diff.ImageDiff(self._platform, '').DiffFiles(file1, | |
| 512 file2) | |
| 513 else: | |
| 514 return text_diff.TestTextDiff(self._platform, '').DiffFiles(file1, | |
| 515 file2) | |
| 516 | |
| 517 def _DeleteBaseline(self, filename): | |
| 518 """Remove the file from repository and delete it from disk. | |
| 519 | |
| 520 Args: | |
| 521 filename: full path of the file to delete. | |
| 522 """ | |
| 523 | |
| 524 if not filename or not os.path.isfile(filename): | |
| 525 return | |
| 526 | |
| 527 if self._repo_type == REPO_SVN: | |
| 528 parent_dir, basename = os.path.split(filename) | |
| 529 original_dir = os.getcwd() | |
| 530 os.chdir(parent_dir) | |
| 531 RunShell(['svn', 'delete', '--force', basename], False) | |
| 532 os.chdir(original_dir) | |
| 533 else: | |
| 534 os.remove(filename) | |
| 535 | |
| 536 def _UpdateRebaselinedTestsInFile(self, backup): | |
| 537 """Update the rebaselined tests in test expectations file. | |
| 538 | |
| 539 Args: | |
| 540 backup: if True, backup the original test expectations file. | |
| 541 | |
| 542 Returns: | |
| 543 no | |
| 544 """ | |
| 545 | |
| 546 if self._rebaselined_tests: | |
| 547 self._test_expectations.RemovePlatformFromFile( | |
| 548 self._rebaselined_tests, self._platform, backup) | |
| 549 else: | |
| 550 logging.info('No test was rebaselined so nothing to remove.') | |
| 551 | |
| 552 def _SvnAdd(self, filename): | |
| 553 """Add the file to SVN repository. | |
| 554 | |
| 555 Args: | |
| 556 filename: full path of the file to add. | |
| 557 | |
| 558 Returns: | |
| 559 True if the file already exists in SVN or is sucessfully added | |
| 560 to SVN. | |
| 561 False otherwise. | |
| 562 """ | |
| 563 | |
| 564 if not filename: | |
| 565 return False | |
| 566 | |
| 567 parent_dir, basename = os.path.split(filename) | |
| 568 if self._repo_type != REPO_SVN or parent_dir == filename: | |
| 569 logging.info("No svn checkout found, skip svn add.") | |
| 570 return True | |
| 571 | |
| 572 original_dir = os.getcwd() | |
| 573 os.chdir(parent_dir) | |
| 574 status_output = RunShell(['svn', 'status', basename], False) | |
| 575 os.chdir(original_dir) | |
| 576 output = status_output.upper() | |
| 577 if output.startswith('A') or output.startswith('M'): | |
| 578 logging.info(' File already added to SVN: "%s"', filename) | |
| 579 return True | |
| 580 | |
| 581 if output.find('IS NOT A WORKING COPY') >= 0: | |
| 582 logging.info(' File is not a working copy, add its parent: "%s"', | |
| 583 parent_dir) | |
| 584 return self._SvnAdd(parent_dir) | |
| 585 | |
| 586 os.chdir(parent_dir) | |
| 587 add_output = RunShell(['svn', 'add', basename], True) | |
| 588 os.chdir(original_dir) | |
| 589 output = add_output.upper().rstrip() | |
| 590 if output.startswith('A') and output.find(basename.upper()) >= 0: | |
| 591 logging.info(' Added new file: "%s"', filename) | |
| 592 self._SvnPropSet(filename) | |
| 593 return True | |
| 594 | |
| 595 if (not status_output) and (add_output.upper().find( | |
| 596 'ALREADY UNDER VERSION CONTROL') >= 0): | |
| 597 logging.info(' File already under SVN and has no change: "%s"', | |
| 598 filename) | |
| 599 return True | |
| 600 | |
| 601 logging.warn(' Failed to add file to SVN: "%s"', filename) | |
| 602 logging.warn(' Svn status output: "%s"', status_output) | |
| 603 logging.warn(' Svn add output: "%s"', add_output) | |
| 604 return False | |
| 605 | |
| 606 def _SvnPropSet(self, filename): | |
| 607 """Set the baseline property | |
| 608 | |
| 609 Args: | |
| 610 filename: full path of the file to add. | |
| 611 | |
| 612 Returns: | |
| 613 True if the file already exists in SVN or is sucessfully added | |
| 614 to SVN. | |
| 615 False otherwise. | |
| 616 """ | |
| 617 ext = os.path.splitext(filename)[1].upper() | |
| 618 if ext != '.TXT' and ext != '.PNG' and ext != '.CHECKSUM': | |
| 619 return | |
| 620 | |
| 621 parent_dir, basename = os.path.split(filename) | |
| 622 original_dir = os.getcwd() | |
| 623 os.chdir(parent_dir) | |
| 624 if ext == '.PNG': | |
| 625 cmd = ['svn', 'pset', 'svn:mime-type', 'image/png', basename] | |
| 626 else: | |
| 627 cmd = ['svn', 'pset', 'svn:eol-style', 'LF', basename] | |
| 628 | |
| 629 logging.debug(' Set svn prop: %s', ' '.join(cmd)) | |
| 630 RunShell(cmd, False) | |
| 631 os.chdir(original_dir) | |
| 632 | |
| 633 def _CreateHtmlBaselineFiles(self, baseline_fullpath): | |
| 634 """Create baseline files (old, new and diff) in html directory. | |
| 635 | |
| 636 The files are used to compare the rebaselining results. | |
| 637 | |
| 638 Args: | |
| 639 baseline_fullpath: full path of the expected baseline file. | |
| 640 """ | |
| 641 | |
| 642 if not baseline_fullpath or not os.path.exists(baseline_fullpath): | |
| 643 return | |
| 644 | |
| 645 # Copy the new baseline to html directory for result comparison. | |
| 646 baseline_filename = os.path.basename(baseline_fullpath) | |
| 647 new_file = GetResultFileFullpath(self._options.html_directory, | |
| 648 baseline_filename, | |
| 649 self._platform, | |
| 650 'new') | |
| 651 shutil.copyfile(baseline_fullpath, new_file) | |
| 652 logging.info(' Html: copied new baseline file from "%s" to "%s".', | |
| 653 baseline_fullpath, new_file) | |
| 654 | |
| 655 # Get the old baseline from SVN and save to the html directory. | |
| 656 output = RunShell(['svn', 'cat', '-r', 'BASE', baseline_fullpath]) | |
| 657 if (not output) or (output.upper().rstrip().endswith( | |
| 658 'NO SUCH FILE OR DIRECTORY')): | |
| 659 logging.info(' No base file: "%s"', baseline_fullpath) | |
| 660 return | |
| 661 base_file = GetResultFileFullpath(self._options.html_directory, | |
| 662 baseline_filename, | |
| 663 self._platform, | |
| 664 'old') | |
| 665 f = open(base_file, 'wb') | |
| 666 f.write(output) | |
| 667 f.close() | |
| 668 logging.info(' Html: created old baseline file: "%s".', | |
| 669 base_file) | |
| 670 | |
| 671 # Get the diff between old and new baselines and save to the html dir. | |
| 672 if baseline_filename.upper().endswith('.TXT'): | |
| 673 # If the user specified a custom diff command in their svn config | |
| 674 # file, then it'll be used when we do svn diff, which we don't want | |
| 675 # to happen since we want the unified diff. Using --diff-cmd=diff | |
| 676 # doesn't always work, since they can have another diff executable | |
| 677 # in their path that gives different line endings. So we use a | |
| 678 # bogus temp directory as the config directory, which gets | |
| 679 # around these problems. | |
| 680 if sys.platform.startswith("win"): | |
| 681 parent_dir = tempfile.gettempdir() | |
| 682 else: | |
| 683 parent_dir = sys.path[0] # tempdir is not secure. | |
| 684 bogus_dir = os.path.join(parent_dir, "temp_svn_config") | |
| 685 logging.debug(' Html: temp config dir: "%s".', bogus_dir) | |
| 686 if not os.path.exists(bogus_dir): | |
| 687 os.mkdir(bogus_dir) | |
| 688 delete_bogus_dir = True | |
| 689 else: | |
| 690 delete_bogus_dir = False | |
| 691 | |
| 692 output = RunShell(["svn", "diff", "--config-dir", bogus_dir, | |
| 693 baseline_fullpath]) | |
| 694 if output: | |
| 695 diff_file = GetResultFileFullpath(self._options.html_directory, | |
| 696 baseline_filename, | |
| 697 self._platform, | |
| 698 'diff') | |
| 699 f = open(diff_file, 'wb') | |
| 700 f.write(output) | |
| 701 f.close() | |
| 702 logging.info(' Html: created baseline diff file: "%s".', | |
| 703 diff_file) | |
| 704 | |
| 705 if delete_bogus_dir: | |
| 706 shutil.rmtree(bogus_dir, True) | |
| 707 logging.debug(' Html: removed temp config dir: "%s".', | |
| 708 bogus_dir) | |
| 709 | |
| 710 | |
| 711 class HtmlGenerator(object): | |
| 712 """Class to generate rebaselining result comparison html.""" | |
| 713 | |
| 714 HTML_REBASELINE = ('<html>' | |
| 715 '<head>' | |
| 716 '<style>' | |
| 717 'body {font-family: sans-serif;}' | |
| 718 '.mainTable {background: #666666;}' | |
| 719 '.mainTable td , .mainTable th {background: white;}' | |
| 720 '.detail {margin-left: 10px; margin-top: 3px;}' | |
| 721 '</style>' | |
| 722 '<title>Rebaselining Result Comparison (%(time)s)' | |
| 723 '</title>' | |
| 724 '</head>' | |
| 725 '<body>' | |
| 726 '<h2>Rebaselining Result Comparison (%(time)s)</h2>' | |
| 727 '%(body)s' | |
| 728 '</body>' | |
| 729 '</html>') | |
| 730 HTML_NO_REBASELINING_TESTS = ( | |
| 731 '<p>No tests found that need rebaselining.</p>') | |
| 732 HTML_TABLE_TEST = ('<table class="mainTable" cellspacing=1 cellpadding=5>' | |
| 733 '%s</table><br>') | |
| 734 HTML_TR_TEST = ('<tr>' | |
| 735 '<th style="background-color: #CDECDE; border-bottom: ' | |
| 736 '1px solid black; font-size: 18pt; font-weight: bold" ' | |
| 737 'colspan="5">' | |
| 738 '<a href="%s">%s</a>' | |
| 739 '</th>' | |
| 740 '</tr>') | |
| 741 HTML_TEST_DETAIL = ('<div class="detail">' | |
| 742 '<tr>' | |
| 743 '<th width="100">Baseline</th>' | |
| 744 '<th width="100">Platform</th>' | |
| 745 '<th width="200">Old</th>' | |
| 746 '<th width="200">New</th>' | |
| 747 '<th width="150">Difference</th>' | |
| 748 '</tr>' | |
| 749 '%s' | |
| 750 '</div>') | |
| 751 HTML_TD_NOLINK = '<td align=center><a>%s</a></td>' | |
| 752 HTML_TD_LINK = '<td align=center><a href="%(uri)s">%(name)s</a></td>' | |
| 753 HTML_TD_LINK_IMG = ('<td><a href="%(uri)s">' | |
| 754 '<img style="width: 200" src="%(uri)s" /></a></td>') | |
| 755 HTML_TR = '<tr>%s</tr>' | |
| 756 | |
| 757 def __init__(self, options, platforms, rebaselining_tests): | |
| 758 self._html_directory = options.html_directory | |
| 759 self._platforms = platforms | |
| 760 self._rebaselining_tests = rebaselining_tests | |
| 761 self._html_file = os.path.join(options.html_directory, | |
| 762 'rebaseline.html') | |
| 763 | |
| 764 def GenerateHtml(self): | |
| 765 """Generate html file for rebaselining result comparison.""" | |
| 766 | |
| 767 logging.info('Generating html file') | |
| 768 | |
| 769 html_body = '' | |
| 770 if not self._rebaselining_tests: | |
| 771 html_body += self.HTML_NO_REBASELINING_TESTS | |
| 772 else: | |
| 773 tests = list(self._rebaselining_tests) | |
| 774 tests.sort() | |
| 775 | |
| 776 test_no = 1 | |
| 777 for test in tests: | |
| 778 logging.info('Test %d: %s', test_no, test) | |
| 779 html_body += self._GenerateHtmlForOneTest(test) | |
| 780 | |
| 781 html = self.HTML_REBASELINE % ({'time': time.asctime(), | |
| 782 'body': html_body}) | |
| 783 logging.debug(html) | |
| 784 | |
| 785 f = open(self._html_file, 'w') | |
| 786 f.write(html) | |
| 787 f.close() | |
| 788 | |
| 789 logging.info('Baseline comparison html generated at "%s"', | |
| 790 self._html_file) | |
| 791 | |
| 792 def ShowHtml(self): | |
| 793 """Launch the rebaselining html in brwoser.""" | |
| 794 | |
| 795 logging.info('Launching html: "%s"', self._html_file) | |
| 796 | |
| 797 html_uri = path_utils.FilenameToUri(self._html_file) | |
| 798 webbrowser.open(html_uri, 1) | |
| 799 | |
| 800 logging.info('Html launched.') | |
| 801 | |
| 802 def _GenerateBaselineLinks(self, test_basename, suffix, platform): | |
| 803 """Generate links for baseline results (old, new and diff). | |
| 804 | |
| 805 Args: | |
| 806 test_basename: base filename of the test | |
| 807 suffix: baseline file suffixes: '.txt', '.png' | |
| 808 platform: win, linux or mac | |
| 809 | |
| 810 Returns: | |
| 811 html links for showing baseline results (old, new and diff) | |
| 812 """ | |
| 813 | |
| 814 baseline_filename = '%s-expected%s' % (test_basename, suffix) | |
| 815 logging.debug(' baseline filename: "%s"', baseline_filename) | |
| 816 | |
| 817 new_file = GetResultFileFullpath(self._html_directory, | |
| 818 baseline_filename, | |
| 819 platform, | |
| 820 'new') | |
| 821 logging.info(' New baseline file: "%s"', new_file) | |
| 822 if not os.path.exists(new_file): | |
| 823 logging.info(' No new baseline file: "%s"', new_file) | |
| 824 return '' | |
| 825 | |
| 826 old_file = GetResultFileFullpath(self._html_directory, | |
| 827 baseline_filename, | |
| 828 platform, | |
| 829 'old') | |
| 830 logging.info(' Old baseline file: "%s"', old_file) | |
| 831 if suffix == '.png': | |
| 832 html_td_link = self.HTML_TD_LINK_IMG | |
| 833 else: | |
| 834 html_td_link = self.HTML_TD_LINK | |
| 835 | |
| 836 links = '' | |
| 837 if os.path.exists(old_file): | |
| 838 links += html_td_link % {'uri': path_utils.FilenameToUri(old_file), | |
| 839 'name': baseline_filename} | |
| 840 else: | |
| 841 logging.info(' No old baseline file: "%s"', old_file) | |
| 842 links += self.HTML_TD_NOLINK % '' | |
| 843 | |
| 844 links += html_td_link % {'uri': path_utils.FilenameToUri(new_file), | |
| 845 'name': baseline_filename} | |
| 846 | |
| 847 diff_file = GetResultFileFullpath(self._html_directory, | |
| 848 baseline_filename, | |
| 849 platform, | |
| 850 'diff') | |
| 851 logging.info(' Baseline diff file: "%s"', diff_file) | |
| 852 if os.path.exists(diff_file): | |
| 853 links += html_td_link % {'uri': path_utils.FilenameToUri( | |
| 854 diff_file), 'name': 'Diff'} | |
| 855 else: | |
| 856 logging.info(' No baseline diff file: "%s"', diff_file) | |
| 857 links += self.HTML_TD_NOLINK % '' | |
| 858 | |
| 859 return links | |
| 860 | |
| 861 def _GenerateHtmlForOneTest(self, test): | |
| 862 """Generate html for one rebaselining test. | |
| 863 | |
| 864 Args: | |
| 865 test: layout test name | |
| 866 | |
| 867 Returns: | |
| 868 html that compares baseline results for the test. | |
| 869 """ | |
| 870 | |
| 871 test_basename = os.path.basename(os.path.splitext(test)[0]) | |
| 872 logging.info(' basename: "%s"', test_basename) | |
| 873 rows = [] | |
| 874 for suffix in BASELINE_SUFFIXES: | |
| 875 if suffix == '.checksum': | |
| 876 continue | |
| 877 | |
| 878 logging.info(' Checking %s files', suffix) | |
| 879 for platform in self._platforms: | |
| 880 links = self._GenerateBaselineLinks(test_basename, suffix, | |
| 881 platform) | |
| 882 if links: | |
| 883 row = self.HTML_TD_NOLINK % self._GetBaselineResultType( | |
| 884 suffix) | |
| 885 row += self.HTML_TD_NOLINK % platform | |
| 886 row += links | |
| 887 logging.debug(' html row: %s', row) | |
| 888 | |
| 889 rows.append(self.HTML_TR % row) | |
| 890 | |
| 891 if rows: | |
| 892 test_path = os.path.join(path_utils.LayoutTestsDir(), test) | |
| 893 html = self.HTML_TR_TEST % (path_utils.FilenameToUri(test_path), | |
| 894 test) | |
| 895 html += self.HTML_TEST_DETAIL % ' '.join(rows) | |
| 896 | |
| 897 logging.debug(' html for test: %s', html) | |
| 898 return self.HTML_TABLE_TEST % html | |
| 899 | |
| 900 return '' | |
| 901 | |
| 902 def _GetBaselineResultType(self, suffix): | |
| 903 """Name of the baseline result type.""" | |
| 904 | |
| 905 if suffix == '.png': | |
| 906 return 'Pixel' | |
| 907 elif suffix == '.txt': | |
| 908 return 'Render Tree' | |
| 909 else: | |
| 910 return 'Other' | |
| 911 | |
| 912 | |
| 913 def main(): | |
| 914 """Main function to produce new baselines.""" | |
| 915 | |
| 916 option_parser = optparse.OptionParser() | |
| 917 option_parser.add_option('-v', '--verbose', | |
| 918 action='store_true', | |
| 919 default=False, | |
| 920 help='include debug-level logging.') | |
| 921 | |
| 922 option_parser.add_option('-p', '--platforms', | |
| 923 default='mac,win,win-xp,win-vista,linux', | |
| 924 help=('Comma delimited list of platforms ' | |
| 925 'that need rebaselining.')) | |
| 926 | |
| 927 option_parser.add_option('-u', '--archive_url', | |
| 928 default=('http://build.chromium.org/buildbot/' | |
| 929 'layout_test_results'), | |
| 930 help=('Url to find the layout test result archive' | |
| 931 ' file.')) | |
| 932 | |
| 933 option_parser.add_option('-w', '--webkit_canary', | |
| 934 action='store_true', | |
| 935 default=False, | |
| 936 help=('If True, pull baselines from webkit.org ' | |
| 937 'canary bot.')) | |
| 938 | |
| 939 option_parser.add_option('-b', '--backup', | |
| 940 action='store_true', | |
| 941 default=False, | |
| 942 help=('Whether or not to backup the original test' | |
| 943 ' expectations file after rebaseline.')) | |
| 944 | |
| 945 option_parser.add_option('-d', '--html_directory', | |
| 946 default='', | |
| 947 help=('The directory that stores the results for' | |
| 948 ' rebaselining comparison.')) | |
| 949 | |
| 950 options = option_parser.parse_args()[0] | |
| 951 | |
| 952 # Set up our logging format. | |
| 953 log_level = logging.INFO | |
| 954 if options.verbose: | |
| 955 log_level = logging.DEBUG | |
| 956 logging.basicConfig(level=log_level, | |
| 957 format=('%(asctime)s %(filename)s:%(lineno)-3d ' | |
| 958 '%(levelname)s %(message)s'), | |
| 959 datefmt='%y%m%d %H:%M:%S') | |
| 960 | |
| 961 # Verify 'platforms' option is valid | |
| 962 if not options.platforms: | |
| 963 logging.error('Invalid "platforms" option. --platforms must be ' | |
| 964 'specified in order to rebaseline.') | |
| 965 sys.exit(1) | |
| 966 platforms = [p.strip().lower() for p in options.platforms.split(',')] | |
| 967 for platform in platforms: | |
| 968 if not platform in REBASELINE_PLATFORM_ORDER: | |
| 969 logging.error('Invalid platform: "%s"' % (platform)) | |
| 970 sys.exit(1) | |
| 971 | |
| 972 # Adjust the platform order so rebaseline tool is running at the order of | |
| 973 # 'mac', 'win' and 'linux'. This is in same order with layout test baseline | |
| 974 # search paths. It simplifies how the rebaseline tool detects duplicate | |
| 975 # baselines. Check _IsDupBaseline method for details. | |
| 976 rebaseline_platforms = [] | |
| 977 for platform in REBASELINE_PLATFORM_ORDER: | |
| 978 if platform in platforms: | |
| 979 rebaseline_platforms.append(platform) | |
| 980 | |
| 981 options.html_directory = SetupHtmlDirectory(options.html_directory) | |
| 982 | |
| 983 rebaselining_tests = set() | |
| 984 backup = options.backup | |
| 985 for platform in rebaseline_platforms: | |
| 986 rebaseliner = Rebaseliner(platform, options) | |
| 987 | |
| 988 logging.info('') | |
| 989 LogDashedString('Rebaseline started', platform) | |
| 990 if rebaseliner.Run(backup): | |
| 991 # Only need to backup one original copy of test expectation file. | |
| 992 backup = False | |
| 993 LogDashedString('Rebaseline done', platform) | |
| 994 else: | |
| 995 LogDashedString('Rebaseline failed', platform, logging.ERROR) | |
| 996 | |
| 997 rebaselining_tests |= set(rebaseliner.GetRebaseliningTests()) | |
| 998 | |
| 999 logging.info('') | |
| 1000 LogDashedString('Rebaselining result comparison started', None) | |
| 1001 html_generator = HtmlGenerator(options, | |
| 1002 rebaseline_platforms, | |
| 1003 rebaselining_tests) | |
| 1004 html_generator.GenerateHtml() | |
| 1005 html_generator.ShowHtml() | |
| 1006 LogDashedString('Rebaselining result comparison done', None) | |
| 1007 | |
| 1008 sys.exit(0) | |
| 1009 | |
| 1010 if '__main__' == __name__: | |
| 1011 main() | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 545145:
Move the layout test scripts into a 'webkitpy' subdirectory in preparation... (Closed)
Base URL: svn://chrome-svn/chrome/trunk/src/