OLD | NEW |
1 #!usr/bin/env python | 1 #!usr/bin/env python |
2 # Copyright (c) 2006-2009 The Chromium Authors. All rights reserved. | 2 # Copyright (c) 2006-2009 The Chromium Authors. All rights reserved. |
3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
5 | 5 |
6 """Rebaselining tool that automatically produces baselines for all platforms. | 6 """Rebaselining tool that automatically produces baselines for all platforms. |
7 | 7 |
8 The script does the following for each platform specified: | 8 The script does the following for each platform specified: |
9 1. Compile a list of tests that need rebaselining. | 9 1. Compile a list of tests that need rebaselining. |
10 2. Download test result archive from buildbot for the platform. | 10 2. Download test result archive from buildbot for the platform. |
(...skipping 33 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
44 'win-vista': 'webkit-dbg-vista', | 44 'win-vista': 'webkit-dbg-vista', |
45 'win-xp': 'webkit-rel', | 45 'win-xp': 'webkit-rel', |
46 'mac': 'webkit-rel-mac5', | 46 'mac': 'webkit-rel-mac5', |
47 'linux': 'webkit-rel-linux', | 47 'linux': 'webkit-rel-linux', |
48 'win-canary': 'webkit-rel-webkit-org', | 48 'win-canary': 'webkit-rel-webkit-org', |
49 'win-vista-canary': 'webkit-dbg-vista', | 49 'win-vista-canary': 'webkit-dbg-vista', |
50 'win-xp-canary': 'webkit-rel-webkit-org', | 50 'win-xp-canary': 'webkit-rel-webkit-org', |
51 'mac-canary': 'webkit-rel-mac-webkit-org', | 51 'mac-canary': 'webkit-rel-mac-webkit-org', |
52 'linux-canary': 'webkit-rel-linux-webkit-org'} | 52 'linux-canary': 'webkit-rel-linux-webkit-org'} |
53 | 53 |
54 | |
55 def RunShellWithReturnCode(command, print_output=False): | 54 def RunShellWithReturnCode(command, print_output=False): |
56 """Executes a command and returns the output and process return code. | 55 """Executes a command and returns the output and process return code. |
57 | 56 |
58 Args: | 57 Args: |
59 command: program and arguments. | 58 command: program and arguments. |
60 print_output: if true, print the command results to standard output. | 59 print_output: if true, print the command results to standard output. |
61 | 60 |
62 Returns: | 61 Returns: |
63 command output, return code | 62 command output, return code |
64 """ | 63 """ |
65 | 64 |
66 # Use a shell for subcommands on Windows to get a PATH search. | 65 # Use a shell for subcommands on Windows to get a PATH search. |
67 use_shell = sys.platform.startswith('win') | 66 use_shell = sys.platform.startswith('win') |
68 p = subprocess.Popen(command, stdout=subprocess.PIPE, | 67 p = subprocess.Popen(command, stdout=subprocess.PIPE, |
69 stderr=subprocess.STDOUT, shell=use_shell) | 68 stderr=subprocess.STDOUT, shell=use_shell) |
70 if print_output: | 69 if print_output: |
71 output_array = [] | 70 output_array = [] |
72 while True: | 71 while True: |
73 line = p.stdout.readline() | 72 line = p.stdout.readline() |
74 if not line: | 73 if not line: |
75 break | 74 break |
76 if print_output: | 75 if print_output: |
77 print line.strip('\n') | 76 print line.strip('\n') |
78 output_array.append(line) | 77 output_array.append(line) |
79 output = ''.join(output_array) | 78 output = ''.join(output_array) |
80 else: | 79 else: |
81 output = p.stdout.read() | 80 output = p.stdout.read() |
82 p.wait() | 81 p.wait() |
83 p.stdout.close() | 82 p.stdout.close() |
84 | 83 |
85 return output, p.returncode | 84 return output, p.returncode |
86 | |
87 | 85 |
88 def RunShell(command, print_output=False): | 86 def RunShell(command, print_output=False): |
89 """Executes a command and returns the output. | 87 """Executes a command and returns the output. |
90 | 88 |
91 Args: | 89 Args: |
92 command: program and arguments. | 90 command: program and arguments. |
93 print_output: if true, print the command results to standard output. | 91 print_output: if true, print the command results to standard output. |
94 | 92 |
95 Returns: | 93 Returns: |
96 command output | 94 command output |
97 """ | 95 """ |
98 | 96 |
99 output, return_code = RunShellWithReturnCode(command, print_output) | 97 output, return_code = RunShellWithReturnCode(command, print_output) |
100 return output | 98 return output |
101 | |
102 | 99 |
103 def LogDashedString(text, platform, logging_level=logging.INFO): | 100 def LogDashedString(text, platform, logging_level=logging.INFO): |
104 """Log text message with dashes on both sides.""" | 101 """Log text message with dashes on both sides.""" |
105 | 102 |
106 msg = text | 103 msg = text |
107 if platform: | 104 if platform: |
108 msg += ': ' + platform | 105 msg += ': ' + platform |
109 if len(msg) < 78: | 106 if len(msg) < 78: |
110 dashes = '-' * ((78 - len(msg)) / 2) | 107 dashes = '-' * ((78 - len(msg)) / 2) |
111 msg = '%s %s %s' % (dashes, msg, dashes) | 108 msg = '%s %s %s' % (dashes, msg, dashes) |
112 | 109 |
113 if logging_level == logging.ERROR: | 110 if logging_level == logging.ERROR: |
114 logging.error(msg) | 111 logging.error(msg) |
115 elif logging_level == logging.WARNING: | 112 elif logging_level == logging.WARNING: |
116 logging.warn(msg) | 113 logging.warn(msg) |
117 else: | 114 else: |
118 logging.info(msg) | 115 logging.info(msg) |
119 | 116 |
120 | 117 |
121 def SetupHtmlDirectory(html_directory): | 118 def SetupHtmlDirectory(html_directory): |
122 """Setup the directory to store html results. | 119 """Setup the directory to store html results. |
123 | 120 |
124 All html related files are stored in the "rebaseline_html" subdirectory. | 121 All html related files are stored in the "rebaseline_html" subdirectory. |
125 | 122 |
126 Args: | 123 Args: |
127 html_directory: parent directory that stores the rebaselining results. | 124 html_directory: parent directory that stores the rebaselining results. |
128 If None, a temp directory is created. | 125 If None, a temp directory is created. |
129 | 126 |
130 Returns: | 127 Returns: |
131 the directory that stores the html related rebaselining results. | 128 the directory that stores the html related rebaselining results. |
132 """ | 129 """ |
133 | 130 |
134 if not html_directory: | 131 if not html_directory: |
135 html_directory = tempfile.mkdtemp() | 132 html_directory = tempfile.mkdtemp() |
136 elif not os.path.exists(html_directory): | 133 elif not os.path.exists(html_directory): |
137 os.mkdir(html_directory) | 134 os.mkdir(html_directory) |
138 | 135 |
139 html_directory = os.path.join(html_directory, 'rebaseline_html') | 136 html_directory = os.path.join(html_directory, 'rebaseline_html') |
140 logging.info('Html directory: "%s"', html_directory) | 137 logging.info('Html directory: "%s"', html_directory) |
141 | 138 |
142 if os.path.exists(html_directory): | 139 if os.path.exists(html_directory): |
143 shutil.rmtree(html_directory, True) | 140 shutil.rmtree(html_directory, True) |
144 logging.info('Deleted file at html directory: "%s"', html_directory) | 141 logging.info('Deleted file at html directory: "%s"', html_directory) |
145 | 142 |
146 if not os.path.exists(html_directory): | 143 if not os.path.exists(html_directory): |
147 os.mkdir(html_directory) | 144 os.mkdir(html_directory) |
148 return html_directory | 145 return html_directory |
149 | 146 |
150 | 147 |
151 def GetResultFileFullpath(html_directory, baseline_filename, platform, | 148 def GetResultFileFullpath(html_directory, baseline_filename, platform, |
152 result_type): | 149 result_type): |
153 """Get full path of the baseline result file. | 150 """Get full path of the baseline result file. |
154 | 151 |
155 Args: | 152 Args: |
156 html_directory: directory that stores the html related files. | 153 html_directory: directory that stores the html related files. |
157 baseline_filename: name of the baseline file. | 154 baseline_filename: name of the baseline file. |
158 platform: win, linux or mac | 155 platform: win, linux or mac |
159 result_type: type of the baseline result: '.txt', '.png'. | 156 result_type: type of the baseline result: '.txt', '.png'. |
160 | 157 |
161 Returns: | 158 Returns: |
162 Full path of the baseline file for rebaselining result comparison. | 159 Full path of the baseline file for rebaselining result comparison. |
163 """ | 160 """ |
164 | 161 |
165 base, ext = os.path.splitext(baseline_filename) | 162 base, ext = os.path.splitext(baseline_filename) |
166 result_filename = '%s-%s-%s%s' % (base, platform, result_type, ext) | 163 result_filename = '%s-%s-%s%s' % (base, platform, result_type, ext) |
167 fullpath = os.path.join(html_directory, result_filename) | 164 fullpath = os.path.join(html_directory, result_filename) |
168 logging.debug(' Result file full path: "%s".', fullpath) | 165 logging.debug(' Result file full path: "%s".', fullpath) |
169 return fullpath | 166 return fullpath |
170 | 167 |
171 | 168 |
172 class Rebaseliner(object): | 169 class Rebaseliner(object): |
173 """Class to produce new baselines for a given platform.""" | 170 """Class to produce new baselines for a given platform.""" |
174 | 171 |
175 REVISION_REGEX = r'<a href=\"(\d+)/\">' | 172 REVISION_REGEX = r'<a href=\"(\d+)/\">' |
176 | 173 |
177 def __init__(self, platform, options): | 174 def __init__(self, platform, options): |
178 self._file_dir = path_utils.GetAbsolutePath( | 175 self._file_dir = path_utils.GetAbsolutePath( |
179 os.path.dirname(sys.argv[0])) | 176 os.path.dirname(os.path.dirname(sys.argv[0]))) |
180 self._platform = platform | 177 self._platform = platform |
181 self._options = options | 178 self._options = options |
182 self._rebaselining_tests = [] | 179 self._rebaselining_tests = [] |
183 self._rebaselined_tests = [] | 180 self._rebaselined_tests = [] |
184 | 181 |
185 # Create tests and expectations helper which is used to: | 182 # Create tests and expectations helper which is used to: |
186 # -. compile list of tests that need rebaselining. | 183 # -. compile list of tests that need rebaselining. |
187 # -. update the tests in test_expectations file after rebaseline | 184 # -. update the tests in test_expectations file after rebaseline is done. |
188 # is done. | 185 self._test_expectations = test_expectations.TestExpectations(None, |
189 self._test_expectations = \ | 186 self._file_dir, |
190 test_expectations.TestExpectations(None, | 187 platform, |
191 self._file_dir, | 188 False, |
192 platform, | 189 False) |
193 False, | 190 |
194 False) | 191 self._repo_type = self._GetRepoType() |
195 | 192 |
196 self._repo_type = self._GetRepoType() | 193 def Run(self, backup): |
197 | 194 """Run rebaseline process.""" |
198 def Run(self, backup): | 195 |
199 """Run rebaseline process.""" | 196 LogDashedString('Compiling rebaselining tests', self._platform) |
200 | 197 if not self._CompileRebaseliningTests(): |
201 LogDashedString('Compiling rebaselining tests', self._platform) | 198 return True |
202 if not self._CompileRebaseliningTests(): | 199 |
| 200 LogDashedString('Downloading archive', self._platform) |
| 201 archive_file = self._DownloadBuildBotArchive() |
| 202 logging.info('') |
| 203 if not archive_file: |
| 204 logging.error('No archive found.') |
| 205 return False |
| 206 |
| 207 LogDashedString('Extracting and adding new baselines', self._platform) |
| 208 if not self._ExtractAndAddNewBaselines(archive_file): |
| 209 return False |
| 210 |
| 211 LogDashedString('Updating rebaselined tests in file', self._platform) |
| 212 self._UpdateRebaselinedTestsInFile(backup) |
| 213 logging.info('') |
| 214 |
| 215 if len(self._rebaselining_tests) != len(self._rebaselined_tests): |
| 216 logging.warning('NOT ALL TESTS THAT NEED REBASELINING HAVE BEEN ' |
| 217 'REBASELINED.') |
| 218 logging.warning(' Total tests needing rebaselining: %d', |
| 219 len(self._rebaselining_tests)) |
| 220 logging.warning(' Total tests rebaselined: %d', |
| 221 len(self._rebaselined_tests)) |
| 222 return False |
| 223 |
| 224 logging.warning('All tests needing rebaselining were successfully ' |
| 225 'rebaselined.') |
| 226 |
| 227 return True |
| 228 |
| 229 def GetRebaseliningTests(self): |
| 230 return self._rebaselining_tests |
| 231 |
| 232 def _GetRepoType(self): |
| 233 """Get the repository type that client is using.""" |
| 234 |
| 235 output, return_code = RunShellWithReturnCode(['svn', 'info'], False) |
| 236 if return_code == 0: |
| 237 return REPO_SVN |
| 238 |
| 239 return REPO_UNKNOWN |
| 240 |
| 241 def _CompileRebaseliningTests(self): |
| 242 """Compile list of tests that need rebaselining for the platform. |
| 243 |
| 244 Returns: |
| 245 List of tests that need rebaselining or |
| 246 None if there is no such test. |
| 247 """ |
| 248 |
| 249 self._rebaselining_tests = self._test_expectations.GetRebaseliningFailures() |
| 250 if not self._rebaselining_tests: |
| 251 logging.warn('No tests found that need rebaselining.') |
| 252 return None |
| 253 |
| 254 logging.info('Total number of tests needing rebaselining for "%s": "%d"', |
| 255 self._platform, len(self._rebaselining_tests)) |
| 256 |
| 257 test_no = 1 |
| 258 for test in self._rebaselining_tests: |
| 259 logging.info(' %d: %s', test_no, test) |
| 260 test_no += 1 |
| 261 |
| 262 return self._rebaselining_tests |
| 263 |
| 264 def _GetLatestRevision(self, url): |
| 265 """Get the latest layout test revision number from buildbot. |
| 266 |
| 267 Args: |
| 268 url: Url to retrieve layout test revision numbers. |
| 269 |
| 270 Returns: |
| 271 latest revision or |
| 272 None on failure. |
| 273 """ |
| 274 |
| 275 logging.debug('Url to retrieve revision: "%s"', url) |
| 276 |
| 277 f = urllib.urlopen(url) |
| 278 content = f.read() |
| 279 f.close() |
| 280 |
| 281 revisions = re.findall(self.REVISION_REGEX, content) |
| 282 if not revisions: |
| 283 logging.error('Failed to find revision, content: "%s"', content) |
| 284 return None |
| 285 |
| 286 revisions.sort(key=int) |
| 287 logging.info('Latest revision: "%s"', revisions[len(revisions) - 1]) |
| 288 return revisions[len(revisions) - 1] |
| 289 |
| 290 def _GetArchiveDirName(self, platform, webkit_canary): |
| 291 """Get name of the layout test archive directory. |
| 292 |
| 293 Returns: |
| 294 Directory name or |
| 295 None on failure |
| 296 """ |
| 297 |
| 298 if webkit_canary: |
| 299 platform += '-canary' |
| 300 |
| 301 if platform in ARCHIVE_DIR_NAME_DICT: |
| 302 return ARCHIVE_DIR_NAME_DICT[platform] |
| 303 else: |
| 304 logging.error('Cannot find platform key %s in archive directory name ' |
| 305 'dictionary', platform) |
| 306 return None |
| 307 |
| 308 def _GetArchiveUrl(self): |
| 309 """Generate the url to download latest layout test archive. |
| 310 |
| 311 Returns: |
| 312 Url to download archive or |
| 313 None on failure |
| 314 """ |
| 315 |
| 316 dir_name = self._GetArchiveDirName(self._platform, |
| 317 self._options.webkit_canary) |
| 318 if not dir_name: |
| 319 return None |
| 320 |
| 321 logging.debug('Buildbot platform dir name: "%s"', dir_name) |
| 322 |
| 323 url_base = '%s/%s/' % (self._options.archive_url, dir_name) |
| 324 latest_revision = self._GetLatestRevision(url_base) |
| 325 if latest_revision is None or latest_revision <= 0: |
| 326 return None |
| 327 |
| 328 archive_url = ('%s%s/layout-test-results.zip' % (url_base, |
| 329 latest_revision)) |
| 330 logging.info('Archive url: "%s"', archive_url) |
| 331 return archive_url |
| 332 |
| 333 def _DownloadBuildBotArchive(self): |
| 334 """Download layout test archive file from buildbot. |
| 335 |
| 336 Returns: |
| 337 True if download succeeded or |
| 338 False otherwise. |
| 339 """ |
| 340 |
| 341 url = self._GetArchiveUrl() |
| 342 if url is None: |
| 343 return None |
| 344 |
| 345 fn = urllib.urlretrieve(url)[0] |
| 346 logging.info('Archive downloaded and saved to file: "%s"', fn) |
| 347 return fn |
| 348 |
| 349 def _ExtractAndAddNewBaselines(self, archive_file): |
| 350 """Extract new baselines from archive and add them to SVN repository. |
| 351 |
| 352 Args: |
| 353 archive_file: full path to the archive file. |
| 354 |
| 355 Returns: |
| 356 List of tests that have been rebaselined or |
| 357 None on failure. |
| 358 """ |
| 359 |
| 360 zip_file = zipfile.ZipFile(archive_file, 'r') |
| 361 zip_namelist = zip_file.namelist() |
| 362 |
| 363 logging.debug('zip file namelist:') |
| 364 for name in zip_namelist: |
| 365 logging.debug(' ' + name) |
| 366 |
| 367 platform = path_utils.PlatformName(self._platform) |
| 368 logging.debug('Platform dir: "%s"', platform) |
| 369 |
| 370 test_no = 1 |
| 371 self._rebaselined_tests = [] |
| 372 for test in self._rebaselining_tests: |
| 373 logging.info('Test %d: %s', test_no, test) |
| 374 |
| 375 found = False |
| 376 svn_error = False |
| 377 test_basename = os.path.splitext(test)[0] |
| 378 for suffix in BASELINE_SUFFIXES: |
| 379 archive_test_name = 'layout-test-results/%s-actual%s' % (test_basename, |
| 380 suffix) |
| 381 logging.debug(' Archive test file name: "%s"', archive_test_name) |
| 382 if not archive_test_name in zip_namelist: |
| 383 logging.info(' %s file not in archive.', suffix) |
| 384 continue |
| 385 |
| 386 found = True |
| 387 logging.info(' %s file found in archive.', suffix) |
| 388 |
| 389 # Extract new baseline from archive and save it to a temp file. |
| 390 data = zip_file.read(archive_test_name) |
| 391 temp_fd, temp_name = tempfile.mkstemp(suffix) |
| 392 f = os.fdopen(temp_fd, 'wb') |
| 393 f.write(data) |
| 394 f.close() |
| 395 |
| 396 expected_filename = '%s-expected%s' % (test_basename, suffix) |
| 397 expected_fullpath = os.path.join( |
| 398 path_utils.ChromiumBaselinePath(platform), expected_filename) |
| 399 expected_fullpath = os.path.normpath(expected_fullpath) |
| 400 logging.debug(' Expected file full path: "%s"', expected_fullpath) |
| 401 |
| 402 # TODO(victorw): for now, the rebaselining tool checks whether |
| 403 # or not THIS baseline is duplicate and should be skipped. |
| 404 # We could improve the tool to check all baselines in upper and lower |
| 405 # levels and remove all duplicated baselines. |
| 406 if self._IsDupBaseline(temp_name, |
| 407 expected_fullpath, |
| 408 test, |
| 409 suffix, |
| 410 self._platform): |
| 411 os.remove(temp_name) |
| 412 self._DeleteBaseline(expected_fullpath) |
| 413 continue |
| 414 |
| 415 # Create the new baseline directory if it doesn't already exist. |
| 416 path_utils.MaybeMakeDirectory(os.path.dirname(expected_fullpath)) |
| 417 |
| 418 shutil.move(temp_name, expected_fullpath) |
| 419 |
| 420 if not self._SvnAdd(expected_fullpath): |
| 421 svn_error = True |
| 422 elif suffix != '.checksum': |
| 423 self._CreateHtmlBaselineFiles(expected_fullpath) |
| 424 |
| 425 if not found: |
| 426 logging.warn(' No new baselines found in archive.') |
| 427 else: |
| 428 if svn_error: |
| 429 logging.warn(' Failed to add baselines to SVN.') |
| 430 else: |
| 431 logging.info(' Rebaseline succeeded.') |
| 432 self._rebaselined_tests.append(test) |
| 433 |
| 434 test_no += 1 |
| 435 |
| 436 zip_file.close() |
| 437 os.remove(archive_file) |
| 438 |
| 439 return self._rebaselined_tests |
| 440 |
| 441 def _IsDupBaseline(self, new_baseline, baseline_path, test, suffix, platform): |
| 442 """Check whether a baseline is duplicate and can fallback to same |
| 443 baseline for another platform. For example, if a test has same baseline |
| 444 on linux and windows, then we only store windows baseline and linux |
| 445 baseline will fallback to the windows version. |
| 446 |
| 447 Args: |
| 448 expected_filename: baseline expectation file name. |
| 449 test: test name. |
| 450 suffix: file suffix of the expected results, including dot; e.g. '.txt' |
| 451 or '.png'. |
| 452 platform: baseline platform 'mac', 'win' or 'linux'. |
| 453 |
| 454 Returns: |
| 455 True if the baseline is unnecessary. |
| 456 False otherwise. |
| 457 """ |
| 458 test_filepath = os.path.join(path_utils.LayoutTestsDir(), test) |
| 459 all_baselines = path_utils.ExpectedBaselines(test_filepath, |
| 460 suffix, |
| 461 platform, |
| 462 True) |
| 463 for (fallback_dir, fallback_file) in all_baselines: |
| 464 if fallback_dir and fallback_file: |
| 465 fallback_fullpath = os.path.normpath( |
| 466 os.path.join(fallback_dir, fallback_file)) |
| 467 if fallback_fullpath.lower() != baseline_path.lower(): |
| 468 if not self._DiffBaselines(new_baseline, fallback_fullpath): |
| 469 logging.info(' Found same baseline at %s', fallback_fullpath) |
203 return True | 470 return True |
204 | 471 else: |
205 LogDashedString('Downloading archive', self._platform) | |
206 archive_file = self._DownloadBuildBotArchive() | |
207 logging.info('') | |
208 if not archive_file: | |
209 logging.error('No archive found.') | |
210 return False | 472 return False |
211 | 473 |
212 LogDashedString('Extracting and adding new baselines', self._platform) | 474 return False |
213 if not self._ExtractAndAddNewBaselines(archive_file): | 475 |
214 return False | 476 def _DiffBaselines(self, file1, file2): |
215 | 477 """Check whether two baselines are different. |
216 LogDashedString('Updating rebaselined tests in file', self._platform) | 478 |
217 self._UpdateRebaselinedTestsInFile(backup) | 479 Args: |
218 logging.info('') | 480 file1, file2: full paths of the baselines to compare. |
219 | 481 |
220 if len(self._rebaselining_tests) != len(self._rebaselined_tests): | 482 Returns: |
221 logging.warning('NOT ALL TESTS THAT NEED REBASELINING HAVE BEEN ' | 483 True if two files are different or have different extensions. |
222 'REBASELINED.') | 484 False otherwise. |
223 logging.warning(' Total tests needing rebaselining: %d', | 485 """ |
224 len(self._rebaselining_tests)) | 486 |
225 logging.warning(' Total tests rebaselined: %d', | 487 ext1 = os.path.splitext(file1)[1].upper() |
226 len(self._rebaselined_tests)) | 488 ext2 = os.path.splitext(file2)[1].upper() |
227 return False | 489 if ext1 != ext2: |
228 | 490 logging.warn('Files to compare have different ext. File1: %s; File2: %s', |
229 logging.warning('All tests needing rebaselining were successfully ' | 491 file1, file2) |
230 'rebaselined.') | 492 return True |
231 | 493 |
232 return True | 494 if ext1 == '.PNG': |
233 | 495 return image_diff.ImageDiff(self._platform, '').DiffFiles(file1, |
234 def GetRebaseliningTests(self): | 496 file2) |
235 return self._rebaselining_tests | 497 else: |
236 | 498 return text_diff.TestTextDiff(self._platform, '').DiffFiles(file1, |
237 def _GetRepoType(self): | 499 file2) |
238 """Get the repository type that client is using.""" | 500 |
239 | 501 def _DeleteBaseline(self, filename): |
240 output, return_code = RunShellWithReturnCode(['svn', 'info'], False) | 502 """Remove the file from repository and delete it from disk. |
241 if return_code == 0: | 503 |
242 return REPO_SVN | 504 Args: |
243 | 505 filename: full path of the file to delete. |
244 return REPO_UNKNOWN | 506 """ |
245 | 507 |
246 def _CompileRebaseliningTests(self): | 508 if not filename or not os.path.isfile(filename): |
247 """Compile list of tests that need rebaselining for the platform. | 509 return |
248 | 510 |
249 Returns: | 511 if self._repo_type == REPO_SVN: |
250 List of tests that need rebaselining or | 512 parent_dir, basename = os.path.split(filename) |
251 None if there is no such test. | 513 original_dir = os.getcwd() |
252 """ | 514 os.chdir(parent_dir) |
253 | 515 RunShell(['svn', 'delete', '--force', basename], False) |
254 self._rebaselining_tests = \ | 516 os.chdir(original_dir) |
255 self._test_expectations.GetRebaseliningFailures() | 517 else: |
256 if not self._rebaselining_tests: | 518 os.remove(filename) |
257 logging.warn('No tests found that need rebaselining.') | 519 |
258 return None | 520 def _UpdateRebaselinedTestsInFile(self, backup): |
259 | 521 """Update the rebaselined tests in test expectations file. |
260 logging.info('Total number of tests needing rebaselining ' | 522 |
261 'for "%s": "%d"', self._platform, | 523 Args: |
262 len(self._rebaselining_tests)) | 524 backup: if True, backup the original test expectations file. |
263 | 525 |
264 test_no = 1 | 526 Returns: |
265 for test in self._rebaselining_tests: | 527 no |
266 logging.info(' %d: %s', test_no, test) | 528 """ |
267 test_no += 1 | 529 |
268 | 530 if self._rebaselined_tests: |
269 return self._rebaselining_tests | 531 self._test_expectations.RemovePlatformFromFile(self._rebaselined_tests, |
270 | 532 self._platform, |
271 def _GetLatestRevision(self, url): | 533 backup) |
272 """Get the latest layout test revision number from buildbot. | 534 else: |
273 | 535 logging.info('No test was rebaselined so nothing to remove.') |
274 Args: | 536 |
275 url: Url to retrieve layout test revision numbers. | 537 def _SvnAdd(self, filename): |
276 | 538 """Add the file to SVN repository. |
277 Returns: | 539 |
278 latest revision or | 540 Args: |
279 None on failure. | 541 filename: full path of the file to add. |
280 """ | 542 |
281 | 543 Returns: |
282 logging.debug('Url to retrieve revision: "%s"', url) | 544 True if the file already exists in SVN or is sucessfully added to SVN. |
283 | 545 False otherwise. |
284 f = urllib.urlopen(url) | 546 """ |
285 content = f.read() | 547 |
286 f.close() | 548 if not filename: |
287 | 549 return False |
288 revisions = re.findall(self.REVISION_REGEX, content) | 550 |
289 if not revisions: | 551 parent_dir, basename = os.path.split(filename) |
290 logging.error('Failed to find revision, content: "%s"', content) | 552 if self._repo_type != REPO_SVN or parent_dir == filename: |
291 return None | 553 logging.info("No svn checkout found, skip svn add.") |
292 | 554 return True |
293 revisions.sort(key=int) | 555 |
294 logging.info('Latest revision: "%s"', revisions[len(revisions) - 1]) | 556 original_dir = os.getcwd() |
295 return revisions[len(revisions) - 1] | 557 os.chdir(parent_dir) |
296 | 558 status_output = RunShell(['svn', 'status', basename], False) |
297 def _GetArchiveDirName(self, platform, webkit_canary): | 559 os.chdir(original_dir) |
298 """Get name of the layout test archive directory. | 560 output = status_output.upper() |
299 | 561 if output.startswith('A') or output.startswith('M'): |
300 Returns: | 562 logging.info(' File already added to SVN: "%s"', filename) |
301 Directory name or | 563 return True |
302 None on failure | 564 |
303 """ | 565 if output.find('IS NOT A WORKING COPY') >= 0: |
304 | 566 logging.info(' File is not a working copy, add its parent: "%s"', |
305 if webkit_canary: | 567 parent_dir) |
306 platform += '-canary' | 568 return self._SvnAdd(parent_dir) |
307 | 569 |
308 if platform in ARCHIVE_DIR_NAME_DICT: | 570 os.chdir(parent_dir) |
309 return ARCHIVE_DIR_NAME_DICT[platform] | 571 add_output = RunShell(['svn', 'add', basename], True) |
310 else: | 572 os.chdir(original_dir) |
311 logging.error('Cannot find platform key %s in archive ' | 573 output = add_output.upper().rstrip() |
312 'directory name dictionary', platform) | 574 if output.startswith('A') and output.find(basename.upper()) >= 0: |
313 return None | 575 logging.info(' Added new file: "%s"', filename) |
314 | 576 self._SvnPropSet(filename) |
315 def _GetArchiveUrl(self): | 577 return True |
316 """Generate the url to download latest layout test archive. | 578 |
317 | 579 if (not status_output) and (add_output.upper().find( |
318 Returns: | 580 'ALREADY UNDER VERSION CONTROL') >= 0): |
319 Url to download archive or | 581 logging.info(' File already under SVN and has no change: "%s"', filename) |
320 None on failure | 582 return True |
321 """ | 583 |
322 | 584 logging.warn(' Failed to add file to SVN: "%s"', filename) |
323 dir_name = self._GetArchiveDirName(self._platform, | 585 logging.warn(' Svn status output: "%s"', status_output) |
324 self._options.webkit_canary) | 586 logging.warn(' Svn add output: "%s"', add_output) |
325 if not dir_name: | 587 return False |
326 return None | 588 |
327 | 589 def _SvnPropSet(self, filename): |
328 logging.debug('Buildbot platform dir name: "%s"', dir_name) | 590 """Set the baseline property |
329 | 591 |
330 url_base = '%s/%s/' % (self._options.archive_url, dir_name) | 592 Args: |
331 latest_revision = self._GetLatestRevision(url_base) | 593 filename: full path of the file to add. |
332 if latest_revision is None or latest_revision <= 0: | 594 |
333 return None | 595 Returns: |
334 | 596 True if the file already exists in SVN or is sucessfully added to SVN. |
335 archive_url = ('%s%s/layout-test-results.zip' % (url_base, | 597 False otherwise. |
336 latest_revision)) | 598 """ |
337 logging.info('Archive url: "%s"', archive_url) | 599 ext = os.path.splitext(filename)[1].upper() |
338 return archive_url | 600 if ext != '.TXT' and ext != '.PNG' and ext != '.CHECKSUM': |
339 | 601 return |
340 def _DownloadBuildBotArchive(self): | 602 |
341 """Download layout test archive file from buildbot. | 603 parent_dir, basename = os.path.split(filename) |
342 | 604 original_dir = os.getcwd() |
343 Returns: | 605 os.chdir(parent_dir) |
344 True if download succeeded or | 606 if ext == '.PNG': |
345 False otherwise. | 607 cmd = [ 'svn', 'pset', 'svn:mime-type', 'image/png', basename ] |
346 """ | 608 else: |
347 | 609 cmd = [ 'svn', 'pset', 'svn:eol-style', 'LF', basename ] |
348 url = self._GetArchiveUrl() | 610 |
349 if url is None: | 611 logging.debug(' Set svn prop: %s', ' '.join(cmd)) |
350 return None | 612 RunShell(cmd, False) |
351 | 613 os.chdir(original_dir) |
352 fn = urllib.urlretrieve(url)[0] | 614 |
353 logging.info('Archive downloaded and saved to file: "%s"', fn) | 615 def _CreateHtmlBaselineFiles(self, baseline_fullpath): |
354 return fn | 616 """Create baseline files (old, new and diff) in html directory. |
355 | 617 |
356 def _ExtractAndAddNewBaselines(self, archive_file): | 618 The files are used to compare the rebaselining results. |
357 """Extract new baselines from archive and add them to SVN repository. | 619 |
358 | 620 Args: |
359 Args: | 621 baseline_fullpath: full path of the expected baseline file. |
360 archive_file: full path to the archive file. | 622 """ |
361 | 623 |
362 Returns: | 624 if not baseline_fullpath or not os.path.exists(baseline_fullpath): |
363 List of tests that have been rebaselined or | 625 return |
364 None on failure. | 626 |
365 """ | 627 # Copy the new baseline to html directory for result comparison. |
366 | 628 baseline_filename = os.path.basename(baseline_fullpath) |
367 zip_file = zipfile.ZipFile(archive_file, 'r') | 629 new_file = GetResultFileFullpath(self._options.html_directory, |
368 zip_namelist = zip_file.namelist() | 630 baseline_filename, |
369 | 631 self._platform, |
370 logging.debug('zip file namelist:') | 632 'new') |
371 for name in zip_namelist: | 633 shutil.copyfile(baseline_fullpath, new_file) |
372 logging.debug(' ' + name) | 634 logging.info(' Html: copied new baseline file from "%s" to "%s".', |
373 | 635 baseline_fullpath, new_file) |
374 platform = path_utils.PlatformName(self._platform) | 636 |
375 logging.debug('Platform dir: "%s"', platform) | 637 # Get the old baseline from SVN and save to the html directory. |
376 | 638 output = RunShell(['svn', 'cat', '-r', 'BASE', baseline_fullpath]) |
377 test_no = 1 | 639 if (not output) or (output.upper().rstrip().endswith( |
378 self._rebaselined_tests = [] | 640 'NO SUCH FILE OR DIRECTORY')): |
379 for test in self._rebaselining_tests: | 641 logging.info(' No base file: "%s"', baseline_fullpath) |
380 logging.info('Test %d: %s', test_no, test) | 642 return |
381 | 643 base_file = GetResultFileFullpath(self._options.html_directory, |
382 found = False | 644 baseline_filename, |
383 svn_error = False | 645 self._platform, |
384 test_basename = os.path.splitext(test)[0] | 646 'old') |
385 for suffix in BASELINE_SUFFIXES: | 647 f = open(base_file, 'wb') |
386 archive_test_name = ('layout-test-results/%s-actual%s' % | 648 f.write(output) |
387 (test_basename, suffix)) | 649 f.close() |
388 logging.debug(' Archive test file name: "%s"', | 650 logging.info(' Html: created old baseline file: "%s".', |
389 archive_test_name) | 651 base_file) |
390 if not archive_test_name in zip_namelist: | 652 |
391 logging.info(' %s file not in archive.', suffix) | 653 # Get the diff between old and new baselines and save to the html directory. |
392 continue | 654 if baseline_filename.upper().endswith('.TXT'): |
393 | 655 # If the user specified a custom diff command in their svn config file, |
394 found = True | 656 # then it'll be used when we do svn diff, which we don't want to happen |
395 logging.info(' %s file found in archive.', suffix) | 657 # since we want the unified diff. Using --diff-cmd=diff doesn't always |
396 | 658 # work, since they can have another diff executable in their path that |
397 # Extract new baseline from archive and save it to a temp file. | 659 # gives different line endings. So we use a bogus temp directory as the |
398 data = zip_file.read(archive_test_name) | 660 # config directory, which gets around these problems. |
399 temp_fd, temp_name = tempfile.mkstemp(suffix) | 661 if sys.platform.startswith("win"): |
400 f = os.fdopen(temp_fd, 'wb') | 662 parent_dir = tempfile.gettempdir() |
401 f.write(data) | 663 else: |
402 f.close() | 664 parent_dir = sys.path[0] # tempdir is not secure. |
403 | 665 bogus_dir = os.path.join(parent_dir, "temp_svn_config") |
404 expected_filename = '%s-expected%s' % (test_basename, suffix) | 666 logging.debug(' Html: temp config dir: "%s".', bogus_dir) |
405 expected_fullpath = os.path.join( | 667 if not os.path.exists(bogus_dir): |
406 path_utils.ChromiumBaselinePath(platform), | 668 os.mkdir(bogus_dir) |
407 expected_filename) | 669 delete_bogus_dir = True |
408 expected_fullpath = os.path.normpath(expected_fullpath) | 670 else: |
409 logging.debug(' Expected file full path: "%s"', | 671 delete_bogus_dir = False |
410 expected_fullpath) | 672 |
411 | 673 output = RunShell(["svn", "diff", "--config-dir", bogus_dir, |
412 # TODO(victorw): for now, the rebaselining tool checks whether | 674 baseline_fullpath]) |
413 # or not THIS baseline is duplicate and should be skipped. | 675 if output: |
414 # We could improve the tool to check all baselines in upper | 676 diff_file = GetResultFileFullpath(self._options.html_directory, |
415 # and lower | |
416 # levels and remove all duplicated baselines. | |
417 if self._IsDupBaseline(temp_name, | |
418 expected_fullpath, | |
419 test, | |
420 suffix, | |
421 self._platform): | |
422 os.remove(temp_name) | |
423 self._DeleteBaseline(expected_fullpath) | |
424 continue | |
425 | |
426 # Create the new baseline directory if it doesn't already | |
427 # exist. | |
428 path_utils.MaybeMakeDirectory( | |
429 os.path.dirname(expected_fullpath)) | |
430 | |
431 shutil.move(temp_name, expected_fullpath) | |
432 | |
433 if not self._SvnAdd(expected_fullpath): | |
434 svn_error = True | |
435 elif suffix != '.checksum': | |
436 self._CreateHtmlBaselineFiles(expected_fullpath) | |
437 | |
438 if not found: | |
439 logging.warn(' No new baselines found in archive.') | |
440 else: | |
441 if svn_error: | |
442 logging.warn(' Failed to add baselines to SVN.') | |
443 else: | |
444 logging.info(' Rebaseline succeeded.') | |
445 self._rebaselined_tests.append(test) | |
446 | |
447 test_no += 1 | |
448 | |
449 zip_file.close() | |
450 os.remove(archive_file) | |
451 | |
452 return self._rebaselined_tests | |
453 | |
454 def _IsDupBaseline(self, new_baseline, baseline_path, test, suffix, | |
455 platform): | |
456 """Check whether a baseline is duplicate and can fallback to same | |
457 baseline for another platform. For example, if a test has same | |
458 baseline on linux and windows, then we only store windows | |
459 baseline and linux baseline will fallback to the windows version. | |
460 | |
461 Args: | |
462 expected_filename: baseline expectation file name. | |
463 test: test name. | |
464 suffix: file suffix of the expected results, including dot; | |
465 e.g. '.txt' or '.png'. | |
466 platform: baseline platform 'mac', 'win' or 'linux'. | |
467 | |
468 Returns: | |
469 True if the baseline is unnecessary. | |
470 False otherwise. | |
471 """ | |
472 test_filepath = os.path.join(path_utils.LayoutTestsDir(), test) | |
473 all_baselines = path_utils.ExpectedBaselines(test_filepath, | |
474 suffix, | |
475 platform, | |
476 True) | |
477 for (fallback_dir, fallback_file) in all_baselines: | |
478 if fallback_dir and fallback_file: | |
479 fallback_fullpath = os.path.normpath( | |
480 os.path.join(fallback_dir, fallback_file)) | |
481 if fallback_fullpath.lower() != baseline_path.lower(): | |
482 if not self._DiffBaselines(new_baseline, | |
483 fallback_fullpath): | |
484 logging.info(' Found same baseline at %s', | |
485 fallback_fullpath) | |
486 return True | |
487 else: | |
488 return False | |
489 | |
490 return False | |
491 | |
492 def _DiffBaselines(self, file1, file2): | |
493 """Check whether two baselines are different. | |
494 | |
495 Args: | |
496 file1, file2: full paths of the baselines to compare. | |
497 | |
498 Returns: | |
499 True if two files are different or have different extensions. | |
500 False otherwise. | |
501 """ | |
502 | |
503 ext1 = os.path.splitext(file1)[1].upper() | |
504 ext2 = os.path.splitext(file2)[1].upper() | |
505 if ext1 != ext2: | |
506 logging.warn('Files to compare have different ext. ' | |
507 'File1: %s; File2: %s', file1, file2) | |
508 return True | |
509 | |
510 if ext1 == '.PNG': | |
511 return image_diff.ImageDiff(self._platform, '').DiffFiles(file1, | |
512 file2) | |
513 else: | |
514 return text_diff.TestTextDiff(self._platform, '').DiffFiles(file1, | |
515 file2) | |
516 | |
517 def _DeleteBaseline(self, filename): | |
518 """Remove the file from repository and delete it from disk. | |
519 | |
520 Args: | |
521 filename: full path of the file to delete. | |
522 """ | |
523 | |
524 if not filename or not os.path.isfile(filename): | |
525 return | |
526 | |
527 if self._repo_type == REPO_SVN: | |
528 parent_dir, basename = os.path.split(filename) | |
529 original_dir = os.getcwd() | |
530 os.chdir(parent_dir) | |
531 RunShell(['svn', 'delete', '--force', basename], False) | |
532 os.chdir(original_dir) | |
533 else: | |
534 os.remove(filename) | |
535 | |
536 def _UpdateRebaselinedTestsInFile(self, backup): | |
537 """Update the rebaselined tests in test expectations file. | |
538 | |
539 Args: | |
540 backup: if True, backup the original test expectations file. | |
541 | |
542 Returns: | |
543 no | |
544 """ | |
545 | |
546 if self._rebaselined_tests: | |
547 self._test_expectations.RemovePlatformFromFile( | |
548 self._rebaselined_tests, self._platform, backup) | |
549 else: | |
550 logging.info('No test was rebaselined so nothing to remove.') | |
551 | |
552 def _SvnAdd(self, filename): | |
553 """Add the file to SVN repository. | |
554 | |
555 Args: | |
556 filename: full path of the file to add. | |
557 | |
558 Returns: | |
559 True if the file already exists in SVN or is sucessfully added | |
560 to SVN. | |
561 False otherwise. | |
562 """ | |
563 | |
564 if not filename: | |
565 return False | |
566 | |
567 parent_dir, basename = os.path.split(filename) | |
568 if self._repo_type != REPO_SVN or parent_dir == filename: | |
569 logging.info("No svn checkout found, skip svn add.") | |
570 return True | |
571 | |
572 original_dir = os.getcwd() | |
573 os.chdir(parent_dir) | |
574 status_output = RunShell(['svn', 'status', basename], False) | |
575 os.chdir(original_dir) | |
576 output = status_output.upper() | |
577 if output.startswith('A') or output.startswith('M'): | |
578 logging.info(' File already added to SVN: "%s"', filename) | |
579 return True | |
580 | |
581 if output.find('IS NOT A WORKING COPY') >= 0: | |
582 logging.info(' File is not a working copy, add its parent: "%s"', | |
583 parent_dir) | |
584 return self._SvnAdd(parent_dir) | |
585 | |
586 os.chdir(parent_dir) | |
587 add_output = RunShell(['svn', 'add', basename], True) | |
588 os.chdir(original_dir) | |
589 output = add_output.upper().rstrip() | |
590 if output.startswith('A') and output.find(basename.upper()) >= 0: | |
591 logging.info(' Added new file: "%s"', filename) | |
592 self._SvnPropSet(filename) | |
593 return True | |
594 | |
595 if (not status_output) and (add_output.upper().find( | |
596 'ALREADY UNDER VERSION CONTROL') >= 0): | |
597 logging.info(' File already under SVN and has no change: "%s"', | |
598 filename) | |
599 return True | |
600 | |
601 logging.warn(' Failed to add file to SVN: "%s"', filename) | |
602 logging.warn(' Svn status output: "%s"', status_output) | |
603 logging.warn(' Svn add output: "%s"', add_output) | |
604 return False | |
605 | |
606 def _SvnPropSet(self, filename): | |
607 """Set the baseline property | |
608 | |
609 Args: | |
610 filename: full path of the file to add. | |
611 | |
612 Returns: | |
613 True if the file already exists in SVN or is sucessfully added | |
614 to SVN. | |
615 False otherwise. | |
616 """ | |
617 ext = os.path.splitext(filename)[1].upper() | |
618 if ext != '.TXT' and ext != '.PNG' and ext != '.CHECKSUM': | |
619 return | |
620 | |
621 parent_dir, basename = os.path.split(filename) | |
622 original_dir = os.getcwd() | |
623 os.chdir(parent_dir) | |
624 if ext == '.PNG': | |
625 cmd = ['svn', 'pset', 'svn:mime-type', 'image/png', basename] | |
626 else: | |
627 cmd = ['svn', 'pset', 'svn:eol-style', 'LF', basename] | |
628 | |
629 logging.debug(' Set svn prop: %s', ' '.join(cmd)) | |
630 RunShell(cmd, False) | |
631 os.chdir(original_dir) | |
632 | |
633 def _CreateHtmlBaselineFiles(self, baseline_fullpath): | |
634 """Create baseline files (old, new and diff) in html directory. | |
635 | |
636 The files are used to compare the rebaselining results. | |
637 | |
638 Args: | |
639 baseline_fullpath: full path of the expected baseline file. | |
640 """ | |
641 | |
642 if not baseline_fullpath or not os.path.exists(baseline_fullpath): | |
643 return | |
644 | |
645 # Copy the new baseline to html directory for result comparison. | |
646 baseline_filename = os.path.basename(baseline_fullpath) | |
647 new_file = GetResultFileFullpath(self._options.html_directory, | |
648 baseline_filename, | |
649 self._platform, | |
650 'new') | |
651 shutil.copyfile(baseline_fullpath, new_file) | |
652 logging.info(' Html: copied new baseline file from "%s" to "%s".', | |
653 baseline_fullpath, new_file) | |
654 | |
655 # Get the old baseline from SVN and save to the html directory. | |
656 output = RunShell(['svn', 'cat', '-r', 'BASE', baseline_fullpath]) | |
657 if (not output) or (output.upper().rstrip().endswith( | |
658 'NO SUCH FILE OR DIRECTORY')): | |
659 logging.info(' No base file: "%s"', baseline_fullpath) | |
660 return | |
661 base_file = GetResultFileFullpath(self._options.html_directory, | |
662 baseline_filename, | 677 baseline_filename, |
663 self._platform, | 678 self._platform, |
664 'old') | 679 'diff') |
665 f = open(base_file, 'wb') | 680 f = open(diff_file, 'wb') |
666 f.write(output) | 681 f.write(output) |
667 f.close() | 682 f.close() |
668 logging.info(' Html: created old baseline file: "%s".', | 683 logging.info(' Html: created baseline diff file: "%s".', |
669 base_file) | 684 diff_file) |
670 | 685 |
671 # Get the diff between old and new baselines and save to the html dir. | 686 if delete_bogus_dir: |
672 if baseline_filename.upper().endswith('.TXT'): | 687 shutil.rmtree(bogus_dir, True) |
673 # If the user specified a custom diff command in their svn config | 688 logging.debug(' Html: removed temp config dir: "%s".', bogus_dir) |
674 # file, then it'll be used when we do svn diff, which we don't want | |
675 # to happen since we want the unified diff. Using --diff-cmd=diff | |
676 # doesn't always work, since they can have another diff executable | |
677 # in their path that gives different line endings. So we use a | |
678 # bogus temp directory as the config directory, which gets | |
679 # around these problems. | |
680 if sys.platform.startswith("win"): | |
681 parent_dir = tempfile.gettempdir() | |
682 else: | |
683 parent_dir = sys.path[0] # tempdir is not secure. | |
684 bogus_dir = os.path.join(parent_dir, "temp_svn_config") | |
685 logging.debug(' Html: temp config dir: "%s".', bogus_dir) | |
686 if not os.path.exists(bogus_dir): | |
687 os.mkdir(bogus_dir) | |
688 delete_bogus_dir = True | |
689 else: | |
690 delete_bogus_dir = False | |
691 | |
692 output = RunShell(["svn", "diff", "--config-dir", bogus_dir, | |
693 baseline_fullpath]) | |
694 if output: | |
695 diff_file = GetResultFileFullpath(self._options.html_directory, | |
696 baseline_filename, | |
697 self._platform, | |
698 'diff') | |
699 f = open(diff_file, 'wb') | |
700 f.write(output) | |
701 f.close() | |
702 logging.info(' Html: created baseline diff file: "%s".', | |
703 diff_file) | |
704 | |
705 if delete_bogus_dir: | |
706 shutil.rmtree(bogus_dir, True) | |
707 logging.debug(' Html: removed temp config dir: "%s".', | |
708 bogus_dir) | |
709 | |
710 | 689 |
711 class HtmlGenerator(object): | 690 class HtmlGenerator(object): |
712 """Class to generate rebaselining result comparison html.""" | 691 """Class to generate rebaselining result comparison html.""" |
713 | 692 |
714 HTML_REBASELINE = ('<html>' | 693 HTML_REBASELINE = ('<html>' |
715 '<head>' | 694 '<head>' |
716 '<style>' | 695 '<style>' |
717 'body {font-family: sans-serif;}' | 696 'body {font-family: sans-serif;}' |
718 '.mainTable {background: #666666;}' | 697 '.mainTable {background: #666666;}' |
719 '.mainTable td , .mainTable th {background: white;}' | 698 '.mainTable td , .mainTable th {background: white;}' |
720 '.detail {margin-left: 10px; margin-top: 3px;}' | 699 '.detail {margin-left: 10px; margin-top: 3px;}' |
721 '</style>' | 700 '</style>' |
722 '<title>Rebaselining Result Comparison (%(time)s)' | 701 '<title>Rebaselining Result Comparison (%(time)s)</title>' |
723 '</title>' | 702 '</head>' |
724 '</head>' | 703 '<body>' |
725 '<body>' | 704 '<h2>Rebaselining Result Comparison (%(time)s)</h2>' |
726 '<h2>Rebaselining Result Comparison (%(time)s)</h2>' | 705 '%(body)s' |
727 '%(body)s' | 706 '</body>' |
728 '</body>' | 707 '</html>') |
729 '</html>') | 708 HTML_NO_REBASELINING_TESTS = '<p>No tests found that need rebaselining.</p>' |
730 HTML_NO_REBASELINING_TESTS = ( | 709 HTML_TABLE_TEST = ('<table class="mainTable" cellspacing=1 cellpadding=5>' |
731 '<p>No tests found that need rebaselining.</p>') | 710 '%s</table><br>') |
732 HTML_TABLE_TEST = ('<table class="mainTable" cellspacing=1 cellpadding=5>' | 711 HTML_TR_TEST = ('<tr>' |
733 '%s</table><br>') | 712 '<th style="background-color: #CDECDE; border-bottom: ' |
734 HTML_TR_TEST = ('<tr>' | 713 '1px solid black; font-size: 18pt; font-weight: bold" ' |
735 '<th style="background-color: #CDECDE; border-bottom: ' | 714 'colspan="5">' |
736 '1px solid black; font-size: 18pt; font-weight: bold" ' | 715 '<a href="%s">%s</a>' |
737 'colspan="5">' | 716 '</th>' |
738 '<a href="%s">%s</a>' | 717 '</tr>') |
739 '</th>' | 718 HTML_TEST_DETAIL = ('<div class="detail">' |
740 '</tr>') | 719 '<tr>' |
741 HTML_TEST_DETAIL = ('<div class="detail">' | 720 '<th width="100">Baseline</th>' |
742 '<tr>' | 721 '<th width="100">Platform</th>' |
743 '<th width="100">Baseline</th>' | 722 '<th width="200">Old</th>' |
744 '<th width="100">Platform</th>' | 723 '<th width="200">New</th>' |
745 '<th width="200">Old</th>' | 724 '<th width="150">Difference</th>' |
746 '<th width="200">New</th>' | 725 '</tr>' |
747 '<th width="150">Difference</th>' | 726 '%s' |
748 '</tr>' | 727 '</div>') |
749 '%s' | 728 HTML_TD_NOLINK = '<td align=center><a>%s</a></td>' |
750 '</div>') | 729 HTML_TD_LINK = '<td align=center><a href="%(uri)s">%(name)s</a></td>' |
751 HTML_TD_NOLINK = '<td align=center><a>%s</a></td>' | 730 HTML_TD_LINK_IMG = ('<td><a href="%(uri)s">' |
752 HTML_TD_LINK = '<td align=center><a href="%(uri)s">%(name)s</a></td>' | 731 '<img style="width: 200" src="%(uri)s" /></a></td>') |
753 HTML_TD_LINK_IMG = ('<td><a href="%(uri)s">' | 732 HTML_TR = '<tr>%s</tr>' |
754 '<img style="width: 200" src="%(uri)s" /></a></td>') | 733 |
755 HTML_TR = '<tr>%s</tr>' | 734 def __init__(self, options, platforms, rebaselining_tests): |
756 | 735 self._html_directory = options.html_directory |
757 def __init__(self, options, platforms, rebaselining_tests): | 736 self._platforms = platforms |
758 self._html_directory = options.html_directory | 737 self._rebaselining_tests = rebaselining_tests |
759 self._platforms = platforms | 738 self._html_file = os.path.join(options.html_directory, 'rebaseline.html') |
760 self._rebaselining_tests = rebaselining_tests | 739 |
761 self._html_file = os.path.join(options.html_directory, | 740 def GenerateHtml(self): |
762 'rebaseline.html') | 741 """Generate html file for rebaselining result comparison.""" |
763 | 742 |
764 def GenerateHtml(self): | 743 logging.info('Generating html file') |
765 """Generate html file for rebaselining result comparison.""" | 744 |
766 | 745 html_body = '' |
767 logging.info('Generating html file') | 746 if not self._rebaselining_tests: |
768 | 747 html_body += self.HTML_NO_REBASELINING_TESTS |
769 html_body = '' | 748 else: |
770 if not self._rebaselining_tests: | 749 tests = list(self._rebaselining_tests) |
771 html_body += self.HTML_NO_REBASELINING_TESTS | 750 tests.sort() |
772 else: | 751 |
773 tests = list(self._rebaselining_tests) | 752 test_no = 1 |
774 tests.sort() | 753 for test in tests: |
775 | 754 logging.info('Test %d: %s', test_no, test) |
776 test_no = 1 | 755 html_body += self._GenerateHtmlForOneTest(test) |
777 for test in tests: | 756 |
778 logging.info('Test %d: %s', test_no, test) | 757 html = self.HTML_REBASELINE % ({'time': time.asctime(), 'body': html_body}) |
779 html_body += self._GenerateHtmlForOneTest(test) | 758 logging.debug(html) |
780 | 759 |
781 html = self.HTML_REBASELINE % ({'time': time.asctime(), | 760 f = open(self._html_file, 'w') |
782 'body': html_body}) | 761 f.write(html) |
783 logging.debug(html) | 762 f.close() |
784 | 763 |
785 f = open(self._html_file, 'w') | 764 logging.info('Baseline comparison html generated at "%s"', |
786 f.write(html) | 765 self._html_file) |
787 f.close() | 766 |
788 | 767 def ShowHtml(self): |
789 logging.info('Baseline comparison html generated at "%s"', | 768 """Launch the rebaselining html in brwoser.""" |
790 self._html_file) | 769 |
791 | 770 logging.info('Launching html: "%s"', self._html_file) |
792 def ShowHtml(self): | 771 |
793 """Launch the rebaselining html in brwoser.""" | 772 html_uri = path_utils.FilenameToUri(self._html_file) |
794 | 773 webbrowser.open(html_uri, 1) |
795 logging.info('Launching html: "%s"', self._html_file) | 774 |
796 | 775 logging.info('Html launched.') |
797 html_uri = path_utils.FilenameToUri(self._html_file) | 776 |
798 webbrowser.open(html_uri, 1) | 777 def _GenerateBaselineLinks(self, test_basename, suffix, platform): |
799 | 778 """Generate links for baseline results (old, new and diff). |
800 logging.info('Html launched.') | 779 |
801 | 780 Args: |
802 def _GenerateBaselineLinks(self, test_basename, suffix, platform): | 781 test_basename: base filename of the test |
803 """Generate links for baseline results (old, new and diff). | 782 suffix: baseline file suffixes: '.txt', '.png' |
804 | 783 platform: win, linux or mac |
805 Args: | 784 |
806 test_basename: base filename of the test | 785 Returns: |
807 suffix: baseline file suffixes: '.txt', '.png' | 786 html links for showing baseline results (old, new and diff) |
808 platform: win, linux or mac | 787 """ |
809 | 788 |
810 Returns: | 789 baseline_filename = '%s-expected%s' % (test_basename, suffix) |
811 html links for showing baseline results (old, new and diff) | 790 logging.debug(' baseline filename: "%s"', baseline_filename) |
812 """ | 791 |
813 | 792 new_file = GetResultFileFullpath(self._html_directory, |
814 baseline_filename = '%s-expected%s' % (test_basename, suffix) | 793 baseline_filename, |
815 logging.debug(' baseline filename: "%s"', baseline_filename) | 794 platform, |
816 | 795 'new') |
817 new_file = GetResultFileFullpath(self._html_directory, | 796 logging.info(' New baseline file: "%s"', new_file) |
818 baseline_filename, | 797 if not os.path.exists(new_file): |
819 platform, | 798 logging.info(' No new baseline file: "%s"', new_file) |
820 'new') | 799 return '' |
821 logging.info(' New baseline file: "%s"', new_file) | 800 |
822 if not os.path.exists(new_file): | 801 old_file = GetResultFileFullpath(self._html_directory, |
823 logging.info(' No new baseline file: "%s"', new_file) | 802 baseline_filename, |
824 return '' | 803 platform, |
825 | 804 'old') |
826 old_file = GetResultFileFullpath(self._html_directory, | 805 logging.info(' Old baseline file: "%s"', old_file) |
827 baseline_filename, | 806 if suffix == '.png': |
828 platform, | 807 html_td_link = self.HTML_TD_LINK_IMG |
829 'old') | 808 else: |
830 logging.info(' Old baseline file: "%s"', old_file) | 809 html_td_link = self.HTML_TD_LINK |
831 if suffix == '.png': | 810 |
832 html_td_link = self.HTML_TD_LINK_IMG | 811 links = '' |
833 else: | 812 if os.path.exists(old_file): |
834 html_td_link = self.HTML_TD_LINK | 813 links += html_td_link % {'uri': path_utils.FilenameToUri(old_file), |
835 | 814 'name': baseline_filename} |
836 links = '' | 815 else: |
837 if os.path.exists(old_file): | 816 logging.info(' No old baseline file: "%s"', old_file) |
838 links += html_td_link % {'uri': path_utils.FilenameToUri(old_file), | 817 links += self.HTML_TD_NOLINK % '' |
839 'name': baseline_filename} | 818 |
840 else: | 819 links += html_td_link % {'uri': path_utils.FilenameToUri(new_file), |
841 logging.info(' No old baseline file: "%s"', old_file) | 820 'name': baseline_filename} |
842 links += self.HTML_TD_NOLINK % '' | 821 |
843 | 822 diff_file = GetResultFileFullpath(self._html_directory, |
844 links += html_td_link % {'uri': path_utils.FilenameToUri(new_file), | 823 baseline_filename, |
845 'name': baseline_filename} | 824 platform, |
846 | 825 'diff') |
847 diff_file = GetResultFileFullpath(self._html_directory, | 826 logging.info(' Baseline diff file: "%s"', diff_file) |
848 baseline_filename, | 827 if os.path.exists(diff_file): |
849 platform, | 828 links += html_td_link % {'uri': path_utils.FilenameToUri(diff_file), |
850 'diff') | 829 'name': 'Diff'} |
851 logging.info(' Baseline diff file: "%s"', diff_file) | 830 else: |
852 if os.path.exists(diff_file): | 831 logging.info(' No baseline diff file: "%s"', diff_file) |
853 links += html_td_link % {'uri': path_utils.FilenameToUri( | 832 links += self.HTML_TD_NOLINK % '' |
854 diff_file), 'name': 'Diff'} | 833 |
855 else: | 834 return links |
856 logging.info(' No baseline diff file: "%s"', diff_file) | 835 |
857 links += self.HTML_TD_NOLINK % '' | 836 def _GenerateHtmlForOneTest(self, test): |
858 | 837 """Generate html for one rebaselining test. |
859 return links | 838 |
860 | 839 Args: |
861 def _GenerateHtmlForOneTest(self, test): | 840 test: layout test name |
862 """Generate html for one rebaselining test. | 841 |
863 | 842 Returns: |
864 Args: | 843 html that compares baseline results for the test. |
865 test: layout test name | 844 """ |
866 | 845 |
867 Returns: | 846 test_basename = os.path.basename(os.path.splitext(test)[0]) |
868 html that compares baseline results for the test. | 847 logging.info(' basename: "%s"', test_basename) |
869 """ | 848 rows = [] |
870 | 849 for suffix in BASELINE_SUFFIXES: |
871 test_basename = os.path.basename(os.path.splitext(test)[0]) | 850 if suffix == '.checksum': |
872 logging.info(' basename: "%s"', test_basename) | 851 continue |
873 rows = [] | 852 |
874 for suffix in BASELINE_SUFFIXES: | 853 logging.info(' Checking %s files', suffix) |
875 if suffix == '.checksum': | 854 for platform in self._platforms: |
876 continue | 855 links = self._GenerateBaselineLinks(test_basename, suffix, platform) |
877 | 856 if links: |
878 logging.info(' Checking %s files', suffix) | 857 row = self.HTML_TD_NOLINK % self._GetBaselineResultType(suffix) |
879 for platform in self._platforms: | 858 row += self.HTML_TD_NOLINK % platform |
880 links = self._GenerateBaselineLinks(test_basename, suffix, | 859 row += links |
881 platform) | 860 logging.debug(' html row: %s', row) |
882 if links: | 861 |
883 row = self.HTML_TD_NOLINK % self._GetBaselineResultType( | 862 rows.append(self.HTML_TR % row) |
884 suffix) | 863 |
885 row += self.HTML_TD_NOLINK % platform | 864 if rows: |
886 row += links | 865 test_path = os.path.join(path_utils.LayoutTestsDir(), test) |
887 logging.debug(' html row: %s', row) | 866 html = self.HTML_TR_TEST % (path_utils.FilenameToUri(test_path), test) |
888 | 867 html += self.HTML_TEST_DETAIL % ' '.join(rows) |
889 rows.append(self.HTML_TR % row) | 868 |
890 | 869 logging.debug(' html for test: %s', html) |
891 if rows: | 870 return self.HTML_TABLE_TEST % html |
892 test_path = os.path.join(path_utils.LayoutTestsDir(), test) | 871 |
893 html = self.HTML_TR_TEST % (path_utils.FilenameToUri(test_path), | 872 return '' |
894 test) | 873 |
895 html += self.HTML_TEST_DETAIL % ' '.join(rows) | 874 def _GetBaselineResultType(self, suffix): |
896 | 875 """Name of the baseline result type.""" |
897 logging.debug(' html for test: %s', html) | 876 |
898 return self.HTML_TABLE_TEST % html | 877 if suffix == '.png': |
899 | 878 return 'Pixel' |
900 return '' | 879 elif suffix == '.txt': |
901 | 880 return 'Render Tree' |
902 def _GetBaselineResultType(self, suffix): | 881 else: |
903 """Name of the baseline result type.""" | 882 return 'Other' |
904 | |
905 if suffix == '.png': | |
906 return 'Pixel' | |
907 elif suffix == '.txt': | |
908 return 'Render Tree' | |
909 else: | |
910 return 'Other' | |
911 | 883 |
912 | 884 |
913 def main(): | 885 def main(): |
914 """Main function to produce new baselines.""" | 886 """Main function to produce new baselines.""" |
915 | 887 |
916 option_parser = optparse.OptionParser() | 888 option_parser = optparse.OptionParser() |
917 option_parser.add_option('-v', '--verbose', | 889 option_parser.add_option('-v', '--verbose', |
918 action='store_true', | 890 action='store_true', |
919 default=False, | 891 default=False, |
920 help='include debug-level logging.') | 892 help='include debug-level logging.') |
921 | 893 |
922 option_parser.add_option('-p', '--platforms', | 894 option_parser.add_option('-p', '--platforms', |
923 default='mac,win,win-xp,win-vista,linux', | 895 default='mac,win,win-xp,win-vista,linux', |
924 help=('Comma delimited list of platforms ' | 896 help=('Comma delimited list of platforms that need ' |
925 'that need rebaselining.')) | 897 'rebaselining.')) |
926 | 898 |
927 option_parser.add_option('-u', '--archive_url', | 899 option_parser.add_option('-u', '--archive_url', |
928 default=('http://build.chromium.org/buildbot/' | 900 default=('http://build.chromium.org/buildbot/' |
929 'layout_test_results'), | 901 'layout_test_results'), |
930 help=('Url to find the layout test result archive' | 902 help=('Url to find the layout test result archive ' |
931 ' file.')) | 903 'file.')) |
932 | 904 |
933 option_parser.add_option('-w', '--webkit_canary', | 905 option_parser.add_option('-w', '--webkit_canary', |
934 action='store_true', | 906 action='store_true', |
935 default=False, | 907 default=False, |
936 help=('If True, pull baselines from webkit.org ' | 908 help=('If True, pull baselines from webkit.org ' |
937 'canary bot.')) | 909 'canary bot.')) |
938 | 910 |
939 option_parser.add_option('-b', '--backup', | 911 option_parser.add_option('-b', '--backup', |
940 action='store_true', | 912 action='store_true', |
941 default=False, | 913 default=False, |
942 help=('Whether or not to backup the original test' | 914 help=('Whether or not to backup the original test ' |
943 ' expectations file after rebaseline.')) | 915 'expectations file after rebaseline.')) |
944 | 916 |
945 option_parser.add_option('-d', '--html_directory', | 917 option_parser.add_option('-d', '--html_directory', |
946 default='', | 918 default='', |
947 help=('The directory that stores the results for' | 919 help=('The directory that stores the results for ' |
948 ' rebaselining comparison.')) | 920 'rebaselining comparison.')) |
949 | 921 |
950 options = option_parser.parse_args()[0] | 922 options = option_parser.parse_args()[0] |
951 | 923 |
952 # Set up our logging format. | 924 # Set up our logging format. |
953 log_level = logging.INFO | 925 log_level = logging.INFO |
954 if options.verbose: | 926 if options.verbose: |
955 log_level = logging.DEBUG | 927 log_level = logging.DEBUG |
956 logging.basicConfig(level=log_level, | 928 logging.basicConfig(level=log_level, |
957 format=('%(asctime)s %(filename)s:%(lineno)-3d ' | 929 format=('%(asctime)s %(filename)s:%(lineno)-3d ' |
958 '%(levelname)s %(message)s'), | 930 '%(levelname)s %(message)s'), |
959 datefmt='%y%m%d %H:%M:%S') | 931 datefmt='%y%m%d %H:%M:%S') |
960 | 932 |
961 # Verify 'platforms' option is valid | 933 # Verify 'platforms' option is valid |
962 if not options.platforms: | 934 if not options.platforms: |
963 logging.error('Invalid "platforms" option. --platforms must be ' | 935 logging.error('Invalid "platforms" option. --platforms must be specified ' |
964 'specified in order to rebaseline.') | 936 'in order to rebaseline.') |
965 sys.exit(1) | 937 sys.exit(1) |
966 platforms = [p.strip().lower() for p in options.platforms.split(',')] | 938 platforms = [p.strip().lower() for p in options.platforms.split(',')] |
967 for platform in platforms: | 939 for platform in platforms: |
968 if not platform in REBASELINE_PLATFORM_ORDER: | 940 if not platform in REBASELINE_PLATFORM_ORDER: |
969 logging.error('Invalid platform: "%s"' % (platform)) | 941 logging.error('Invalid platform: "%s"' % (platform)) |
970 sys.exit(1) | 942 sys.exit(1) |
971 | 943 |
972 # Adjust the platform order so rebaseline tool is running at the order of | 944 # Adjust the platform order so rebaseline tool is running at the order of |
973 # 'mac', 'win' and 'linux'. This is in same order with layout test baseline | 945 # 'mac', 'win' and 'linux'. This is in same order with layout test baseline |
974 # search paths. It simplifies how the rebaseline tool detects duplicate | 946 # search paths. It simplifies how the rebaseline tool detects duplicate |
975 # baselines. Check _IsDupBaseline method for details. | 947 # baselines. Check _IsDupBaseline method for details. |
976 rebaseline_platforms = [] | 948 rebaseline_platforms = [] |
977 for platform in REBASELINE_PLATFORM_ORDER: | 949 for platform in REBASELINE_PLATFORM_ORDER: |
978 if platform in platforms: | 950 if platform in platforms: |
979 rebaseline_platforms.append(platform) | 951 rebaseline_platforms.append(platform) |
980 | 952 |
981 options.html_directory = SetupHtmlDirectory(options.html_directory) | 953 options.html_directory = SetupHtmlDirectory(options.html_directory) |
982 | 954 |
983 rebaselining_tests = set() | 955 rebaselining_tests = set() |
984 backup = options.backup | 956 backup = options.backup |
985 for platform in rebaseline_platforms: | 957 for platform in rebaseline_platforms: |
986 rebaseliner = Rebaseliner(platform, options) | 958 rebaseliner = Rebaseliner(platform, options) |
987 | |
988 logging.info('') | |
989 LogDashedString('Rebaseline started', platform) | |
990 if rebaseliner.Run(backup): | |
991 # Only need to backup one original copy of test expectation file. | |
992 backup = False | |
993 LogDashedString('Rebaseline done', platform) | |
994 else: | |
995 LogDashedString('Rebaseline failed', platform, logging.ERROR) | |
996 | |
997 rebaselining_tests |= set(rebaseliner.GetRebaseliningTests()) | |
998 | 959 |
999 logging.info('') | 960 logging.info('') |
1000 LogDashedString('Rebaselining result comparison started', None) | 961 LogDashedString('Rebaseline started', platform) |
1001 html_generator = HtmlGenerator(options, | 962 if rebaseliner.Run(backup): |
1002 rebaseline_platforms, | 963 # Only need to backup one original copy of test expectation file. |
1003 rebaselining_tests) | 964 backup = False |
1004 html_generator.GenerateHtml() | 965 LogDashedString('Rebaseline done', platform) |
1005 html_generator.ShowHtml() | 966 else: |
1006 LogDashedString('Rebaselining result comparison done', None) | 967 LogDashedString('Rebaseline failed', platform, logging.ERROR) |
1007 | 968 |
1008 sys.exit(0) | 969 rebaselining_tests |= set(rebaseliner.GetRebaseliningTests()) |
| 970 |
| 971 logging.info('') |
| 972 LogDashedString('Rebaselining result comparison started', None) |
| 973 html_generator = HtmlGenerator(options, |
| 974 rebaseline_platforms, |
| 975 rebaselining_tests) |
| 976 html_generator.GenerateHtml() |
| 977 html_generator.ShowHtml() |
| 978 LogDashedString('Rebaselining result comparison done', None) |
| 979 |
| 980 sys.exit(0) |
1009 | 981 |
1010 if '__main__' == __name__: | 982 if '__main__' == __name__: |
1011 main() | 983 main() |
OLD | NEW |