| OLD | NEW |
|---|
| (Empty) |
| 1 #!/bin/env/python | |
| 2 # Copyright (c) 2006-2009 The Chromium Authors. All rights reserved. | |
| 3 # Use of this source code is governed by a BSD-style license that can be | |
| 4 # found in the LICENSE file. | |
| 5 | |
| 6 import os | |
| 7 import zipfile | |
| 8 | |
| 9 from failure_finder import FailureFinder | |
| 10 | |
| 11 TEST_BUILDER_OUTPUT = """090723 10:38:22 test_shell_thread.py:289 | |
| 12 ERROR chrome/fast/forms/textarea-metrics.html failed: | |
| 13 Text diff mismatch | |
| 14 090723 10:38:21 test_shell_thread.py:289 | |
| 15 ERROR chrome/fast/dom/xss-DENIED-javascript-variations.html failed: | |
| 16 Text diff mismatch | |
| 17 090723 10:37:58 test_shell_thread.py:289 | |
| 18 ERROR LayoutTests/plugins/bindings-test.html failed: | |
| 19 Text diff mismatch | |
| 20 | |
| 21 ------------------------------------------------------------------------------ | |
| 22 Expected to crash, but passed (1): | |
| 23 chrome/fast/forms/textarea-metrics.html | |
| 24 | |
| 25 Regressions: Unexpected failures (2): | |
| 26 chrome/fast/dom/xss-DENIED-javascript-variations.html = FAIL | |
| 27 LayoutTests/plugins/bindings-test.html = FAIL | |
| 28 ------------------------------------------------------------------------------ | |
| 29 """ | |
| 30 | |
| 31 TEST_FAILURE_1 = ("layout-test-results/chrome/fast/forms/" | |
| 32 "textarea-metrics-actual.txt") | |
| 33 TEST_FAILURE_2 = ("layout-test-results/chrome/fast/dom/" | |
| 34 "xss-DENIED-javascript-variations-actual.txt") | |
| 35 TEST_FAILURE_3 = ("layout-test-results/LayoutTests/plugins/" | |
| 36 "bindings-test-actual.txt") | |
| 37 | |
| 38 TEST_ARCHIVE_OUTPUT = """ | |
| 39 Adding layout-test-results\pending\fast\repaint\not-real-actual.checksum | |
| 40 Adding layout-test-results\pending\fast\repaint\not-real-actual.png | |
| 41 Adding layout-test-results\pending\fast\repaint\not-real-actual.txt | |
| 42 last change: 22057 | |
| 43 build name: webkit-rel | |
| 44 host name: codf138 | |
| 45 saving results to \\my\test\location\webkit-rel\22057 | |
| 46 program finished with exit code 0 | |
| 47 """ | |
| 48 | |
| 49 TEST_TEST_EXPECTATIONS = """ | |
| 50 BUG1234 chrome/fast/forms/textarea-metrics.html = CRASH | |
| 51 """ | |
| 52 | |
| 53 TEST_BUILDER_LOG_FILE = "TEST_builder.log" | |
| 54 TEST_ARCHIVE_LOG_FILE = "TEST_archive.log" | |
| 55 TEST_DUMMY_ZIP_FILE = "TEST_zipfile.zip" | |
| 56 TEST_EXPECTATIONS_FILE = "TEST_expectations.txt" | |
| 57 | |
| 58 WEBKIT_BUILDER_NUMBER = "9800" | |
| 59 WEBKIT_FAILURES = ( | |
| 60 ["LayoutTests/fast/backgrounds/animated-svg-as-mask.html", | |
| 61 "LayoutTests/fast/backgrounds/background-clip-text.html", | |
| 62 "LayoutTests/fast/backgrounds/mask-composite.html", | |
| 63 "LayoutTests/fast/backgrounds/repeat/mask-negative-offset-repeat.html", | |
| 64 "LayoutTests/fast/backgrounds/svg-as-background-3.html", | |
| 65 "LayoutTests/fast/backgrounds/svg-as-background-6.html", | |
| 66 "LayoutTests/fast/backgrounds/svg-as-mask.html", | |
| 67 "LayoutTests/fast/block/float/013.html", | |
| 68 "LayoutTests/fast/block/float/nested-clearance.html", | |
| 69 "LayoutTests/fast/block/positioning/047.html"]) | |
| 70 | |
| 71 CHROMIUM_BASELINE = "chrome/fast/forms/basic-buttons.html" | |
| 72 EXPECTED_CHROMIUM_LOCAL_BASELINE = "./chrome/fast/forms/basic-buttons.html" | |
| 73 EXPECTED_CHROMIUM_URL_BASELINE = ("http://src.chromium.org/viewvc/chrome/" | |
| 74 "trunk/src/webkit/data/layout_tests/chrome/" | |
| 75 "fast/forms/basic-buttons.html") | |
| 76 | |
| 77 WEBKIT_BASELINE = "LayoutTests/fast/forms/11423.html" | |
| 78 EXPECTED_WEBKIT_LOCAL_BASELINE = "./LayoutTests/fast/forms/11423.html" | |
| 79 EXPECTED_WEBKIT_URL_BASELINE = ( | |
| 80 "http://svn.webkit.org/repository/webkit/trunk/" | |
| 81 "LayoutTests/fast/forms/11423.html") | |
| 82 | |
| 83 TEST_ZIP_FILE = ("http://build.chromium.org/buildbot/layout_test_results/" | |
| 84 "webkit-rel/21432/layout-test-results.zip") | |
| 85 | |
| 86 EXPECTED_REVISION = "20861" | |
| 87 EXPECTED_BUILD_NAME = "webkit-rel" | |
| 88 | |
| 89 SVG_TEST_EXPECTATION = ( | |
| 90 "LayoutTests/svg/custom/foreign-object-skew-expected.png") | |
| 91 SVG_TEST_EXPECTATION_UPSTREAM = ("LayoutTests/svg/custom/" | |
| 92 "foreign-object-skew-expected-upstream.png") | |
| 93 WEBARCHIVE_TEST_EXPECTATION = ("LayoutTests/webarchive/adopt-attribute-" | |
| 94 "styled-body-webarchive-expected.webarchive") | |
| 95 DOM_TEST_EXPECTATION = ("LayoutTests/fast/dom/" | |
| 96 "attribute-downcast-right-expected.txt") | |
| 97 DOM_TEST_EXPECTATION_UPSTREAM = ("LayoutTests/fast/dom/" | |
| 98 "attribute-downcast-right-" | |
| 99 "expected-upstream.png") | |
| 100 | |
| 101 TEST_EXPECTATIONS = """ | |
| 102 BUG1234 WONTFIX : LayoutTests/fast/backgrounds/svg-as-background-3.html = FAIL | |
| 103 BUG3456 WIN : LayoutTests/fast/backgrounds/svg-as-background-6.html = CRASH | |
| 104 BUG4567 : LayoutTests/fast/backgrounds/svg-as-mask.html = PASS | |
| 105 WONTFIX : LayoutTests/fast/block/ = FAIL | |
| 106 """ | |
| 107 | |
| 108 EXPECT_EXACT_MATCH = "LayoutTests/fast/backgrounds/svg-as-background-6.html" | |
| 109 EXPECT_GENERAL_MATCH = "LayoutTests/fast/block/float/013.html" | |
| 110 EXPECT_NO_MATCH = "LayoutTests/fast/backgrounds/svg-as-background-99.html" | |
| 111 | |
| 112 WEBKIT_ORG = "webkit.org" | |
| 113 CHROMIUM_ORG = "chromium.org" | |
| 114 | |
| 115 | |
| 116 class FailureFinderTest(object): | |
| 117 | |
| 118 def runTests(self): | |
| 119 all_tests_passed = True | |
| 120 | |
| 121 tests = ["testWhitespaceInBuilderName", | |
| 122 "testGetLastBuild", | |
| 123 "testFindMatchesInBuilderOutput", | |
| 124 "testScrapeBuilderOutput", | |
| 125 "testGetChromiumBaseline", | |
| 126 "testGetWebkitBaseline", | |
| 127 "testZipDownload", | |
| 128 "testUseLocalOutput", | |
| 129 "testTranslateBuildToZip", | |
| 130 "testGetBaseline", | |
| 131 "testFindTestExpectations", | |
| 132 "testFull"] | |
| 133 | |
| 134 for test in tests: | |
| 135 try: | |
| 136 result = eval(test + "()") | |
| 137 if result: | |
| 138 print "[ OK ] %s" % test | |
| 139 else: | |
| 140 all_tests_passed = False | |
| 141 print "[ FAIL ] %s" % test | |
| 142 except: | |
| 143 print "[ ERROR ] %s" % test | |
| 144 return all_tests_passed | |
| 145 | |
| 146 | |
| 147 def _getBasicFailureFinder(): | |
| 148 return FailureFinder(None, "Webkit", False, "", ".", 10, False) | |
| 149 | |
| 150 | |
| 151 def _testLastBuild(failure_finder): | |
| 152 try: | |
| 153 last_build = failure_finder.GetLastBuild() | |
| 154 # Verify that last_build is not empty and is a number. | |
| 155 build = int(last_build) | |
| 156 return (build > 0) | |
| 157 except: | |
| 158 return False | |
| 159 | |
| 160 | |
| 161 def testGetLastBuild(): | |
| 162 test = _getBasicFailureFinder() | |
| 163 return _testLastBuild(test) | |
| 164 | |
| 165 | |
| 166 def testWhitespaceInBuilderName(): | |
| 167 test = _getBasicFailureFinder() | |
| 168 test.SetPlatform("Webkit (webkit.org)") | |
| 169 return _testLastBuild(test) | |
| 170 | |
| 171 | |
| 172 def testScrapeBuilderOutput(): | |
| 173 | |
| 174 # Try on the default builder. | |
| 175 test = _getBasicFailureFinder() | |
| 176 test.build = "9800" | |
| 177 output = test._ScrapeBuilderOutput() | |
| 178 if not output: | |
| 179 return False | |
| 180 | |
| 181 # Try on a crazy builder on the FYI waterfall. | |
| 182 test = _getBasicFailureFinder() | |
| 183 test.build = "1766" | |
| 184 test.SetPlatform("Webkit Linux (webkit.org)") | |
| 185 output = test._ScrapeBuilderOutput() | |
| 186 if not output: | |
| 187 return False | |
| 188 | |
| 189 return True | |
| 190 | |
| 191 | |
| 192 def testFindMatchesInBuilderOutput(): | |
| 193 test = _getBasicFailureFinder() | |
| 194 test.exclude_known_failures = True | |
| 195 matches = test._FindMatchesInBuilderOutput(TEST_BUILDER_OUTPUT) | |
| 196 # Verify that we found x matches. | |
| 197 if len(matches) != 2: | |
| 198 print "Did not find all unexpected failures." | |
| 199 return False | |
| 200 | |
| 201 test.exclude_known_failures = False | |
| 202 matches = test._FindMatchesInBuilderOutput(TEST_BUILDER_OUTPUT) | |
| 203 if len(matches) != 3: | |
| 204 print "Did not find all failures." | |
| 205 return False | |
| 206 return True | |
| 207 | |
| 208 | |
| 209 def _testBaseline(test_name, expected_local, expected_url): | |
| 210 test = _getBasicFailureFinder() | |
| 211 # Test baseline that is obviously in Chromium's tree. | |
| 212 baseline = test._GetBaseline(test_name, ".", False) | |
| 213 try: | |
| 214 os.remove(baseline.local_file) | |
| 215 if (baseline.local_file != expected_local or | |
| 216 baseline.baseline_url != expected_url): | |
| 217 return False | |
| 218 except: | |
| 219 return False | |
| 220 return True | |
| 221 | |
| 222 | |
| 223 def testGetChromiumBaseline(): | |
| 224 return _testBaseline(CHROMIUM_BASELINE, EXPECTED_CHROMIUM_LOCAL_BASELINE, | |
| 225 EXPECTED_CHROMIUM_URL_BASELINE) | |
| 226 | |
| 227 | |
| 228 def testGetWebkitBaseline(): | |
| 229 return _testBaseline(WEBKIT_BASELINE, EXPECTED_WEBKIT_LOCAL_BASELINE, | |
| 230 EXPECTED_WEBKIT_URL_BASELINE) | |
| 231 | |
| 232 | |
| 233 def testUseLocalOutput(): | |
| 234 test_result = True | |
| 235 try: | |
| 236 _writeFile(TEST_BUILDER_LOG_FILE, TEST_BUILDER_OUTPUT) | |
| 237 _writeFile(TEST_ARCHIVE_LOG_FILE, TEST_ARCHIVE_OUTPUT) | |
| 238 _writeFile(TEST_EXPECTATIONS_FILE, TEST_TEST_EXPECTATIONS) | |
| 239 zip = zipfile.ZipFile(TEST_DUMMY_ZIP_FILE, 'w') | |
| 240 zip.write(TEST_BUILDER_LOG_FILE, TEST_FAILURE_1) | |
| 241 zip.write(TEST_BUILDER_LOG_FILE, TEST_FAILURE_2) | |
| 242 zip.write(TEST_BUILDER_LOG_FILE, TEST_FAILURE_3) | |
| 243 zip.close() | |
| 244 test = _getBasicFailureFinder() | |
| 245 test.archive_step_log_file = TEST_ARCHIVE_LOG_FILE | |
| 246 test.builder_output_log_file = TEST_BUILDER_LOG_FILE | |
| 247 test.test_expectations_file = TEST_EXPECTATIONS_FILE | |
| 248 test.zip_file = TEST_DUMMY_ZIP_FILE | |
| 249 test.dont_download = True | |
| 250 test.exclude_known_failures = True | |
| 251 test.delete_zip_file = False | |
| 252 failures = test.GetFailures() | |
| 253 if not failures or len(failures) != 2: | |
| 254 print "Did not get expected number of failures :" | |
| 255 for failure in failures: | |
| 256 print failure.test_path | |
| 257 test_result = False | |
| 258 finally: | |
| 259 os.remove(TEST_BUILDER_LOG_FILE) | |
| 260 os.remove(TEST_ARCHIVE_LOG_FILE) | |
| 261 os.remove(TEST_EXPECTATIONS_FILE) | |
| 262 os.remove(TEST_DUMMY_ZIP_FILE) | |
| 263 return test_result | |
| 264 | |
| 265 | |
| 266 def _writeFile(filename, contents): | |
| 267 myfile = open(filename, 'w') | |
| 268 myfile.write(contents) | |
| 269 myfile.close() | |
| 270 | |
| 271 | |
| 272 def testZipDownload(): | |
| 273 test = _getBasicFailureFinder() | |
| 274 try: | |
| 275 test._DownloadFile(TEST_ZIP_FILE, "test.zip", "b") # "b" -> binary | |
| 276 os.remove("test.zip") | |
| 277 return True | |
| 278 except: | |
| 279 return False | |
| 280 | |
| 281 | |
| 282 def testTranslateBuildToZip(): | |
| 283 test = _getBasicFailureFinder() | |
| 284 test.build = WEBKIT_BUILDER_NUMBER | |
| 285 revision, build_name = test._GetRevisionAndBuildFromArchiveStep() | |
| 286 if revision != EXPECTED_REVISION or build_name != EXPECTED_BUILD_NAME: | |
| 287 return False | |
| 288 return True | |
| 289 | |
| 290 | |
| 291 def testGetBaseline(): | |
| 292 test = _getBasicFailureFinder() | |
| 293 result = True | |
| 294 test.platform = "chromium-mac" | |
| 295 baseline = test._GetBaseline(WEBARCHIVE_TEST_EXPECTATION, ".") | |
| 296 if not baseline.local_file or baseline.baseline_url.find(WEBKIT_ORG) == -1: | |
| 297 result = False | |
| 298 print "Webarchive layout test not found at webkit.org: %s" % url | |
| 299 test.platform = "chromium-win" | |
| 300 baseline = test._GetBaseline(SVG_TEST_EXPECTATION, ".") | |
| 301 if (not baseline.local_file or | |
| 302 baseline.baseline_url.find(CHROMIUM_ORG) == -1): | |
| 303 result = False | |
| 304 print "SVG layout test found at %s, not chromium.org" % url | |
| 305 baseline = test._GetBaseline(SVG_TEST_EXPECTATION, ".", True) | |
| 306 if not baseline.local_file or baseline.baseline_url.find(WEBKIT_ORG) == -1: | |
| 307 result = False | |
| 308 print "Upstream SVG layout test NOT found at webkit.org!" | |
| 309 baseline = test._GetBaseline(DOM_TEST_EXPECTATION, ".", True) | |
| 310 if (not baseline.local_file or | |
| 311 baseline.baseline_url.find("/platform/") > -1): | |
| 312 result = False | |
| 313 print ("Upstream SVG layout test found in a " | |
| 314 "platform directory: %s" % url) | |
| 315 os.remove(WEBARCHIVE_TEST_EXPECTATION) | |
| 316 os.remove(SVG_TEST_EXPECTATION) | |
| 317 os.remove(SVG_TEST_EXPECTATION_UPSTREAM) | |
| 318 os.remove(DOM_TEST_EXPECTATION_UPSTREAM) | |
| 319 deleteDir("LayoutTests") | |
| 320 return result | |
| 321 | |
| 322 | |
| 323 def deleteDir(directory): | |
| 324 """ Recursively deletes empty directories given a root. | |
| 325 This method will throw an exception if they are not empty. """ | |
| 326 for root, dirs, files in os.walk(directory, topdown=False): | |
| 327 for d in dirs: | |
| 328 try: | |
| 329 os.rmdir(os.path.join(root, d)) | |
| 330 except: | |
| 331 pass | |
| 332 os.rmdir(directory) | |
| 333 | |
| 334 | |
| 335 def testFull(): | |
| 336 """ Verifies that the entire system works end-to-end. """ | |
| 337 test = _getBasicFailureFinder() | |
| 338 test.build = WEBKIT_BUILDER_NUMBER | |
| 339 test.dont_download = True # Dry run only, no downloading needed. | |
| 340 failures = test.GetFailures() | |
| 341 # Verify that the max failures parameter works. | |
| 342 if not failures or len(failures) > 10: | |
| 343 "Got no failures or too many failures." | |
| 344 return False | |
| 345 | |
| 346 # Verify the failures match the list of expected failures. | |
| 347 for failure in failures: | |
| 348 if not (failure.test_path in WEBKIT_FAILURES): | |
| 349 print "Found a failure I did not expect to see." | |
| 350 return False | |
| 351 | |
| 352 return True | |
| 353 | |
| 354 | |
| 355 def testFindTestExpectations(): | |
| 356 test = _getBasicFailureFinder() | |
| 357 test._test_expectations_cache = TEST_EXPECTATIONS | |
| 358 match = test._GetTestExpectationsLine(EXPECT_EXACT_MATCH) | |
| 359 if not match: | |
| 360 return False | |
| 361 match = test._GetTestExpectationsLine(EXPECT_GENERAL_MATCH) | |
| 362 if not match: | |
| 363 return False | |
| 364 match = test._GetTestExpectationsLine(EXPECT_NO_MATCH) | |
| 365 return not match | |
| 366 | |
| 367 | |
| 368 if __name__ == "__main__": | |
| 369 fft = FailureFinderTest() | |
| 370 result = fft.runTests() | |
| 371 if result: | |
| 372 print "All tests passed." | |
| 373 else: | |
| 374 print "Not all tests passed." | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 545145:
Move the layout test scripts into a 'webkitpy' subdirectory in preparation... (Closed)
Base URL: svn://chrome-svn/chrome/trunk/src/