Chromium Code Reviews Chromium Code Reviews
Issue 11043007:
Add the ability to get the Javascript console errors in Firefox in the test output when tests fail. (Closed)
Base URL: http://dart.googlecode.com/svn/branches/bleeding_edge/dart/| OLD | NEW |
|---|---|
| 1 #!/usr/bin/python | 1 #!/usr/bin/python |
| 2 | 2 |
| 3 # Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file | 3 # Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file |
| 4 # for details. All rights reserved. Use of this source code is governed by a | 4 # for details. All rights reserved. Use of this source code is governed by a |
| 5 # BSD-style license that can be found in the LICENSE file. | 5 # BSD-style license that can be found in the LICENSE file. |
| 6 # | 6 # |
| 7 | 7 |
| 8 """Script to actually open a browser and perform the test, and reports back with | 8 """Script to actually open a browser and perform the test, and reports back with |
| 9 the result. It uses Selenium WebDriver when possible for running the tests. It | 9 the result. It uses Selenium WebDriver when possible for running the tests. It |
| 10 uses Selenium RC for Safari. | 10 uses Selenium RC for Safari. |
| (...skipping 146 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 157 options.binary_location = executable_path | 157 options.binary_location = executable_path |
| 158 elif platform.system() == 'Windows': | 158 elif platform.system() == 'Windows': |
| 159 options.binary_location = os.path.join(dartium_dir, 'chrome.exe') | 159 options.binary_location = os.path.join(dartium_dir, 'chrome.exe') |
| 160 elif platform.system() == 'Darwin': | 160 elif platform.system() == 'Darwin': |
| 161 options.binary_location = os.path.join(dartium_dir, 'Chromium.app', | 161 options.binary_location = os.path.join(dartium_dir, 'Chromium.app', |
| 162 'Contents', 'MacOS', 'Chromium') | 162 'Contents', 'MacOS', 'Chromium') |
| 163 else: | 163 else: |
| 164 options.binary_location = os.path.join(dartium_dir, 'chrome') | 164 options.binary_location = os.path.join(dartium_dir, 'chrome') |
| 165 return selenium.webdriver.Chrome(chrome_options=options) | 165 return selenium.webdriver.Chrome(chrome_options=options) |
| 166 elif browser == 'ff': | 166 elif browser == 'ff': |
| 167 script_dir = os.path.dirname(os.path.abspath(__file__)) | |
| 167 profile = selenium.webdriver.firefox.firefox_profile.FirefoxProfile() | 168 profile = selenium.webdriver.firefox.firefox_profile.FirefoxProfile() |
| 168 profile.set_preference('dom.max_script_run_time', 0) | 169 profile.set_preference('dom.max_script_run_time', 0) |
| 169 profile.set_preference('dom.max_chrome_script_run_time', 0) | 170 profile.set_preference('dom.max_chrome_script_run_time', 0) |
| 170 profile.set_preference('app.update.auto', True) | 171 profile.set_preference('app.update.auto', True) |
| 171 profile.set_preference('app.update.enabled', True) | 172 profile.set_preference('app.update.enabled', True) |
| 173 xpi = os.path.join(script_dir, 'extensions', 'firefox', 'ConsoleCollector.xpi') | |
| 174 profile.add_extension(xpi); | |
| 172 return selenium.webdriver.Firefox(firefox_profile=profile) | 175 return selenium.webdriver.Firefox(firefox_profile=profile) |
| 173 elif browser == 'ie' and platform.system() == 'Windows': | 176 elif browser == 'ie' and platform.system() == 'Windows': |
| 174 return selenium.webdriver.Ie() | 177 return selenium.webdriver.Ie() |
| 175 elif browser == 'safari' and platform.system() == 'Darwin': | 178 elif browser == 'safari' and platform.system() == 'Darwin': |
| 176 # TODO(efortuna): Ensure our preferences (no pop-up blocking) file is the | 179 # TODO(efortuna): Ensure our preferences (no pop-up blocking) file is the |
| 177 # same (Safari auto-deletes when it has too many "crashes," or in our case, | 180 # same (Safari auto-deletes when it has too many "crashes," or in our case, |
| 178 # timeouts). Come up with a less hacky way to do this. | 181 # timeouts). Come up with a less hacky way to do this. |
| 179 backup_safari_prefs = os.path.dirname(__file__) + '/com.apple.Safari.plist' | 182 backup_safari_prefs = os.path.dirname(__file__) + '/com.apple.Safari.plist' |
| 180 if os.path.exists(backup_safari_prefs): | 183 if os.path.exists(backup_safari_prefs): |
| 181 shutil.copy(backup_safari_prefs, | 184 shutil.copy(backup_safari_prefs, |
| (...skipping 31 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 213 | 216 |
| 214 # A timeout exception is thrown if nothing happens within the time limit. | 217 # A timeout exception is thrown if nothing happens within the time limit. |
| 215 if browser != 'chrome': | 218 if browser != 'chrome': |
| 216 browser.close() | 219 browser.close() |
| 217 try: | 220 try: |
| 218 browser.quit() | 221 browser.quit() |
| 219 except selenium.common.exceptions.WebDriverException: | 222 except selenium.common.exceptions.WebDriverException: |
| 220 # TODO(efortuna): Figure out why this crashes.... and avoid? | 223 # TODO(efortuna): Figure out why this crashes.... and avoid? |
| 221 pass | 224 pass |
| 222 | 225 |
| 223 def report_results(mode, source): | 226 def report_results(mode, source, browser_name, browser): |
| 224 # TODO(vsm): Add a failure check for Dromaeo. | 227 # TODO(vsm): Add a failure check for Dromaeo. |
| 225 if mode != 'correctness': | 228 if mode != 'correctness': |
| 226 # We're running a performance test. | 229 # We're running a performance test. |
| 227 print source.encode('utf8') | 230 print source.encode('utf8') |
| 228 sys.stdout.flush() | 231 sys.stdout.flush() |
| 229 if 'NaN' in source: | 232 if 'NaN' in source: |
| 230 return 1 | 233 return 1 |
| 231 else: | 234 else: |
| 232 return 0 | 235 return 0 |
| 233 else: | 236 else: |
| 234 # We're running a correctness test. Mark test as passing if all individual | 237 # We're running a correctness test. Mark test as passing if all individual |
| 235 # test cases pass. | 238 # test cases pass. |
| 236 if 'FAIL' not in source and 'PASS' in source: | 239 if 'FAIL' not in source and 'PASS' in source: |
| 237 print 'Content-Type: text/plain\nPASS' | 240 print 'Content-Type: text/plain\nPASS' |
| 238 return 0 | 241 return 0 |
| 239 else: | 242 else: |
| 240 #The hacky way to get document.getElementById('body').innerHTML for this | 243 #The hacky way to get document.getElementById('body').innerHTML for this |
| 241 # webpage, without the JavaScript. | 244 # webpage, without the JavaScript. |
| 242 #TODO(efortuna): Access these elements in a nicer way using DOM parser. | 245 #TODO(efortuna): Access these elements in a nicer way using DOM parser. |
| 243 index = source.find('<body>') | 246 index = source.find('<body>') |
| 244 index += len('<body>') | 247 index += len('<body>') |
| 245 end_index = source.find('</body') | 248 end_index = source.find('</body') |
| 246 print unicode(source[index : end_index]).encode("utf-8") | 249 print unicode(source[index : end_index]).encode("utf-8") |
| 250 if browser_name == 'ff': | |
|
Emily Fortuna
2012/10/02 22:40:49
consider checking if browser is selenium.webdriver
gram
2012/10/04 18:43:12
I tried this; it didn't work, so I'm sticking to t
| |
| 251 logs = browser.execute_script( | |
| 252 "return window.ConsoleCollector.read()"); | |
| 253 for msg in logs: | |
| 254 print('%s:%s:%s:%s %s' %( | |
| 255 msg['source'], msg['line'], msg['column'], | |
| 256 msg['category'], msg['message'])) | |
| 247 return 1 | 257 return 1 |
| 248 | 258 |
| 249 | 259 |
| 250 def run_batch_tests(): | 260 def run_batch_tests(): |
| 251 ''' | 261 ''' |
| 252 Runs a batch of in-browser tests in the same browser process. Batching | 262 Runs a batch of in-browser tests in the same browser process. Batching |
| 253 gives faster throughput and makes tests less subject to browser starting | 263 gives faster throughput and makes tests less subject to browser starting |
| 254 flakiness, issues with too many browser processes running, etc. | 264 flakiness, issues with too many browser processes running, etc. |
| 255 | 265 |
| 256 When running this function, stdin/stdout is used to communicate with the test | 266 When running this function, stdin/stdout is used to communicate with the test |
| (...skipping 42 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 299 browser = start_browser(browser_name, executable_path, html_out) | 309 browser = start_browser(browser_name, executable_path, html_out) |
| 300 | 310 |
| 301 source = run_test_in_browser(browser, html_out, timeout, mode) | 311 source = run_test_in_browser(browser, html_out, timeout, mode) |
| 302 | 312 |
| 303 # Test is done. Write end token to stderr and flush. | 313 # Test is done. Write end token to stderr and flush. |
| 304 sys.stderr.write('>>> EOF STDERR\n') | 314 sys.stderr.write('>>> EOF STDERR\n') |
| 305 sys.stderr.flush() | 315 sys.stderr.flush() |
| 306 | 316 |
| 307 # print one of: | 317 # print one of: |
| 308 # >>> TEST {PASS, FAIL, OK, CRASH, FAIL, TIMEOUT} | 318 # >>> TEST {PASS, FAIL, OK, CRASH, FAIL, TIMEOUT} |
| 309 status = report_results(mode, source) | 319 status = report_results(mode, source, browser_name, browser) |
| 310 if status == 0: | 320 if status == 0: |
| 311 print '>>> TEST PASS' | 321 print '>>> TEST PASS' |
| 312 elif source == TIMEOUT_ERROR_MSG: | 322 elif source == TIMEOUT_ERROR_MSG: |
| 313 print '>>> TEST TIMEOUT' | 323 print '>>> TEST TIMEOUT' |
| 314 else: | 324 else: |
| 315 print '>>> TEST FAIL' | 325 print '>>> TEST FAIL' |
| 316 sys.stdout.flush() | 326 sys.stdout.flush() |
| 317 finally: | 327 finally: |
| 318 close_browser(browser) | 328 close_browser(browser) |
| 319 | 329 |
| 320 | 330 |
| 321 def main(args): | 331 def main(args): |
| 322 # Run in batch mode if the --batch flag is passed. | 332 # Run in batch mode if the --batch flag is passed. |
| 323 # TODO(jmesserly): reconcile with the existing args parsing | 333 # TODO(jmesserly): reconcile with the existing args parsing |
| 324 if '--batch' in args: | 334 if '--batch' in args: |
| 325 return run_batch_tests() | 335 return run_batch_tests() |
| 326 | 336 |
| 327 # Run a single test | 337 # Run a single test |
| 328 html_out, browser_name, executable_path, timeout, mode = parse_args() | 338 html_out, browser_name, executable_path, timeout, mode = parse_args() |
| 329 browser = start_browser(browser_name, executable_path, html_out) | 339 browser = start_browser(browser_name, executable_path, html_out) |
| 330 | 340 |
| 331 try: | 341 try: |
| 332 output = run_test_in_browser(browser, html_out, timeout, mode) | 342 output = run_test_in_browser(browser, html_out, timeout, mode) |
| 333 return report_results(mode, output) | 343 return report_results(mode, output, browser_name, browser) |
| 334 finally: | 344 finally: |
| 335 close_browser(browser) | 345 close_browser(browser) |
| 336 | 346 |
| 337 if __name__ == "__main__": | 347 if __name__ == "__main__": |
| 338 sys.exit(main(sys.argv)) | 348 sys.exit(main(sys.argv)) |
| OLD | NEW |
