| OLD | NEW |
| 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 /** | 5 /** |
| 6 * Classes and methods for executing tests. | 6 * Classes and methods for executing tests. |
| 7 * | 7 * |
| 8 * This module includes: | 8 * This module includes: |
| 9 * - Managing parallel execution of tests, including timeout checks. | 9 * - Managing parallel execution of tests, including timeout checks. |
| 10 * - Evaluating the output of each test as pass/fail/crash/timeout. | 10 * - Evaluating the output of each test as pass/fail/crash/timeout. |
| 11 */ | 11 */ |
| 12 #library("test_runner"); | 12 #library("test_runner"); |
| 13 | 13 |
| 14 #import("dart:io"); | 14 #import("dart:io"); |
| 15 #import("dart:isolate"); | 15 #import("dart:isolate"); |
| 16 #import("status_file_parser.dart"); | 16 #import("status_file_parser.dart"); |
| 17 #import("test_progress.dart"); | 17 #import("test_progress.dart"); |
| 18 #import("test_suite.dart"); | 18 #import("test_suite.dart"); |
| 19 | 19 |
| 20 final int NO_TIMEOUT = 0; | 20 const int NO_TIMEOUT = 0; |
| 21 final int SLOW_TIMEOUT_MULTIPLIER = 4; | 21 const int SLOW_TIMEOUT_MULTIPLIER = 4; |
| 22 | 22 |
| 23 /** A command executed as a step in a test case. */ | 23 /** A command executed as a step in a test case. */ |
| 24 class Command { | 24 class Command { |
| 25 /** Path to the executable of this command. */ | 25 /** Path to the executable of this command. */ |
| 26 String executable; | 26 String executable; |
| 27 | 27 |
| 28 /** Command line arguments to the executable. */ | 28 /** Command line arguments to the executable. */ |
| 29 List<String> arguments; | 29 List<String> arguments; |
| 30 | 30 |
| 31 /** The actual command line that will be executed. */ | 31 /** The actual command line that will be executed. */ |
| (...skipping 277 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 309 // Browser case: | 309 // Browser case: |
| 310 // If the browser test failed, it may have been because DumpRenderTree | 310 // If the browser test failed, it may have been because DumpRenderTree |
| 311 // and the virtual framebuffer X server didn't hook up, or DRT crashed with | 311 // and the virtual framebuffer X server didn't hook up, or DRT crashed with |
| 312 // a core dump. Sometimes DRT crashes after it has set the stdout to PASS, | 312 // a core dump. Sometimes DRT crashes after it has set the stdout to PASS, |
| 313 // so we have to do this check first. | 313 // so we have to do this check first. |
| 314 for (String line in super.stderr) { | 314 for (String line in super.stderr) { |
| 315 if (line.contains('Gtk-WARNING **: cannot open display: :99') || | 315 if (line.contains('Gtk-WARNING **: cannot open display: :99') || |
| 316 line.contains('Failed to run command. return code=1')) { | 316 line.contains('Failed to run command. return code=1')) { |
| 317 // If we get the X server error, or DRT crashes with a core dump, retry | 317 // If we get the X server error, or DRT crashes with a core dump, retry |
| 318 // the test. | 318 // the test. |
| 319 if ((testCase as Dynamic).numRetries > 0) { | 319 if ((testCase as BrowserTestCase).numRetries > 0) { |
| 320 requestRetry = true; | 320 requestRetry = true; |
| 321 } | 321 } |
| 322 return true; | 322 return true; |
| 323 } | 323 } |
| 324 } | 324 } |
| 325 | 325 |
| 326 // Browser tests fail unless stdout contains | 326 // Browser tests fail unless stdout contains |
| 327 // 'Content-Type: text/plain' followed by 'PASS'. | 327 // 'Content-Type: text/plain' followed by 'PASS'. |
| 328 bool has_content_type = false; | 328 bool has_content_type = false; |
| 329 for (String line in super.stdout) { | 329 for (String line in super.stdout) { |
| (...skipping 214 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 544 timeoutTimer.cancel(); | 544 timeoutTimer.cancel(); |
| 545 if (testCase.output.unexpectedOutput | 545 if (testCase.output.unexpectedOutput |
| 546 && testCase.configuration['verbose'] != null | 546 && testCase.configuration['verbose'] != null |
| 547 && testCase.configuration['verbose']) { | 547 && testCase.configuration['verbose']) { |
| 548 print(testCase.displayName); | 548 print(testCase.displayName); |
| 549 for (var line in testCase.output.stderr) print(line); | 549 for (var line in testCase.output.stderr) print(line); |
| 550 for (var line in testCase.output.stdout) print(line); | 550 for (var line in testCase.output.stdout) print(line); |
| 551 } | 551 } |
| 552 if (allowRetries && testCase.usesWebDriver | 552 if (allowRetries && testCase.usesWebDriver |
| 553 && testCase.output.unexpectedOutput | 553 && testCase.output.unexpectedOutput |
| 554 && (testCase as Dynamic).numRetries > 0) { | 554 && (testCase as BrowserTestCase).numRetries > 0) { |
| 555 // Selenium tests can be flaky. Try rerunning. | 555 // Selenium tests can be flaky. Try rerunning. |
| 556 testCase.output.requestRetry = true; | 556 testCase.output.requestRetry = true; |
| 557 } | 557 } |
| 558 if (testCase.output.requestRetry) { | 558 if (testCase.output.requestRetry) { |
| 559 testCase.output.requestRetry = false; | 559 testCase.output.requestRetry = false; |
| 560 this.timedOut = false; | 560 this.timedOut = false; |
| 561 (testCase as Dynamic).numRetries--; | 561 (testCase as BrowserTestCase).numRetries--; |
| 562 print("Potential flake. Re-running ${testCase.displayName} " | 562 print("Potential flake. Re-running ${testCase.displayName} " |
| 563 "(${(testCase as Dynamic).numRetries} attempt(s) remains)"); | 563 "(${(testCase as BrowserTestCase).numRetries} attempt(s) remains)"); |
| 564 this.start(); | 564 this.start(); |
| 565 } else { | 565 } else { |
| 566 testCase.completed(); | 566 testCase.completed(); |
| 567 } | 567 } |
| 568 } | 568 } |
| 569 | 569 |
| 570 /** | 570 /** |
| 571 * Process exit handler called at the end of every command. It internally | 571 * Process exit handler called at the end of every command. It internally |
| 572 * treats all but the last command as compilation steps. The last command is | 572 * treats all but the last command as compilation steps. The last command is |
| 573 * the actual test and its output is analyzed in [testComplete]. | 573 * the actual test and its output is analyzed in [testComplete]. |
| (...skipping 454 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1028 final StringInputStream stdoutStringStream = | 1028 final StringInputStream stdoutStringStream = |
| 1029 new StringInputStream(p.stdout); | 1029 new StringInputStream(p.stdout); |
| 1030 p.onError = (e) { | 1030 p.onError = (e) { |
| 1031 print("Error starting process:"); | 1031 print("Error starting process:"); |
| 1032 print(" Command: $cmd ${Strings.join(arg, ' ')}"); | 1032 print(" Command: $cmd ${Strings.join(arg, ' ')}"); |
| 1033 print(" Error: $e"); | 1033 print(" Error: $e"); |
| 1034 }; | 1034 }; |
| 1035 stdoutStringStream.onLine = () { | 1035 stdoutStringStream.onLine = () { |
| 1036 var line = stdoutStringStream.readLine(); | 1036 var line = stdoutStringStream.readLine(); |
| 1037 while (null != line) { | 1037 while (null != line) { |
| 1038 if (const RegExp(@".*selenium-server-standalone.*").hasMatch(line)) { | 1038 if (const RegExp(r".*selenium-server-standalone.*").hasMatch(line)) { |
| 1039 _seleniumAlreadyRunning = true; | 1039 _seleniumAlreadyRunning = true; |
| 1040 resumeTesting(); | 1040 resumeTesting(); |
| 1041 } | 1041 } |
| 1042 line = stdoutStringStream.readLine(); | 1042 line = stdoutStringStream.readLine(); |
| 1043 } | 1043 } |
| 1044 if (!_isSeleniumAvailable) { | 1044 if (!_isSeleniumAvailable) { |
| 1045 _startSeleniumServer(); | 1045 _startSeleniumServer(); |
| 1046 } | 1046 } |
| 1047 }; | 1047 }; |
| 1048 } | 1048 } |
| (...skipping 12 matching lines...) Expand all Loading... |
| 1061 /** | 1061 /** |
| 1062 * Monitor the output of the Selenium server, to know when we are ready to | 1062 * Monitor the output of the Selenium server, to know when we are ready to |
| 1063 * begin running tests. | 1063 * begin running tests. |
| 1064 * source: Output(Stream) from the Java server. | 1064 * source: Output(Stream) from the Java server. |
| 1065 */ | 1065 */ |
| 1066 Function makeSeleniumServerHandler(StringInputStream source) { | 1066 Function makeSeleniumServerHandler(StringInputStream source) { |
| 1067 return () { | 1067 return () { |
| 1068 if (source.closed) return; // TODO(whesse): Remove when bug is fixed. | 1068 if (source.closed) return; // TODO(whesse): Remove when bug is fixed. |
| 1069 var line = source.readLine(); | 1069 var line = source.readLine(); |
| 1070 while (null != line) { | 1070 while (null != line) { |
| 1071 if (const RegExp(@".*Started.*Server.*").hasMatch(line) || | 1071 if (const RegExp(r".*Started.*Server.*").hasMatch(line) || |
| 1072 const RegExp(@"Exception.*Selenium is already running.*").hasMatch( | 1072 const RegExp(r"Exception.*Selenium is already running.*").hasMatch( |
| 1073 line)) { | 1073 line)) { |
| 1074 resumeTesting(); | 1074 resumeTesting(); |
| 1075 } | 1075 } |
| 1076 line = source.readLine(); | 1076 line = source.readLine(); |
| 1077 } | 1077 } |
| 1078 }; | 1078 }; |
| 1079 } | 1079 } |
| 1080 | 1080 |
| 1081 /** | 1081 /** |
| 1082 * For browser tests using Safari or Opera, we need to use the Selenium 1.0 | 1082 * For browser tests using Safari or Opera, we need to use the Selenium 1.0 |
| 1083 * Java server. | 1083 * Java server. |
| 1084 */ | 1084 */ |
| 1085 void _startSeleniumServer() { | 1085 void _startSeleniumServer() { |
| 1086 // Get the absolute path to the Selenium jar. | 1086 // Get the absolute path to the Selenium jar. |
| 1087 String filePath = new Options().script; | 1087 String filePath = new Options().script; |
| 1088 String pathSep = Platform.pathSeparator; | 1088 String pathSep = Platform.pathSeparator; |
| 1089 int index = filePath.lastIndexOf(pathSep); | 1089 int index = filePath.lastIndexOf(pathSep); |
| 1090 filePath = '${filePath.substring(0, index)}${pathSep}testing${pathSep}'; | 1090 filePath = '${filePath.substring(0, index)}${pathSep}testing${pathSep}'; |
| 1091 var lister = new Directory(filePath).list(); | 1091 var lister = new Directory(filePath).list(); |
| 1092 lister.onFile = (String file) { | 1092 lister.onFile = (String file) { |
| 1093 if (const RegExp(@"selenium-server-standalone-.*\.jar").hasMatch(file) | 1093 if (const RegExp(r"selenium-server-standalone-.*\.jar").hasMatch(file) |
| 1094 && _seleniumServer == null) { | 1094 && _seleniumServer == null) { |
| 1095 _seleniumServer = Process.start('java', ['-jar', file]); | 1095 _seleniumServer = Process.start('java', ['-jar', file]); |
| 1096 _seleniumServer.onError = (e) { | 1096 _seleniumServer.onError = (e) { |
| 1097 print("Error starting process:"); | 1097 print("Error starting process:"); |
| 1098 print(" Command: java -jar $file"); | 1098 print(" Command: java -jar $file"); |
| 1099 print(" Error: $e"); | 1099 print(" Error: $e"); |
| 1100 }; | 1100 }; |
| 1101 // Heads up: there seems to an obscure data race of some form in | 1101 // Heads up: there seems to an obscure data race of some form in |
| 1102 // the VM between launching the server process and launching the test | 1102 // the VM between launching the server process and launching the test |
| 1103 // tasks that disappears when you read IO (which is convenient, since | 1103 // tasks that disappears when you read IO (which is convenient, since |
| (...skipping 82 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1186 // the developer doesn't waste his or her time trying to fix a bunch of | 1186 // the developer doesn't waste his or her time trying to fix a bunch of |
| 1187 // tests that appear to be broken but were actually just flakes that | 1187 // tests that appear to be broken but were actually just flakes that |
| 1188 // didn't get retried because there had already been one failure. | 1188 // didn't get retried because there had already been one failure. |
| 1189 bool allowRetry = _MAX_FAILED_NO_RETRY > _progress.numFailedTests; | 1189 bool allowRetry = _MAX_FAILED_NO_RETRY > _progress.numFailedTests; |
| 1190 new RunningProcess(test, allowRetry, this).start(); | 1190 new RunningProcess(test, allowRetry, this).start(); |
| 1191 } | 1191 } |
| 1192 _numProcesses++; | 1192 _numProcesses++; |
| 1193 } | 1193 } |
| 1194 } | 1194 } |
| 1195 } | 1195 } |
| OLD | NEW |