| OLD | NEW |
| 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 library test_configurations; | 5 library test_configurations; |
| 6 | 6 |
| 7 import "dart:async"; | 7 import "dart:async"; |
| 8 import 'dart:io'; | 8 import 'dart:io'; |
| 9 import "dart:math" as math; | 9 import "dart:math" as math; |
| 10 | 10 |
| 11 import "browser_controller.dart"; | 11 import "browser_controller.dart"; |
| 12 import "co19_test_config.dart"; | 12 import "co19_test_config.dart"; |
| 13 import "http_server.dart"; | 13 import "http_server.dart"; |
| 14 import "path.dart"; | 14 import "path.dart"; |
| 15 import "test_progress.dart"; | 15 import "test_progress.dart"; |
| 16 import "test_runner.dart"; | 16 import "test_runner.dart"; |
| 17 import "test_suite.dart"; | 17 import "test_suite.dart"; |
| 18 import "utils.dart"; | 18 import "utils.dart"; |
| 19 import "vm_test_config.dart"; | 19 import "vm_test_config.dart"; |
| 20 | 20 |
| 21 /** | 21 /** |
| 22 * The directories that contain test suites which follow the conventions | 22 * The directories that contain test suites which follow the conventions |
| 23 * required by [StandardTestSuite]'s forDirectory constructor. | 23 * required by [StandardTestSuite]'s forDirectory constructor. |
| 24 * New test suites should follow this convention because it makes it much | 24 * New test suites should follow this convention because it makes it much |
| 25 * simpler to add them to test.dart. Existing test suites should be | 25 * simpler to add them to test.dart. Existing test suites should be |
| 26 * moved to here, if possible. | 26 * moved to here, if possible. |
| 27 */ | 27 */ |
| 28 final TEST_SUITE_DIRECTORIES = [ | 28 final TEST_SUITE_DIRECTORIES = [ |
| 29 new Path('pkg'), | 29 new Path('pkg'), |
| 30 new Path('third_party/pkg_tested'), | 30 new Path('third_party/pkg_tested'), |
| 31 new Path('runtime/tests/vm'), | 31 new Path('runtime/tests/vm'), |
| 32 new Path('runtime/observatory/tests/service'), | 32 new Path('runtime/observatory/tests/service'), |
| 33 new Path('samples'), | 33 new Path('samples'), |
| 34 new Path('samples-dev'), | 34 new Path('samples-dev'), |
| 35 new Path('tests/benchmark_smoke'), | 35 new Path('tests/benchmark_smoke'), |
| 36 new Path('tests/chrome'), | 36 new Path('tests/chrome'), |
| 37 new Path('tests/compiler/dart2js'), | 37 new Path('tests/compiler/dart2js'), |
| 38 new Path('tests/compiler/dart2js_extra'), | 38 new Path('tests/compiler/dart2js_extra'), |
| 39 new Path('tests/compiler/dart2js_native'), | 39 new Path('tests/compiler/dart2js_native'), |
| 40 new Path('tests/corelib'), | 40 new Path('tests/corelib'), |
| 41 new Path('tests/html'), | 41 new Path('tests/html'), |
| 42 new Path('tests/isolate'), | 42 new Path('tests/isolate'), |
| 43 new Path('tests/language'), | 43 new Path('tests/language'), |
| 44 new Path('tests/lib'), | 44 new Path('tests/lib'), |
| 45 new Path('tests/standalone'), | 45 new Path('tests/standalone'), |
| 46 new Path('tests/try'), | 46 new Path('tests/try'), |
| 47 new Path('tests/utils'), | 47 new Path('tests/utils'), |
| 48 new Path('utils/tests/css'), | 48 new Path('utils/tests/css'), |
| 49 new Path('utils/tests/peg'), | 49 new Path('utils/tests/peg'), |
| 50 ]; | 50 ]; |
| 51 | 51 |
| 52 void testConfigurations(List<Map> configurations) { | 52 void testConfigurations(List<Map> configurations) { |
| 53 var startTime = new DateTime.now(); | 53 var startTime = new DateTime.now(); |
| 54 // Extract global options from first configuration. | 54 // Extract global options from first configuration. |
| 55 var firstConf = configurations[0]; | 55 var firstConf = configurations[0]; |
| 56 var maxProcesses = firstConf['tasks']; | 56 var maxProcesses = firstConf['tasks']; |
| 57 var progressIndicator = firstConf['progress']; | 57 var progressIndicator = firstConf['progress']; |
| 58 // TODO(kustermann): Remove this option once the buildbots don't use it | 58 // TODO(kustermann): Remove this option once the buildbots don't use it |
| 59 // anymore. | 59 // anymore. |
| 60 var failureSummary = firstConf['failure-summary']; | 60 var failureSummary = firstConf['failure-summary']; |
| 61 BuildbotProgressIndicator.stepName = firstConf['step_name']; | 61 BuildbotProgressIndicator.stepName = firstConf['step_name']; |
| 62 var verbose = firstConf['verbose']; | 62 var verbose = firstConf['verbose']; |
| 63 var printTiming = firstConf['time']; | 63 var printTiming = firstConf['time']; |
| 64 var listTests = firstConf['list']; | 64 var listTests = firstConf['list']; |
| 65 | 65 |
| 66 var reportInJson = firstConf['report_in_json']; | 66 var reportInJson = firstConf['report_in_json']; |
| 67 | 67 |
| 68 var recordingPath = firstConf['record_to_file']; | 68 var recordingPath = firstConf['record_to_file']; |
| 69 var recordingOutputPath = firstConf['replay_from_file']; | 69 var recordingOutputPath = firstConf['replay_from_file']; |
| 70 | 70 |
| 71 Browser.deleteCache = firstConf['clear_browser_cache']; | 71 Browser.deleteCache = firstConf['clear_browser_cache']; |
| 72 | 72 |
| 73 if (recordingPath != null && recordingOutputPath != null) { | 73 if (recordingPath != null && recordingOutputPath != null) { |
| 74 print("Fatal: Can't have the '--record_to_file' and '--replay_from_file'" | 74 print("Fatal: Can't have the '--record_to_file' and '--replay_from_file'" |
| 75 "at the same time. Exiting ..."); | 75 "at the same time. Exiting ..."); |
| 76 exit(1); | 76 exit(1); |
| 77 } | 77 } |
| 78 | 78 |
| 79 if (!firstConf['append_logs']) { | 79 if (!firstConf['append_logs']) { |
| 80 var files = [new File(TestUtils.flakyFileName()), | 80 var files = [ |
| 81 new File(TestUtils.testOutcomeFileName())]; | 81 new File(TestUtils.flakyFileName()), |
| 82 new File(TestUtils.testOutcomeFileName()) |
| 83 ]; |
| 82 for (var file in files) { | 84 for (var file in files) { |
| 83 if (file.existsSync()) { | 85 if (file.existsSync()) { |
| 84 file.deleteSync(); | 86 file.deleteSync(); |
| 85 } | 87 } |
| 86 } | 88 } |
| 87 } | 89 } |
| 88 | 90 |
| 89 DebugLogger.init(firstConf['write_debug_log'] ? | 91 DebugLogger.init( |
| 90 TestUtils.debugLogfile() : null, append: firstConf['append_logs']); | 92 firstConf['write_debug_log'] ? TestUtils.debugLogfile() : null, |
| 93 append: firstConf['append_logs']); |
| 91 | 94 |
| 92 // Print the configurations being run by this execution of | 95 // Print the configurations being run by this execution of |
| 93 // test.dart. However, don't do it if the silent progress indicator | 96 // test.dart. However, don't do it if the silent progress indicator |
| 94 // is used. This is only needed because of the junit tests. | 97 // is used. This is only needed because of the junit tests. |
| 95 if (progressIndicator != 'silent') { | 98 if (progressIndicator != 'silent') { |
| 96 List output_words = configurations.length > 1 ? | 99 List output_words = configurations.length > 1 |
| 97 ['Test configurations:'] : ['Test configuration:']; | 100 ? ['Test configurations:'] |
| 101 : ['Test configuration:']; |
| 98 for (Map conf in configurations) { | 102 for (Map conf in configurations) { |
| 99 List settings = ['compiler', 'runtime', 'mode', 'arch'] | 103 List settings = ['compiler', 'runtime', 'mode', 'arch'] |
| 100 .map((name) => conf[name]).toList(); | 104 .map((name) => conf[name]) |
| 105 .toList(); |
| 101 if (conf['checked']) settings.add('checked'); | 106 if (conf['checked']) settings.add('checked'); |
| 102 if (conf['noopt']) settings.add('noopt'); | 107 if (conf['noopt']) settings.add('noopt'); |
| 103 output_words.add(settings.join('_')); | 108 output_words.add(settings.join('_')); |
| 104 } | 109 } |
| 105 print(output_words.join(' ')); | 110 print(output_words.join(' ')); |
| 106 } | 111 } |
| 107 | 112 |
| 108 var runningBrowserTests = configurations.any((config) { | 113 var runningBrowserTests = configurations.any((config) { |
| 109 return TestUtils.isBrowserRuntime(config['runtime']); | 114 return TestUtils.isBrowserRuntime(config['runtime']); |
| 110 }); | 115 }); |
| 111 | 116 |
| 112 List<Future> serverFutures = []; | 117 List<Future> serverFutures = []; |
| 113 var testSuites = new List<TestSuite>(); | 118 var testSuites = new List<TestSuite>(); |
| 114 var maxBrowserProcesses = maxProcesses; | 119 var maxBrowserProcesses = maxProcesses; |
| 115 if (configurations.length > 1 && | 120 if (configurations.length > 1 && |
| 116 (configurations[0]['test_server_port'] != 0 || | 121 (configurations[0]['test_server_port'] != 0 || |
| 117 configurations[0]['test_server_cross_origin_port'] != 0)) { | 122 configurations[0]['test_server_cross_origin_port'] != 0)) { |
| 118 print("If the http server ports are specified, only one configuration" | 123 print("If the http server ports are specified, only one configuration" |
| 119 " may be run at a time"); | 124 " may be run at a time"); |
| 120 exit(1); | 125 exit(1); |
| 121 } | 126 } |
| 122 for (var conf in configurations) { | 127 for (var conf in configurations) { |
| 123 Map<String, RegExp> selectors = conf['selectors']; | 128 Map<String, RegExp> selectors = conf['selectors']; |
| 124 var useContentSecurityPolicy = conf['csp']; | 129 var useContentSecurityPolicy = conf['csp']; |
| 125 if (!listTests && runningBrowserTests) { | 130 if (!listTests && runningBrowserTests) { |
| 126 // Start global http servers that serve the entire dart repo. | 131 // Start global http servers that serve the entire dart repo. |
| 127 // The http server is available on window.location.port, and a second | 132 // The http server is available on window.location.port, and a second |
| 128 // server for cross-domain tests can be found by calling | 133 // server for cross-domain tests can be found by calling |
| 129 // getCrossOriginPortNumber(). | 134 // getCrossOriginPortNumber(). |
| 130 var servers = new TestingServers(new Path(TestUtils.buildDir(conf)), | 135 var servers = new TestingServers( |
| 131 useContentSecurityPolicy, | 136 new Path(TestUtils.buildDir(conf)), |
| 132 conf['runtime'], | 137 useContentSecurityPolicy, |
| 133 null, | 138 conf['runtime'], |
| 134 conf['package_root']); | 139 null, |
| 140 conf['package_root']); |
| 135 serverFutures.add(servers.startServers(conf['local_ip'], | 141 serverFutures.add(servers.startServers(conf['local_ip'], |
| 136 port: conf['test_server_port'], | 142 port: conf['test_server_port'], |
| 137 crossOriginPort: conf['test_server_cross_origin_port'])); | 143 crossOriginPort: conf['test_server_cross_origin_port'])); |
| 138 conf['_servers_'] = servers; | 144 conf['_servers_'] = servers; |
| 139 if (verbose) { | 145 if (verbose) { |
| 140 serverFutures.last.then((_) { | 146 serverFutures.last.then((_) { |
| 141 var commandline = servers.httpServerCommandline(); | 147 var commandline = servers.httpServerCommandline(); |
| 142 print('Started HttpServers: $commandline'); | 148 print('Started HttpServers: $commandline'); |
| 143 }); | 149 }); |
| 144 } | 150 } |
| 145 } | 151 } |
| 146 | 152 |
| 147 if (conf['runtime'].startsWith('ie')) { | 153 if (conf['runtime'].startsWith('ie')) { |
| 148 // NOTE: We've experienced random timeouts of tests on ie9/ie10. The | 154 // NOTE: We've experienced random timeouts of tests on ie9/ie10. The |
| 149 // underlying issue has not been determined yet. Our current hypothesis | 155 // underlying issue has not been determined yet. Our current hypothesis |
| 150 // is that windows does not handle the IE processes independently. | 156 // is that windows does not handle the IE processes independently. |
| 151 // If we have more than one browser and kill a browser we are seeing | 157 // If we have more than one browser and kill a browser we are seeing |
| 152 // issues with starting up a new browser just after killing the hanging | 158 // issues with starting up a new browser just after killing the hanging |
| 153 // browser. | 159 // browser. |
| 154 maxBrowserProcesses = 1; | 160 maxBrowserProcesses = 1; |
| 155 } else if (conf['runtime'].startsWith('safari')) { | 161 } else if (conf['runtime'].startsWith('safari')) { |
| 156 // Safari does not allow us to run from a fresh profile, so we can only | 162 // Safari does not allow us to run from a fresh profile, so we can only |
| 157 // use one browser. Additionally, you can not start two simulators | 163 // use one browser. Additionally, you can not start two simulators |
| 158 // for mobile safari simultainiously. | 164 // for mobile safari simultainiously. |
| 159 maxBrowserProcesses = 1; | 165 maxBrowserProcesses = 1; |
| 160 } else if (conf['runtime'] == 'chrome' && | 166 } else if (conf['runtime'] == 'chrome' && |
| 161 Platform.operatingSystem == 'macos') { | 167 Platform.operatingSystem == 'macos') { |
| 162 // Chrome on mac results in random timeouts. | 168 // Chrome on mac results in random timeouts. |
| 163 // Issue: https://github.com/dart-lang/sdk/issues/23891 | 169 // Issue: https://github.com/dart-lang/sdk/issues/23891 |
| 164 // This change does not fix the problem. | 170 // This change does not fix the problem. |
| 165 maxBrowserProcesses = math.max(1, maxBrowserProcesses ~/ 2); | 171 maxBrowserProcesses = math.max(1, maxBrowserProcesses ~/ 2); |
| 166 } else if (conf['runtime'] != 'drt') { | 172 } else if (conf['runtime'] != 'drt') { |
| 167 // Even on machines with more than 16 processors, don't open more | 173 // Even on machines with more than 16 processors, don't open more |
| 168 // than 15 browser instances, to avoid overloading the machine. | 174 // than 15 browser instances, to avoid overloading the machine. |
| 169 // This is especially important when running locally on powerful | 175 // This is especially important when running locally on powerful |
| 170 // desktops. | 176 // desktops. |
| 171 maxBrowserProcesses = math.min(maxBrowserProcesses, 15); | 177 maxBrowserProcesses = math.min(maxBrowserProcesses, 15); |
| 172 } | 178 } |
| 173 | 179 |
| 174 // If we specifically pass in a suite only run that. | 180 // If we specifically pass in a suite only run that. |
| 175 if (conf['suite_dir'] != null) { | 181 if (conf['suite_dir'] != null) { |
| 176 var suite_path = new Path(conf['suite_dir']); | 182 var suite_path = new Path(conf['suite_dir']); |
| 177 testSuites.add(new PKGTestSuite(conf, suite_path)); | 183 testSuites.add(new PKGTestSuite(conf, suite_path)); |
| 178 } else { | 184 } else { |
| 179 for (String key in selectors.keys) { | 185 for (String key in selectors.keys) { |
| 180 if (key == 'co19') { | 186 if (key == 'co19') { |
| 181 testSuites.add(new Co19TestSuite(conf)); | 187 testSuites.add(new Co19TestSuite(conf)); |
| 182 } else if (conf['compiler'] == 'none' && | 188 } else if (conf['compiler'] == 'none' && |
| 183 conf['runtime'] == 'vm' && | 189 conf['runtime'] == 'vm' && |
| 184 key == 'vm') { | 190 key == 'vm') { |
| 185 // vm tests contain both cc tests (added here) and dart tests (added | 191 // vm tests contain both cc tests (added here) and dart tests (added |
| 186 // in [TEST_SUITE_DIRECTORIES]). | 192 // in [TEST_SUITE_DIRECTORIES]). |
| 187 testSuites.add(new VMTestSuite(conf)); | 193 testSuites.add(new VMTestSuite(conf)); |
| 188 } else if (conf['analyzer']) { | 194 } else if (conf['analyzer']) { |
| 189 if (key == 'analyze_library') { | 195 if (key == 'analyze_library') { |
| 190 testSuites.add(new AnalyzeLibraryTestSuite(conf)); | 196 testSuites.add(new AnalyzeLibraryTestSuite(conf)); |
| 191 } | 197 } |
| 192 } else if (conf['compiler'] == 'none' && | 198 } else if (conf['compiler'] == 'none' && |
| 193 conf['runtime'] == 'vm' && | 199 conf['runtime'] == 'vm' && |
| 194 key == 'pkgbuild') { | 200 key == 'pkgbuild') { |
| 195 if (!conf['use_repository_packages'] && | 201 if (!conf['use_repository_packages'] && |
| 196 !conf['use_public_packages']) { | 202 !conf['use_public_packages']) { |
| 197 print("You need to use either --use-repository-packages or " | 203 print("You need to use either --use-repository-packages or " |
| 198 "--use-public-packages with the pkgbuild test suite!"); | 204 "--use-public-packages with the pkgbuild test suite!"); |
| 199 exit(1); | 205 exit(1); |
| 200 } | 206 } |
| 201 if (!conf['use_sdk']) { | 207 if (!conf['use_sdk']) { |
| 202 print("Running the 'pkgbuild' test suite requires " | 208 print("Running the 'pkgbuild' test suite requires " |
| 203 "passing the '--use-sdk' to test.py"); | 209 "passing the '--use-sdk' to test.py"); |
| 204 exit(1); | 210 exit(1); |
| 205 } | 211 } |
| 206 testSuites.add( | 212 testSuites.add( |
| 207 new PkgBuildTestSuite(conf, 'pkgbuild', 'pkg/pkgbuild.status')); | 213 new PkgBuildTestSuite(conf, 'pkgbuild', 'pkg/pkgbuild.status')); |
| 208 } | 214 } |
| 209 } | 215 } |
| 210 | 216 |
| 211 for (final testSuiteDir in TEST_SUITE_DIRECTORIES) { | 217 for (final testSuiteDir in TEST_SUITE_DIRECTORIES) { |
| 212 final name = testSuiteDir.filename; | 218 final name = testSuiteDir.filename; |
| 213 if (selectors.containsKey(name)) { | 219 if (selectors.containsKey(name)) { |
| 214 testSuites.add( | 220 testSuites |
| 215 new StandardTestSuite.forDirectory(conf, testSuiteDir)); | 221 .add(new StandardTestSuite.forDirectory(conf, testSuiteDir)); |
| 216 } | 222 } |
| 217 } | 223 } |
| 218 } | 224 } |
| 219 } | 225 } |
| 220 | 226 |
| 221 void allTestsFinished() { | 227 void allTestsFinished() { |
| 222 for (var conf in configurations) { | 228 for (var conf in configurations) { |
| 223 if (conf.containsKey('_servers_')) { | 229 if (conf.containsKey('_servers_')) { |
| 224 conf['_servers_'].stopServers(); | 230 conf['_servers_'].stopServers(); |
| 225 } | 231 } |
| (...skipping 19 matching lines...) Expand all Loading... |
| 245 eventListener.add(new StatusFileUpdatePrinter()); | 251 eventListener.add(new StatusFileUpdatePrinter()); |
| 246 } | 252 } |
| 247 eventListener.add(new SummaryPrinter()); | 253 eventListener.add(new SummaryPrinter()); |
| 248 eventListener.add(new FlakyLogWriter()); | 254 eventListener.add(new FlakyLogWriter()); |
| 249 if (printFailures) { | 255 if (printFailures) { |
| 250 // The buildbot has it's own failure summary since it needs to wrap it | 256 // The buildbot has it's own failure summary since it needs to wrap it |
| 251 // into '@@@'-annotated sections. | 257 // into '@@@'-annotated sections. |
| 252 var printFailureSummary = progressIndicator != 'buildbot'; | 258 var printFailureSummary = progressIndicator != 'buildbot'; |
| 253 eventListener.add(new TestFailurePrinter(printFailureSummary, formatter)); | 259 eventListener.add(new TestFailurePrinter(printFailureSummary, formatter)); |
| 254 } | 260 } |
| 255 eventListener.add(progressIndicatorFromName(progressIndicator, | 261 eventListener.add( |
| 256 startTime, | 262 progressIndicatorFromName(progressIndicator, startTime, formatter)); |
| 257 formatter)); | |
| 258 if (printTiming) { | 263 if (printTiming) { |
| 259 eventListener.add(new TimingPrinter(startTime)); | 264 eventListener.add(new TimingPrinter(startTime)); |
| 260 } | 265 } |
| 261 eventListener.add(new SkippedCompilationsPrinter()); | 266 eventListener.add(new SkippedCompilationsPrinter()); |
| 262 eventListener.add(new LeftOverTempDirPrinter()); | 267 eventListener.add(new LeftOverTempDirPrinter()); |
| 263 } | 268 } |
| 264 if (firstConf['write_test_outcome_log']) { | 269 if (firstConf['write_test_outcome_log']) { |
| 265 eventListener.add(new TestOutcomeLogWriter()); | 270 eventListener.add(new TestOutcomeLogWriter()); |
| 266 } | 271 } |
| 267 if (firstConf['copy_coredumps']) { | 272 if (firstConf['copy_coredumps']) { |
| 268 eventListener.add(new UnexpectedCrashDumpArchiver()); | 273 eventListener.add(new UnexpectedCrashDumpArchiver()); |
| 269 } | 274 } |
| 270 | 275 |
| 271 // The only progress indicator when listing tests should be the | 276 // The only progress indicator when listing tests should be the |
| 272 // the summary printer. | 277 // the summary printer. |
| 273 if (listTests) { | 278 if (listTests) { |
| 274 eventListener.add(new SummaryPrinter(jsonOnly: reportInJson)); | 279 eventListener.add(new SummaryPrinter(jsonOnly: reportInJson)); |
| 275 } else { | 280 } else { |
| 276 eventListener.add(new ExitCodeSetter()); | 281 eventListener.add(new ExitCodeSetter()); |
| 277 } | 282 } |
| 278 | 283 |
| 279 void startProcessQueue() { | 284 void startProcessQueue() { |
| 280 // [firstConf] is needed here, since the ProcessQueue needs to know the | 285 // [firstConf] is needed here, since the ProcessQueue needs to know the |
| 281 // settings of 'noBatch' and 'local_ip' | 286 // settings of 'noBatch' and 'local_ip' |
| 282 new ProcessQueue(firstConf, | 287 new ProcessQueue( |
| 283 maxProcesses, | 288 firstConf, |
| 284 maxBrowserProcesses, | 289 maxProcesses, |
| 285 startTime, | 290 maxBrowserProcesses, |
| 286 testSuites, | 291 startTime, |
| 287 eventListener, | 292 testSuites, |
| 288 allTestsFinished, | 293 eventListener, |
| 289 verbose, | 294 allTestsFinished, |
| 290 recordingPath, | 295 verbose, |
| 291 recordingOutputPath); | 296 recordingPath, |
| 297 recordingOutputPath); |
| 292 } | 298 } |
| 293 | 299 |
| 294 // Start all the HTTP servers required before starting the process queue. | 300 // Start all the HTTP servers required before starting the process queue. |
| 295 if (serverFutures.isEmpty) { | 301 if (serverFutures.isEmpty) { |
| 296 startProcessQueue(); | 302 startProcessQueue(); |
| 297 } else { | 303 } else { |
| 298 Future.wait(serverFutures).then((_) => startProcessQueue()); | 304 Future.wait(serverFutures).then((_) => startProcessQueue()); |
| 299 } | 305 } |
| 300 } | 306 } |
| OLD | NEW |