OLD | NEW |
1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file |
2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
4 | 4 |
5 import 'dart:async'; | 5 library test_configurations; |
| 6 |
| 7 import "dart:async"; |
| 8 import 'dart:convert'; |
6 import 'dart:io'; | 9 import 'dart:io'; |
7 import 'dart:math' as math; | 10 import "dart:math" as math; |
8 | 11 |
9 import 'android.dart'; | 12 import 'android.dart'; |
10 import 'browser_controller.dart'; | 13 import "browser_controller.dart"; |
11 import 'co19_test_config.dart'; | 14 import "co19_test_config.dart"; |
12 import 'configuration.dart'; | 15 import "http_server.dart"; |
13 import 'path.dart'; | 16 import "path.dart"; |
14 import 'test_progress.dart'; | 17 import "test_progress.dart"; |
15 import 'test_runner.dart'; | 18 import "test_runner.dart"; |
16 import 'test_suite.dart'; | 19 import "test_suite.dart"; |
17 import 'utils.dart'; | 20 import "utils.dart"; |
18 import 'vm_test_config.dart'; | 21 import "vm_test_config.dart"; |
19 | 22 |
20 /** | 23 /** |
21 * The directories that contain test suites which follow the conventions | 24 * The directories that contain test suites which follow the conventions |
22 * required by [StandardTestSuite]'s forDirectory constructor. | 25 * required by [StandardTestSuite]'s forDirectory constructor. |
23 * New test suites should follow this convention because it makes it much | 26 * New test suites should follow this convention because it makes it much |
24 * simpler to add them to test.dart. Existing test suites should be | 27 * simpler to add them to test.dart. Existing test suites should be |
25 * moved to here, if possible. | 28 * moved to here, if possible. |
26 */ | 29 */ |
27 final TEST_SUITE_DIRECTORIES = [ | 30 final TEST_SUITE_DIRECTORIES = [ |
28 new Path('pkg'), | 31 new Path('pkg'), |
(...skipping 19 matching lines...) Expand all Loading... |
48 new Path('tests/lib_strong'), | 51 new Path('tests/lib_strong'), |
49 new Path('tests/standalone'), | 52 new Path('tests/standalone'), |
50 new Path('tests/utils'), | 53 new Path('tests/utils'), |
51 new Path('utils/tests/css'), | 54 new Path('utils/tests/css'), |
52 new Path('utils/tests/peg'), | 55 new Path('utils/tests/peg'), |
53 ]; | 56 ]; |
54 | 57 |
55 // This file is created by gclient runhooks. | 58 // This file is created by gclient runhooks. |
56 final VS_TOOLCHAIN_FILE = new Path("build/win_toolchain.json"); | 59 final VS_TOOLCHAIN_FILE = new Path("build/win_toolchain.json"); |
57 | 60 |
58 Future testConfigurations(List<Configuration> configurations) async { | 61 Future testConfigurations(List<Map<String, dynamic>> configurations) async { |
59 var startTime = new DateTime.now(); | 62 var startTime = new DateTime.now(); |
60 // Extract global options from first configuration. | 63 // Extract global options from first configuration. |
61 var firstConf = configurations[0]; | 64 var firstConf = configurations[0]; |
62 var maxProcesses = firstConf.taskCount; | 65 var maxProcesses = firstConf['tasks'] as int; |
63 var progressIndicator = firstConf.progress; | 66 var progressIndicator = firstConf['progress'] as String; |
64 BuildbotProgressIndicator.stepName = firstConf.stepName; | 67 BuildbotProgressIndicator.stepName = firstConf['step_name'] as String; |
65 var verbose = firstConf.isVerbose; | 68 var verbose = firstConf['verbose'] as bool; |
66 var printTiming = firstConf.printTiming; | 69 var printTiming = firstConf['time'] as bool; |
67 var listTests = firstConf.listTests; | 70 var listTests = firstConf['list'] as bool; |
68 | 71 |
69 var reportInJson = firstConf.reportInJson; | 72 var reportInJson = firstConf['report_in_json'] as bool; |
70 var recordingPath = firstConf.recordingPath; | |
71 var replayPath = firstConf.replayPath; | |
72 | 73 |
73 Browser.resetBrowserConfiguration = firstConf.resetBrowser; | 74 var recordingPath = firstConf['record_to_file'] as String; |
| 75 var recordingOutputPath = firstConf['replay_from_file'] as String; |
74 | 76 |
75 if (recordingPath != null && replayPath != null) { | 77 Browser.resetBrowserConfiguration = |
| 78 firstConf['reset_browser_configuration'] as bool; |
| 79 |
| 80 if (recordingPath != null && recordingOutputPath != null) { |
76 print("Fatal: Can't have the '--record_to_file' and '--replay_from_file'" | 81 print("Fatal: Can't have the '--record_to_file' and '--replay_from_file'" |
77 "at the same time. Exiting ..."); | 82 "at the same time. Exiting ..."); |
78 exit(1); | 83 exit(1); |
79 } | 84 } |
80 | 85 |
81 if (!firstConf.appendLogs) { | 86 if (!(firstConf['append_logs'] as bool)) { |
82 var files = [ | 87 var files = [ |
83 new File(TestUtils.flakyFileName), | 88 new File(TestUtils.flakyFileName), |
84 new File(TestUtils.testOutcomeFileName) | 89 new File(TestUtils.testOutcomeFileName) |
85 ]; | 90 ]; |
86 for (var file in files) { | 91 for (var file in files) { |
87 if (file.existsSync()) { | 92 if (file.existsSync()) { |
88 file.deleteSync(); | 93 file.deleteSync(); |
89 } | 94 } |
90 } | 95 } |
91 } | 96 } |
92 | 97 |
93 DebugLogger.init(firstConf.writeDebugLog ? TestUtils.debugLogFilePath : null, | 98 DebugLogger.init( |
94 append: firstConf.appendLogs); | 99 firstConf['write_debug_log'] as bool ? TestUtils.debugLogFilePath : null, |
| 100 append: firstConf['append_logs'] as bool); |
95 | 101 |
96 // Print the configurations being run by this execution of | 102 // Print the configurations being run by this execution of |
97 // test.dart. However, don't do it if the silent progress indicator | 103 // test.dart. However, don't do it if the silent progress indicator |
98 // is used. This is only needed because of the junit tests. | 104 // is used. This is only needed because of the junit tests. |
99 if (progressIndicator != Progress.silent) { | 105 if (progressIndicator != 'silent') { |
100 var outputWords = configurations.length > 1 | 106 var outputWords = configurations.length > 1 |
101 ? ['Test configurations:'] | 107 ? ['Test configurations:'] |
102 : ['Test configuration:']; | 108 : ['Test configuration:']; |
103 | 109 for (Map conf in configurations) { |
104 for (var configuration in configurations) { | 110 List settings = ['compiler', 'runtime', 'mode', 'arch'] |
105 var settings = [ | 111 .map((name) => conf[name]) |
106 configuration.compiler.name, | 112 .toList(); |
107 configuration.runtime.name, | 113 if (conf['checked'] as bool) settings.add('checked'); |
108 configuration.mode.name, | 114 if (conf['strong'] as bool) settings.add('strong'); |
109 configuration.architecture.name | |
110 ]; | |
111 if (configuration.isChecked) settings.add('checked'); | |
112 if (configuration.isStrong) settings.add('strong'); | |
113 if (configuration.useFastStartup) settings.add('fast-startup'); | |
114 outputWords.add(settings.join('_')); | 115 outputWords.add(settings.join('_')); |
115 } | 116 } |
116 print(outputWords.join(' ')); | 117 print(outputWords.join(' ')); |
117 } | 118 } |
118 | 119 |
119 var runningBrowserTests = | 120 var runningBrowserTests = configurations.any((config) { |
120 configurations.any((config) => config.runtime.isBrowser); | 121 return TestUtils.isBrowserRuntime(config['runtime'] as String); |
| 122 }); |
121 | 123 |
122 var serverFutures = <Future>[]; | 124 List<Future> serverFutures = []; |
123 var testSuites = <TestSuite>[]; | 125 var testSuites = <TestSuite>[]; |
124 var maxBrowserProcesses = maxProcesses; | 126 var maxBrowserProcesses = maxProcesses; |
125 if (configurations.length > 1 && | 127 if (configurations.length > 1 && |
126 (configurations[0].testServerPort != 0 || | 128 (configurations[0]['test_server_port'] != 0 || |
127 configurations[0].testServerCrossOriginPort != 0)) { | 129 configurations[0]['test_server_cross_origin_port'] != 0)) { |
128 print("If the http server ports are specified, only one configuration" | 130 print("If the http server ports are specified, only one configuration" |
129 " may be run at a time"); | 131 " may be run at a time"); |
130 exit(1); | 132 exit(1); |
131 } | 133 } |
132 | 134 for (var conf in configurations) { |
133 for (var configuration in configurations) { | 135 var selectors = conf['selectors'] as Map<String, RegExp>; |
| 136 var useContentSecurityPolicy = conf['csp'] as bool; |
134 if (!listTests && runningBrowserTests) { | 137 if (!listTests && runningBrowserTests) { |
135 serverFutures.add(configuration.startServers()); | 138 // Start global http servers that serve the entire dart repo. |
| 139 // The http server is available on window.location.port, and a second |
| 140 // server for cross-domain tests can be found by calling |
| 141 // getCrossOriginPortNumber(). |
| 142 var servers = new TestingServers( |
| 143 TestUtils.buildDir(conf), |
| 144 useContentSecurityPolicy, |
| 145 conf['runtime'] as String, |
| 146 null, |
| 147 conf['package_root'] as String, |
| 148 conf['packages'] as String); |
| 149 serverFutures.add(servers.startServers(conf['local_ip'] as String, |
| 150 port: conf['test_server_port'] as int, |
| 151 crossOriginPort: conf['test_server_cross_origin_port'] as int)); |
| 152 conf['_servers_'] = servers; |
| 153 if (verbose) { |
| 154 serverFutures.last.then((_) { |
| 155 var commandline = servers.httpServerCommandLine(); |
| 156 print('Started HttpServers: $commandline'); |
| 157 }); |
| 158 } |
136 } | 159 } |
137 | 160 |
138 if (configuration.runtime.isIE) { | 161 if ((conf['runtime'] as String).startsWith('ie')) { |
139 // NOTE: We've experienced random timeouts of tests on ie9/ie10. The | 162 // NOTE: We've experienced random timeouts of tests on ie9/ie10. The |
140 // underlying issue has not been determined yet. Our current hypothesis | 163 // underlying issue has not been determined yet. Our current hypothesis |
141 // is that windows does not handle the IE processes independently. | 164 // is that windows does not handle the IE processes independently. |
142 // If we have more than one browser and kill a browser we are seeing | 165 // If we have more than one browser and kill a browser we are seeing |
143 // issues with starting up a new browser just after killing the hanging | 166 // issues with starting up a new browser just after killing the hanging |
144 // browser. | 167 // browser. |
145 maxBrowserProcesses = 1; | 168 maxBrowserProcesses = 1; |
146 } else if (configuration.runtime.isSafari) { | 169 } else if ((conf['runtime'] as String).startsWith('safari')) { |
147 // Safari does not allow us to run from a fresh profile, so we can only | 170 // Safari does not allow us to run from a fresh profile, so we can only |
148 // use one browser. Additionally, you can not start two simulators | 171 // use one browser. Additionally, you can not start two simulators |
149 // for mobile safari simultainiously. | 172 // for mobile safari simultainiously. |
150 maxBrowserProcesses = 1; | 173 maxBrowserProcesses = 1; |
151 } else if (configuration.runtime == Runtime.chrome && | 174 } else if ((conf['runtime'] as String) == 'chrome' && |
152 Platform.operatingSystem == 'macos') { | 175 Platform.operatingSystem == 'macos') { |
153 // Chrome on mac results in random timeouts. | 176 // Chrome on mac results in random timeouts. |
154 // Issue: https://github.com/dart-lang/sdk/issues/23891 | 177 // Issue: https://github.com/dart-lang/sdk/issues/23891 |
155 // This change does not fix the problem. | 178 // This change does not fix the problem. |
156 maxBrowserProcesses = math.max(1, maxBrowserProcesses ~/ 2); | 179 maxBrowserProcesses = math.max(1, maxBrowserProcesses ~/ 2); |
157 } else if (configuration.runtime != Runtime.drt) { | 180 } else if ((conf['runtime'] as String) != 'drt') { |
158 // Even on machines with more than 16 processors, don't open more | 181 // Even on machines with more than 16 processors, don't open more |
159 // than 15 browser instances, to avoid overloading the machine. | 182 // than 15 browser instances, to avoid overloading the machine. |
160 // This is especially important when running locally on powerful | 183 // This is especially important when running locally on powerful |
161 // desktops. | 184 // desktops. |
162 maxBrowserProcesses = math.min(maxBrowserProcesses, 15); | 185 maxBrowserProcesses = math.min(maxBrowserProcesses, 15); |
163 } | 186 } |
164 | 187 |
165 // If we specifically pass in a suite only run that. | 188 // If we specifically pass in a suite only run that. |
166 if (configuration.suiteDirectory != null) { | 189 if (conf['suite_dir'] != null) { |
167 var suitePath = new Path(configuration.suiteDirectory); | 190 var suite_path = new Path(conf['suite_dir'] as String); |
168 testSuites.add(new PKGTestSuite(configuration, suitePath)); | 191 testSuites.add(new PKGTestSuite(conf, suite_path)); |
169 } else { | 192 } else { |
170 for (var testSuiteDir in TEST_SUITE_DIRECTORIES) { | 193 for (final testSuiteDir in TEST_SUITE_DIRECTORIES) { |
171 var name = testSuiteDir.filename; | 194 final name = testSuiteDir.filename; |
172 if (configuration.selectors.containsKey(name)) { | 195 if (selectors.containsKey(name)) { |
173 testSuites.add( | 196 testSuites |
174 new StandardTestSuite.forDirectory(configuration, testSuiteDir)); | 197 .add(new StandardTestSuite.forDirectory(conf, testSuiteDir)); |
175 } | 198 } |
176 } | 199 } |
177 | 200 for (String key in selectors.keys) { |
178 for (var key in configuration.selectors.keys) { | |
179 if (key == 'co19') { | 201 if (key == 'co19') { |
180 testSuites.add(new Co19TestSuite(configuration)); | 202 testSuites.add(new Co19TestSuite(conf)); |
181 } else if (configuration.compiler == Compiler.none && | 203 } else if ((conf['compiler'] == 'dartk' || |
182 configuration.runtime == Runtime.vm && | 204 conf['compiler'] == 'none') && |
| 205 conf['runtime'] == 'vm' && |
183 key == 'vm') { | 206 key == 'vm') { |
184 // vm tests contain both cc tests (added here) and dart tests (added | 207 // vm tests contain both cc tests (added here) and dart tests (added |
185 // in [TEST_SUITE_DIRECTORIES]). | 208 // in [TEST_SUITE_DIRECTORIES]). |
186 testSuites.add(new VMTestSuite(configuration)); | 209 testSuites.add(new VMTestSuite(conf)); |
187 } else if (configuration.compiler == Compiler.dart2analyzer) { | 210 } else if (conf['analyzer'] as bool) { |
188 if (key == 'analyze_library') { | 211 if (key == 'analyze_library') { |
189 testSuites.add(new AnalyzeLibraryTestSuite(configuration)); | 212 testSuites.add(new AnalyzeLibraryTestSuite(conf)); |
190 } | 213 } |
191 } | 214 } |
192 } | 215 } |
193 } | 216 } |
194 } | 217 } |
195 | 218 |
196 void allTestsFinished() { | 219 void allTestsFinished() { |
197 for (var configuration in configurations) { | 220 for (var conf in configurations) { |
198 configuration.stopServers(); | 221 if (conf.containsKey('_servers_')) { |
| 222 conf['_servers_'].stopServers(); |
| 223 } |
199 } | 224 } |
200 | |
201 DebugLogger.close(); | 225 DebugLogger.close(); |
202 TestUtils.deleteTempSnapshotDirectory(configurations[0]); | 226 TestUtils.deleteTempSnapshotDirectory(configurations[0]); |
203 } | 227 } |
204 | 228 |
205 var eventListener = <EventListener>[]; | 229 var eventListener = <EventListener>[]; |
206 | 230 |
207 // We don't print progress if we list tests. | 231 // We don't print progress if we list tests. |
208 if (progressIndicator != Progress.silent && !listTests) { | 232 if (progressIndicator != 'silent' && !listTests) { |
209 var printFailures = true; | 233 var printFailures = true; |
210 var formatter = Formatter.normal; | 234 var formatter = Formatter.normal; |
211 if (progressIndicator == Progress.color) { | 235 if (progressIndicator == 'color') { |
212 progressIndicator = Progress.compact; | 236 progressIndicator = 'compact'; |
213 formatter = Formatter.color; | 237 formatter = Formatter.color; |
214 } | 238 } |
215 if (progressIndicator == Progress.diff) { | 239 if (progressIndicator == 'diff') { |
216 progressIndicator = Progress.compact; | 240 progressIndicator = 'compact'; |
217 formatter = Formatter.color; | 241 formatter = Formatter.color; |
218 printFailures = false; | 242 printFailures = false; |
219 eventListener.add(new StatusFileUpdatePrinter()); | 243 eventListener.add(new StatusFileUpdatePrinter()); |
220 } | 244 } |
221 eventListener.add(new SummaryPrinter()); | 245 eventListener.add(new SummaryPrinter()); |
222 eventListener.add(new FlakyLogWriter()); | 246 eventListener.add(new FlakyLogWriter()); |
223 if (printFailures) { | 247 if (printFailures) { |
224 // The buildbot has it's own failure summary since it needs to wrap it | 248 // The buildbot has it's own failure summary since it needs to wrap it |
225 // into '@@@'-annotated sections. | 249 // into '@@@'-annotated sections. |
226 var printFailureSummary = progressIndicator != Progress.buildbot; | 250 var printFailureSummary = progressIndicator != 'buildbot'; |
227 eventListener.add(new TestFailurePrinter(printFailureSummary, formatter)); | 251 eventListener.add(new TestFailurePrinter(printFailureSummary, formatter)); |
228 } | 252 } |
229 eventListener.add(ProgressIndicator.fromProgress( | 253 eventListener.add( |
230 progressIndicator, startTime, formatter)); | 254 ProgressIndicator.fromName(progressIndicator, startTime, formatter)); |
231 if (printTiming) { | 255 if (printTiming) { |
232 eventListener.add(new TimingPrinter(startTime)); | 256 eventListener.add(new TimingPrinter(startTime)); |
233 } | 257 } |
234 eventListener.add(new SkippedCompilationsPrinter()); | 258 eventListener.add(new SkippedCompilationsPrinter()); |
235 } | 259 } |
236 | 260 if (firstConf['write_test_outcome_log'] as bool) { |
237 if (firstConf.writeTestOutcomeLog) { | |
238 eventListener.add(new TestOutcomeLogWriter()); | 261 eventListener.add(new TestOutcomeLogWriter()); |
239 } | 262 } |
240 | 263 if (firstConf['copy_coredumps'] as bool) { |
241 if (firstConf.copyCoreDumps) { | |
242 eventListener.add(new UnexpectedCrashLogger()); | 264 eventListener.add(new UnexpectedCrashLogger()); |
243 } | 265 } |
244 | 266 |
245 // The only progress indicator when listing tests should be the | 267 // The only progress indicator when listing tests should be the |
246 // the summary printer. | 268 // the summary printer. |
247 if (listTests) { | 269 if (listTests) { |
248 eventListener.add(new SummaryPrinter(jsonOnly: reportInJson)); | 270 eventListener.add(new SummaryPrinter(jsonOnly: reportInJson)); |
249 } else { | 271 } else { |
250 eventListener.add(new ExitCodeSetter()); | 272 eventListener.add(new ExitCodeSetter()); |
251 eventListener.add(new IgnoredTestMonitor()); | 273 eventListener.add(new IgnoredTestMonitor()); |
252 } | 274 } |
253 | 275 |
254 // If any of the configurations need to access android devices we'll first | 276 // If any of the configurations need to access android devices we'll first |
255 // make a pool of all available adb devices. | 277 // make a pool of all available adb devices. |
256 AdbDevicePool adbDevicePool; | 278 AdbDevicePool adbDevicePool; |
257 var needsAdbDevicePool = configurations.any((conf) { | 279 var needsAdbDevicePool = configurations.any((Map conf) { |
258 return conf.runtime == Runtime.dartPrecompiled && | 280 return conf['runtime'] == 'dart_precompiled' && conf['system'] == 'android'; |
259 conf.system == System.android; | |
260 }); | 281 }); |
261 if (needsAdbDevicePool) { | 282 if (needsAdbDevicePool) { |
262 adbDevicePool = await AdbDevicePool.create(); | 283 adbDevicePool = await AdbDevicePool.create(); |
263 } | 284 } |
264 | 285 |
265 // Start all the HTTP servers required before starting the process queue. | 286 // Start all the HTTP servers required before starting the process queue. |
266 if (!serverFutures.isEmpty) { | 287 if (!serverFutures.isEmpty) { |
267 await Future.wait(serverFutures); | 288 await Future.wait(serverFutures); |
268 } | 289 } |
269 | 290 |
| 291 if (Platform.isWindows) { |
| 292 // When running tests on Windows, use cdb from depot_tools to dump |
| 293 // stack traces of tests timing out. |
| 294 try { |
| 295 var text = |
| 296 await new File(VS_TOOLCHAIN_FILE.toNativePath()).readAsString(); |
| 297 firstConf['win_sdk_path'] = JSON.decode(text)['win_sdk']; |
| 298 } on dynamic { |
| 299 // Ignore errors here. If win_sdk is not found, stack trace dumping |
| 300 // for timeouts won't work. |
| 301 } |
| 302 } |
| 303 |
270 // [firstConf] is needed here, since the ProcessQueue needs to know the | 304 // [firstConf] is needed here, since the ProcessQueue needs to know the |
271 // settings of 'noBatch' and 'local_ip' | 305 // settings of 'noBatch' and 'local_ip' |
272 new ProcessQueue( | 306 new ProcessQueue( |
273 firstConf, | 307 firstConf, |
274 maxProcesses, | 308 maxProcesses, |
275 maxBrowserProcesses, | 309 maxBrowserProcesses, |
276 startTime, | 310 startTime, |
277 testSuites, | 311 testSuites, |
278 eventListener, | 312 eventListener, |
279 allTestsFinished, | 313 allTestsFinished, |
280 verbose, | 314 verbose, |
281 recordingPath, | 315 recordingPath, |
282 replayPath, | 316 recordingOutputPath, |
283 adbDevicePool); | 317 adbDevicePool); |
284 } | 318 } |
OLD | NEW |