OLD | NEW |
---|---|
1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file |
2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
4 | 4 |
5 library test_configurations; | 5 import 'dart:async'; |
6 | |
7 import "dart:async"; | |
8 import 'dart:convert'; | |
9 import 'dart:io'; | 6 import 'dart:io'; |
10 import "dart:math" as math; | 7 import 'dart:math' as math; |
11 | 8 |
12 import 'android.dart'; | 9 import 'android.dart'; |
13 import "browser_controller.dart"; | 10 import 'browser_controller.dart'; |
14 import "co19_test_config.dart"; | 11 import 'co19_test_config.dart'; |
15 import "http_server.dart"; | 12 import 'configuration.dart'; |
16 import "path.dart"; | 13 import 'path.dart'; |
17 import "test_progress.dart"; | 14 import 'test_progress.dart'; |
18 import "test_runner.dart"; | 15 import 'test_runner.dart'; |
19 import "test_suite.dart"; | 16 import 'test_suite.dart'; |
20 import "utils.dart"; | 17 import 'utils.dart'; |
21 import "vm_test_config.dart"; | 18 import 'vm_test_config.dart'; |
22 | 19 |
23 /** | 20 /** |
24 * The directories that contain test suites which follow the conventions | 21 * The directories that contain test suites which follow the conventions |
25 * required by [StandardTestSuite]'s forDirectory constructor. | 22 * required by [StandardTestSuite]'s forDirectory constructor. |
26 * New test suites should follow this convention because it makes it much | 23 * New test suites should follow this convention because it makes it much |
27 * simpler to add them to test.dart. Existing test suites should be | 24 * simpler to add them to test.dart. Existing test suites should be |
28 * moved to here, if possible. | 25 * moved to here, if possible. |
29 */ | 26 */ |
30 final TEST_SUITE_DIRECTORIES = [ | 27 final TEST_SUITE_DIRECTORIES = [ |
31 new Path('pkg'), | 28 new Path('pkg'), |
(...skipping 19 matching lines...) Expand all Loading... | |
51 new Path('tests/lib_strong'), | 48 new Path('tests/lib_strong'), |
52 new Path('tests/standalone'), | 49 new Path('tests/standalone'), |
53 new Path('tests/utils'), | 50 new Path('tests/utils'), |
54 new Path('utils/tests/css'), | 51 new Path('utils/tests/css'), |
55 new Path('utils/tests/peg'), | 52 new Path('utils/tests/peg'), |
56 ]; | 53 ]; |
57 | 54 |
58 // This file is created by gclient runhooks. | 55 // This file is created by gclient runhooks. |
59 final VS_TOOLCHAIN_FILE = new Path("build/win_toolchain.json"); | 56 final VS_TOOLCHAIN_FILE = new Path("build/win_toolchain.json"); |
60 | 57 |
61 Future testConfigurations(List<Map<String, dynamic>> configurations) async { | 58 Future testConfigurations(List<Configuration> configurations) async { |
62 var startTime = new DateTime.now(); | 59 var startTime = new DateTime.now(); |
63 // Extract global options from first configuration. | 60 // Extract global options from first configuration. |
64 var firstConf = configurations[0]; | 61 var firstConf = configurations[0]; |
65 var maxProcesses = firstConf['tasks'] as int; | 62 var maxProcesses = firstConf.taskCount; |
66 var progressIndicator = firstConf['progress'] as String; | 63 var progressIndicator = firstConf.progress; |
67 BuildbotProgressIndicator.stepName = firstConf['step_name'] as String; | 64 BuildbotProgressIndicator.stepName = firstConf.stepName; |
68 var verbose = firstConf['verbose'] as bool; | 65 var verbose = firstConf.isVerbose; |
69 var printTiming = firstConf['time'] as bool; | 66 var printTiming = firstConf.printTiming; |
70 var listTests = firstConf['list'] as bool; | 67 var listTests = firstConf.listTests; |
71 | 68 |
72 var reportInJson = firstConf['report_in_json'] as bool; | 69 var reportInJson = firstConf.reportInJson; |
70 var recordingPath = firstConf.recordingPath; | |
71 var replayPath = firstConf.replayPath; | |
73 | 72 |
74 var recordingPath = firstConf['record_to_file'] as String; | 73 Browser.resetBrowserConfiguration = firstConf.resetBrowser; |
75 var recordingOutputPath = firstConf['replay_from_file'] as String; | |
76 | 74 |
77 Browser.resetBrowserConfiguration = | 75 if (recordingPath != null && replayPath != null) { |
78 firstConf['reset_browser_configuration'] as bool; | |
79 | |
80 if (recordingPath != null && recordingOutputPath != null) { | |
81 print("Fatal: Can't have the '--record_to_file' and '--replay_from_file'" | 76 print("Fatal: Can't have the '--record_to_file' and '--replay_from_file'" |
82 "at the same time. Exiting ..."); | 77 "at the same time. Exiting ..."); |
83 exit(1); | 78 exit(1); |
84 } | 79 } |
85 | 80 |
86 if (!(firstConf['append_logs'] as bool)) { | 81 if (!firstConf.appendLogs) { |
87 var files = [ | 82 var files = [ |
88 new File(TestUtils.flakyFileName), | 83 new File(TestUtils.flakyFileName), |
89 new File(TestUtils.testOutcomeFileName) | 84 new File(TestUtils.testOutcomeFileName) |
90 ]; | 85 ]; |
91 for (var file in files) { | 86 for (var file in files) { |
92 if (file.existsSync()) { | 87 if (file.existsSync()) { |
93 file.deleteSync(); | 88 file.deleteSync(); |
94 } | 89 } |
95 } | 90 } |
96 } | 91 } |
97 | 92 |
98 DebugLogger.init( | 93 DebugLogger.init(firstConf.writeDebugLog ? TestUtils.debugLogFilePath : null, |
99 firstConf['write_debug_log'] as bool ? TestUtils.debugLogFilePath : null, | 94 append: firstConf.appendLogs); |
100 append: firstConf['append_logs'] as bool); | |
101 | 95 |
102 // Print the configurations being run by this execution of | 96 // Print the configurations being run by this execution of |
103 // test.dart. However, don't do it if the silent progress indicator | 97 // test.dart. However, don't do it if the silent progress indicator |
104 // is used. This is only needed because of the junit tests. | 98 // is used. This is only needed because of the junit tests. |
105 if (progressIndicator != 'silent') { | 99 if (progressIndicator != Progress.silent) { |
106 var outputWords = configurations.length > 1 | 100 var outputWords = configurations.length > 1 |
107 ? ['Test configurations:'] | 101 ? ['Test configurations:'] |
108 : ['Test configuration:']; | 102 : ['Test configuration:']; |
109 for (Map conf in configurations) { | 103 |
110 List settings = ['compiler', 'runtime', 'mode', 'arch'] | 104 for (var configuration in configurations) { |
111 .map((name) => conf[name]) | 105 var settings = [ |
112 .toList(); | 106 configuration.compiler.name, |
113 if (conf['checked'] as bool) settings.add('checked'); | 107 configuration.runtime.name, |
114 if (conf['strong'] as bool) settings.add('strong'); | 108 configuration.mode.name, |
109 configuration.architecture.name | |
110 ]; | |
111 if (configuration.isChecked) settings.add('checked'); | |
112 if (configuration.isStrong) settings.add('strong'); | |
Bill Hesse
2017/05/29 13:08:28
Please add printing of "fast-startup" if fast star
Bob Nystrom
2017/05/30 23:29:31
Done.
| |
115 outputWords.add(settings.join('_')); | 113 outputWords.add(settings.join('_')); |
116 } | 114 } |
117 print(outputWords.join(' ')); | 115 print(outputWords.join(' ')); |
118 } | 116 } |
119 | 117 |
120 var runningBrowserTests = configurations.any((config) { | 118 var runningBrowserTests = |
121 return TestUtils.isBrowserRuntime(config['runtime'] as String); | 119 configurations.any((config) => config.runtime.isBrowser); |
122 }); | |
123 | 120 |
124 List<Future> serverFutures = []; | 121 var serverFutures = <Future>[]; |
125 var testSuites = <TestSuite>[]; | 122 var testSuites = <TestSuite>[]; |
126 var maxBrowserProcesses = maxProcesses; | 123 var maxBrowserProcesses = maxProcesses; |
127 if (configurations.length > 1 && | 124 if (configurations.length > 1 && |
128 (configurations[0]['test_server_port'] != 0 || | 125 (configurations[0].testServerPort != 0 || |
129 configurations[0]['test_server_cross_origin_port'] != 0)) { | 126 configurations[0].testServerCrossOriginPort != 0)) { |
130 print("If the http server ports are specified, only one configuration" | 127 print("If the http server ports are specified, only one configuration" |
131 " may be run at a time"); | 128 " may be run at a time"); |
132 exit(1); | 129 exit(1); |
133 } | 130 } |
134 for (var conf in configurations) { | 131 |
135 var selectors = conf['selectors'] as Map<String, RegExp>; | 132 for (var configuration in configurations) { |
136 var useContentSecurityPolicy = conf['csp'] as bool; | |
137 if (!listTests && runningBrowserTests) { | 133 if (!listTests && runningBrowserTests) { |
138 // Start global http servers that serve the entire dart repo. | 134 serverFutures.add(configuration.startServers()); |
139 // The http server is available on window.location.port, and a second | |
140 // server for cross-domain tests can be found by calling | |
141 // getCrossOriginPortNumber(). | |
142 var servers = new TestingServers( | |
143 TestUtils.buildDir(conf), | |
144 useContentSecurityPolicy, | |
145 conf['runtime'] as String, | |
146 null, | |
147 conf['package_root'] as String, | |
148 conf['packages'] as String); | |
149 serverFutures.add(servers.startServers(conf['local_ip'] as String, | |
150 port: conf['test_server_port'] as int, | |
151 crossOriginPort: conf['test_server_cross_origin_port'] as int)); | |
152 conf['_servers_'] = servers; | |
153 if (verbose) { | |
154 serverFutures.last.then((_) { | |
155 var commandline = servers.httpServerCommandLine(); | |
156 print('Started HttpServers: $commandline'); | |
157 }); | |
158 } | |
159 } | 135 } |
160 | 136 |
161 if ((conf['runtime'] as String).startsWith('ie')) { | 137 if (configuration.runtime.isIE) { |
162 // NOTE: We've experienced random timeouts of tests on ie9/ie10. The | 138 // NOTE: We've experienced random timeouts of tests on ie9/ie10. The |
163 // underlying issue has not been determined yet. Our current hypothesis | 139 // underlying issue has not been determined yet. Our current hypothesis |
164 // is that windows does not handle the IE processes independently. | 140 // is that windows does not handle the IE processes independently. |
165 // If we have more than one browser and kill a browser we are seeing | 141 // If we have more than one browser and kill a browser we are seeing |
166 // issues with starting up a new browser just after killing the hanging | 142 // issues with starting up a new browser just after killing the hanging |
167 // browser. | 143 // browser. |
168 maxBrowserProcesses = 1; | 144 maxBrowserProcesses = 1; |
169 } else if ((conf['runtime'] as String).startsWith('safari')) { | 145 } else if (configuration.runtime.isSafari) { |
170 // Safari does not allow us to run from a fresh profile, so we can only | 146 // Safari does not allow us to run from a fresh profile, so we can only |
171 // use one browser. Additionally, you can not start two simulators | 147 // use one browser. Additionally, you can not start two simulators |
172 // for mobile safari simultainiously. | 148 // for mobile safari simultainiously. |
173 maxBrowserProcesses = 1; | 149 maxBrowserProcesses = 1; |
174 } else if ((conf['runtime'] as String) == 'chrome' && | 150 } else if (configuration.runtime == Runtime.chrome && |
175 Platform.operatingSystem == 'macos') { | 151 Platform.operatingSystem == 'macos') { |
176 // Chrome on mac results in random timeouts. | 152 // Chrome on mac results in random timeouts. |
177 // Issue: https://github.com/dart-lang/sdk/issues/23891 | 153 // Issue: https://github.com/dart-lang/sdk/issues/23891 |
178 // This change does not fix the problem. | 154 // This change does not fix the problem. |
179 maxBrowserProcesses = math.max(1, maxBrowserProcesses ~/ 2); | 155 maxBrowserProcesses = math.max(1, maxBrowserProcesses ~/ 2); |
180 } else if ((conf['runtime'] as String) != 'drt') { | 156 } else if (configuration.runtime != Runtime.drt) { |
181 // Even on machines with more than 16 processors, don't open more | 157 // Even on machines with more than 16 processors, don't open more |
182 // than 15 browser instances, to avoid overloading the machine. | 158 // than 15 browser instances, to avoid overloading the machine. |
183 // This is especially important when running locally on powerful | 159 // This is especially important when running locally on powerful |
184 // desktops. | 160 // desktops. |
185 maxBrowserProcesses = math.min(maxBrowserProcesses, 15); | 161 maxBrowserProcesses = math.min(maxBrowserProcesses, 15); |
186 } | 162 } |
187 | 163 |
188 // If we specifically pass in a suite only run that. | 164 // If we specifically pass in a suite only run that. |
189 if (conf['suite_dir'] != null) { | 165 if (configuration.suiteDirectory != null) { |
190 var suite_path = new Path(conf['suite_dir'] as String); | 166 var suitePath = new Path(configuration.suiteDirectory); |
191 testSuites.add(new PKGTestSuite(conf, suite_path)); | 167 testSuites.add(new PKGTestSuite(configuration, suitePath)); |
192 } else { | 168 } else { |
193 for (final testSuiteDir in TEST_SUITE_DIRECTORIES) { | 169 for (var testSuiteDir in TEST_SUITE_DIRECTORIES) { |
194 final name = testSuiteDir.filename; | 170 var name = testSuiteDir.filename; |
195 if (selectors.containsKey(name)) { | 171 if (configuration.selectors.containsKey(name)) { |
196 testSuites | 172 testSuites.add( |
197 .add(new StandardTestSuite.forDirectory(conf, testSuiteDir)); | 173 new StandardTestSuite.forDirectory(configuration, testSuiteDir)); |
198 } | 174 } |
199 } | 175 } |
200 for (String key in selectors.keys) { | 176 |
177 for (var key in configuration.selectors.keys) { | |
201 if (key == 'co19') { | 178 if (key == 'co19') { |
202 testSuites.add(new Co19TestSuite(conf)); | 179 testSuites.add(new Co19TestSuite(configuration)); |
203 } else if (conf['compiler'] == 'none' && | 180 } else if (configuration.compiler == Compiler.none && |
204 conf['runtime'] == 'vm' && | 181 configuration.runtime == Runtime.vm && |
205 key == 'vm') { | 182 key == 'vm') { |
206 // vm tests contain both cc tests (added here) and dart tests (added | 183 // vm tests contain both cc tests (added here) and dart tests (added |
207 // in [TEST_SUITE_DIRECTORIES]). | 184 // in [TEST_SUITE_DIRECTORIES]). |
208 testSuites.add(new VMTestSuite(conf)); | 185 testSuites.add(new VMTestSuite(configuration)); |
209 } else if (conf['analyzer'] as bool) { | 186 } else if (configuration.compiler == Compiler.dart2analyzer) { |
210 if (key == 'analyze_library') { | 187 if (key == 'analyze_library') { |
211 testSuites.add(new AnalyzeLibraryTestSuite(conf)); | 188 testSuites.add(new AnalyzeLibraryTestSuite(configuration)); |
212 } | 189 } |
213 } | 190 } |
214 } | 191 } |
215 } | 192 } |
216 } | 193 } |
217 | 194 |
218 void allTestsFinished() { | 195 void allTestsFinished() { |
219 for (var conf in configurations) { | 196 for (var configuration in configurations) { |
220 if (conf.containsKey('_servers_')) { | 197 configuration.stopServers(); |
221 conf['_servers_'].stopServers(); | |
222 } | |
223 } | 198 } |
199 | |
224 DebugLogger.close(); | 200 DebugLogger.close(); |
225 TestUtils.deleteTempSnapshotDirectory(configurations[0]); | 201 TestUtils.deleteTempSnapshotDirectory(configurations[0]); |
226 } | 202 } |
227 | 203 |
228 var eventListener = <EventListener>[]; | 204 var eventListener = <EventListener>[]; |
229 | 205 |
230 // We don't print progress if we list tests. | 206 // We don't print progress if we list tests. |
231 if (progressIndicator != 'silent' && !listTests) { | 207 if (progressIndicator != Progress.silent && !listTests) { |
232 var printFailures = true; | 208 var printFailures = true; |
233 var formatter = Formatter.normal; | 209 var formatter = Formatter.normal; |
234 if (progressIndicator == 'color') { | 210 if (progressIndicator == Progress.color) { |
235 progressIndicator = 'compact'; | 211 progressIndicator = Progress.compact; |
236 formatter = Formatter.color; | 212 formatter = Formatter.color; |
237 } | 213 } |
238 if (progressIndicator == 'diff') { | 214 if (progressIndicator == Progress.diff) { |
239 progressIndicator = 'compact'; | 215 progressIndicator = Progress.compact; |
240 formatter = Formatter.color; | 216 formatter = Formatter.color; |
241 printFailures = false; | 217 printFailures = false; |
242 eventListener.add(new StatusFileUpdatePrinter()); | 218 eventListener.add(new StatusFileUpdatePrinter()); |
243 } | 219 } |
244 eventListener.add(new SummaryPrinter()); | 220 eventListener.add(new SummaryPrinter()); |
245 eventListener.add(new FlakyLogWriter()); | 221 eventListener.add(new FlakyLogWriter()); |
246 if (printFailures) { | 222 if (printFailures) { |
247 // The buildbot has it's own failure summary since it needs to wrap it | 223 // The buildbot has it's own failure summary since it needs to wrap it |
248 // into '@@@'-annotated sections. | 224 // into '@@@'-annotated sections. |
249 var printFailureSummary = progressIndicator != 'buildbot'; | 225 var printFailureSummary = progressIndicator != Progress.buildbot; |
250 eventListener.add(new TestFailurePrinter(printFailureSummary, formatter)); | 226 eventListener.add(new TestFailurePrinter(printFailureSummary, formatter)); |
251 } | 227 } |
252 eventListener.add( | 228 eventListener.add(ProgressIndicator.fromProgress( |
253 ProgressIndicator.fromName(progressIndicator, startTime, formatter)); | 229 progressIndicator, startTime, formatter)); |
254 if (printTiming) { | 230 if (printTiming) { |
255 eventListener.add(new TimingPrinter(startTime)); | 231 eventListener.add(new TimingPrinter(startTime)); |
256 } | 232 } |
257 eventListener.add(new SkippedCompilationsPrinter()); | 233 eventListener.add(new SkippedCompilationsPrinter()); |
258 } | 234 } |
259 if (firstConf['write_test_outcome_log'] as bool) { | 235 |
236 if (firstConf.writeTestOutcomeLog) { | |
260 eventListener.add(new TestOutcomeLogWriter()); | 237 eventListener.add(new TestOutcomeLogWriter()); |
261 } | 238 } |
262 if (firstConf['copy_coredumps'] as bool) { | 239 |
240 if (firstConf.copyCoreDumps) { | |
263 eventListener.add(new UnexpectedCrashLogger()); | 241 eventListener.add(new UnexpectedCrashLogger()); |
264 } | 242 } |
265 | 243 |
266 // The only progress indicator when listing tests should be the | 244 // The only progress indicator when listing tests should be the |
267 // the summary printer. | 245 // the summary printer. |
268 if (listTests) { | 246 if (listTests) { |
269 eventListener.add(new SummaryPrinter(jsonOnly: reportInJson)); | 247 eventListener.add(new SummaryPrinter(jsonOnly: reportInJson)); |
270 } else { | 248 } else { |
271 eventListener.add(new ExitCodeSetter()); | 249 eventListener.add(new ExitCodeSetter()); |
272 eventListener.add(new IgnoredTestMonitor()); | 250 eventListener.add(new IgnoredTestMonitor()); |
273 } | 251 } |
274 | 252 |
275 // If any of the configurations need to access android devices we'll first | 253 // If any of the configurations need to access android devices we'll first |
276 // make a pool of all available adb devices. | 254 // make a pool of all available adb devices. |
277 AdbDevicePool adbDevicePool; | 255 AdbDevicePool adbDevicePool; |
278 var needsAdbDevicePool = configurations.any((Map conf) { | 256 var needsAdbDevicePool = configurations.any((conf) { |
279 return conf['runtime'] == 'dart_precompiled' && conf['system'] == 'android'; | 257 return conf.runtime == Runtime.dartPrecompiled && |
258 conf.system == System.android; | |
280 }); | 259 }); |
281 if (needsAdbDevicePool) { | 260 if (needsAdbDevicePool) { |
282 adbDevicePool = await AdbDevicePool.create(); | 261 adbDevicePool = await AdbDevicePool.create(); |
283 } | 262 } |
284 | 263 |
285 // Start all the HTTP servers required before starting the process queue. | 264 // Start all the HTTP servers required before starting the process queue. |
286 if (!serverFutures.isEmpty) { | 265 if (!serverFutures.isEmpty) { |
287 await Future.wait(serverFutures); | 266 await Future.wait(serverFutures); |
288 } | 267 } |
289 | 268 |
290 if (Platform.isWindows) { | |
291 // When running tests on Windows, use cdb from depot_tools to dump | |
292 // stack traces of tests timing out. | |
293 try { | |
294 var text = | |
295 await new File(VS_TOOLCHAIN_FILE.toNativePath()).readAsString(); | |
296 firstConf['win_sdk_path'] = JSON.decode(text)['win_sdk']; | |
297 } on dynamic { | |
298 // Ignore errors here. If win_sdk is not found, stack trace dumping | |
299 // for timeouts won't work. | |
300 } | |
301 } | |
302 | |
303 // [firstConf] is needed here, since the ProcessQueue needs to know the | 269 // [firstConf] is needed here, since the ProcessQueue needs to know the |
304 // settings of 'noBatch' and 'local_ip' | 270 // settings of 'noBatch' and 'local_ip' |
305 new ProcessQueue( | 271 new ProcessQueue( |
306 firstConf, | 272 firstConf, |
307 maxProcesses, | 273 maxProcesses, |
308 maxBrowserProcesses, | 274 maxBrowserProcesses, |
309 startTime, | 275 startTime, |
310 testSuites, | 276 testSuites, |
311 eventListener, | 277 eventListener, |
312 allTestsFinished, | 278 allTestsFinished, |
313 verbose, | 279 verbose, |
314 recordingPath, | 280 recordingPath, |
315 recordingOutputPath, | 281 replayPath, |
316 adbDevicePool); | 282 adbDevicePool); |
317 } | 283 } |
OLD | NEW |