OLD | NEW |
1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file |
2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
4 | 4 |
5 library test_configurations; | 5 library test_configurations; |
6 | 6 |
7 import "dart:async"; | 7 import "dart:async"; |
8 import 'dart:io'; | 8 import 'dart:io'; |
9 import "dart:math" as math; | 9 import "dart:math" as math; |
10 | 10 |
11 import "browser_controller.dart"; | 11 import "browser_controller.dart"; |
12 import "co19_test_config.dart"; | 12 import "co19_test_config.dart"; |
13 import "http_server.dart"; | 13 import "http_server.dart"; |
14 import "path.dart"; | 14 import "path.dart"; |
15 import "test_progress.dart"; | 15 import "test_progress.dart"; |
16 import "test_runner.dart"; | 16 import "test_runner.dart"; |
17 import "test_suite.dart"; | 17 import "test_suite.dart"; |
18 import "utils.dart"; | 18 import "utils.dart"; |
19 import "vm_test_config.dart"; | 19 import "vm_test_config.dart"; |
20 | 20 |
21 /** | 21 /** |
22 * The directories that contain test suites which follow the conventions | 22 * The directories that contain test suites which follow the conventions |
23 * required by [StandardTestSuite]'s forDirectory constructor. | 23 * required by [StandardTestSuite]'s forDirectory constructor. |
24 * New test suites should follow this convention because it makes it much | 24 * New test suites should follow this convention because it makes it much |
25 * simpler to add them to test.dart. Existing test suites should be | 25 * simpler to add them to test.dart. Existing test suites should be |
26 * moved to here, if possible. | 26 * moved to here, if possible. |
27 */ | 27 */ |
28 final TEST_SUITE_DIRECTORIES = [ | 28 final TEST_SUITE_DIRECTORIES = [ |
29 new Path('pkg'), | 29 new Path('pkg'), |
30 new Path('third_party/pkg_tested'), | 30 new Path('third_party/pkg_tested'), |
31 new Path('runtime/tests/vm'), | 31 new Path('runtime/tests/vm'), |
32 new Path('runtime/observatory/tests/service'), | 32 new Path('runtime/observatory/tests/service'), |
33 new Path('samples'), | 33 new Path('samples'), |
34 new Path('samples-dev'), | 34 new Path('samples-dev'), |
35 new Path('tests/benchmark_smoke'), | 35 new Path('tests/benchmark_smoke'), |
36 new Path('tests/chrome'), | 36 new Path('tests/chrome'), |
37 new Path('tests/compiler/dart2js'), | 37 new Path('tests/compiler/dart2js'), |
38 new Path('tests/compiler/dart2js_extra'), | 38 new Path('tests/compiler/dart2js_extra'), |
39 new Path('tests/compiler/dart2js_native'), | 39 new Path('tests/compiler/dart2js_native'), |
40 new Path('tests/corelib'), | 40 new Path('tests/corelib'), |
41 new Path('tests/html'), | 41 new Path('tests/html'), |
42 new Path('tests/isolate'), | 42 new Path('tests/isolate'), |
43 new Path('tests/language'), | 43 new Path('tests/language'), |
44 new Path('tests/lib'), | 44 new Path('tests/lib'), |
45 new Path('tests/standalone'), | 45 new Path('tests/standalone'), |
46 new Path('tests/try'), | 46 new Path('tests/try'), |
47 new Path('tests/utils'), | 47 new Path('tests/utils'), |
48 new Path('utils/tests/css'), | 48 new Path('utils/tests/css'), |
49 new Path('utils/tests/peg'), | 49 new Path('utils/tests/peg'), |
50 ]; | 50 ]; |
51 | 51 |
52 void testConfigurations(List<Map> configurations) { | 52 void testConfigurations(List<Map> configurations) { |
53 var startTime = new DateTime.now(); | 53 var startTime = new DateTime.now(); |
54 // Extract global options from first configuration. | 54 // Extract global options from first configuration. |
55 var firstConf = configurations[0]; | 55 var firstConf = configurations[0]; |
56 var maxProcesses = firstConf['tasks']; | 56 var maxProcesses = firstConf['tasks']; |
57 var progressIndicator = firstConf['progress']; | 57 var progressIndicator = firstConf['progress']; |
58 // TODO(kustermann): Remove this option once the buildbots don't use it | 58 // TODO(kustermann): Remove this option once the buildbots don't use it |
59 // anymore. | 59 // anymore. |
60 var failureSummary = firstConf['failure-summary']; | 60 var failureSummary = firstConf['failure-summary']; |
61 BuildbotProgressIndicator.stepName = firstConf['step_name']; | 61 BuildbotProgressIndicator.stepName = firstConf['step_name']; |
62 var verbose = firstConf['verbose']; | 62 var verbose = firstConf['verbose']; |
63 var printTiming = firstConf['time']; | 63 var printTiming = firstConf['time']; |
64 var listTests = firstConf['list']; | 64 var listTests = firstConf['list']; |
65 | 65 |
66 var reportInJson = firstConf['report_in_json']; | 66 var reportInJson = firstConf['report_in_json']; |
67 | 67 |
68 var recordingPath = firstConf['record_to_file']; | 68 var recordingPath = firstConf['record_to_file']; |
69 var recordingOutputPath = firstConf['replay_from_file']; | 69 var recordingOutputPath = firstConf['replay_from_file']; |
70 | 70 |
71 Browser.deleteCache = firstConf['clear_browser_cache']; | 71 Browser.deleteCache = firstConf['clear_browser_cache']; |
72 | 72 |
73 if (recordingPath != null && recordingOutputPath != null) { | 73 if (recordingPath != null && recordingOutputPath != null) { |
74 print("Fatal: Can't have the '--record_to_file' and '--replay_from_file'" | 74 print("Fatal: Can't have the '--record_to_file' and '--replay_from_file'" |
75 "at the same time. Exiting ..."); | 75 "at the same time. Exiting ..."); |
76 exit(1); | 76 exit(1); |
77 } | 77 } |
78 | 78 |
79 if (!firstConf['append_logs']) { | 79 if (!firstConf['append_logs']) { |
80 var files = [new File(TestUtils.flakyFileName()), | 80 var files = [ |
81 new File(TestUtils.testOutcomeFileName())]; | 81 new File(TestUtils.flakyFileName()), |
| 82 new File(TestUtils.testOutcomeFileName()) |
| 83 ]; |
82 for (var file in files) { | 84 for (var file in files) { |
83 if (file.existsSync()) { | 85 if (file.existsSync()) { |
84 file.deleteSync(); | 86 file.deleteSync(); |
85 } | 87 } |
86 } | 88 } |
87 } | 89 } |
88 | 90 |
89 DebugLogger.init(firstConf['write_debug_log'] ? | 91 DebugLogger.init( |
90 TestUtils.debugLogfile() : null, append: firstConf['append_logs']); | 92 firstConf['write_debug_log'] ? TestUtils.debugLogfile() : null, |
| 93 append: firstConf['append_logs']); |
91 | 94 |
92 // Print the configurations being run by this execution of | 95 // Print the configurations being run by this execution of |
93 // test.dart. However, don't do it if the silent progress indicator | 96 // test.dart. However, don't do it if the silent progress indicator |
94 // is used. This is only needed because of the junit tests. | 97 // is used. This is only needed because of the junit tests. |
95 if (progressIndicator != 'silent') { | 98 if (progressIndicator != 'silent') { |
96 List output_words = configurations.length > 1 ? | 99 List output_words = configurations.length > 1 |
97 ['Test configurations:'] : ['Test configuration:']; | 100 ? ['Test configurations:'] |
| 101 : ['Test configuration:']; |
98 for (Map conf in configurations) { | 102 for (Map conf in configurations) { |
99 List settings = ['compiler', 'runtime', 'mode', 'arch'] | 103 List settings = ['compiler', 'runtime', 'mode', 'arch'] |
100 .map((name) => conf[name]).toList(); | 104 .map((name) => conf[name]) |
| 105 .toList(); |
101 if (conf['checked']) settings.add('checked'); | 106 if (conf['checked']) settings.add('checked'); |
102 if (conf['noopt']) settings.add('noopt'); | 107 if (conf['noopt']) settings.add('noopt'); |
103 output_words.add(settings.join('_')); | 108 output_words.add(settings.join('_')); |
104 } | 109 } |
105 print(output_words.join(' ')); | 110 print(output_words.join(' ')); |
106 } | 111 } |
107 | 112 |
108 var runningBrowserTests = configurations.any((config) { | 113 var runningBrowserTests = configurations.any((config) { |
109 return TestUtils.isBrowserRuntime(config['runtime']); | 114 return TestUtils.isBrowserRuntime(config['runtime']); |
110 }); | 115 }); |
111 | 116 |
112 List<Future> serverFutures = []; | 117 List<Future> serverFutures = []; |
113 var testSuites = new List<TestSuite>(); | 118 var testSuites = new List<TestSuite>(); |
114 var maxBrowserProcesses = maxProcesses; | 119 var maxBrowserProcesses = maxProcesses; |
115 if (configurations.length > 1 && | 120 if (configurations.length > 1 && |
116 (configurations[0]['test_server_port'] != 0 || | 121 (configurations[0]['test_server_port'] != 0 || |
117 configurations[0]['test_server_cross_origin_port'] != 0)) { | 122 configurations[0]['test_server_cross_origin_port'] != 0)) { |
118 print("If the http server ports are specified, only one configuration" | 123 print("If the http server ports are specified, only one configuration" |
119 " may be run at a time"); | 124 " may be run at a time"); |
120 exit(1); | 125 exit(1); |
121 } | 126 } |
122 for (var conf in configurations) { | 127 for (var conf in configurations) { |
123 Map<String, RegExp> selectors = conf['selectors']; | 128 Map<String, RegExp> selectors = conf['selectors']; |
124 var useContentSecurityPolicy = conf['csp']; | 129 var useContentSecurityPolicy = conf['csp']; |
125 if (!listTests && runningBrowserTests) { | 130 if (!listTests && runningBrowserTests) { |
126 // Start global http servers that serve the entire dart repo. | 131 // Start global http servers that serve the entire dart repo. |
127 // The http server is available on window.location.port, and a second | 132 // The http server is available on window.location.port, and a second |
128 // server for cross-domain tests can be found by calling | 133 // server for cross-domain tests can be found by calling |
129 // getCrossOriginPortNumber(). | 134 // getCrossOriginPortNumber(). |
130 var servers = new TestingServers(new Path(TestUtils.buildDir(conf)), | 135 var servers = new TestingServers( |
131 useContentSecurityPolicy, | 136 new Path(TestUtils.buildDir(conf)), |
132 conf['runtime'], | 137 useContentSecurityPolicy, |
133 null, | 138 conf['runtime'], |
134 conf['package_root']); | 139 null, |
| 140 conf['package_root']); |
135 serverFutures.add(servers.startServers(conf['local_ip'], | 141 serverFutures.add(servers.startServers(conf['local_ip'], |
136 port: conf['test_server_port'], | 142 port: conf['test_server_port'], |
137 crossOriginPort: conf['test_server_cross_origin_port'])); | 143 crossOriginPort: conf['test_server_cross_origin_port'])); |
138 conf['_servers_'] = servers; | 144 conf['_servers_'] = servers; |
139 if (verbose) { | 145 if (verbose) { |
140 serverFutures.last.then((_) { | 146 serverFutures.last.then((_) { |
141 var commandline = servers.httpServerCommandline(); | 147 var commandline = servers.httpServerCommandline(); |
142 print('Started HttpServers: $commandline'); | 148 print('Started HttpServers: $commandline'); |
143 }); | 149 }); |
144 } | 150 } |
145 } | 151 } |
146 | 152 |
147 if (conf['runtime'].startsWith('ie')) { | 153 if (conf['runtime'].startsWith('ie')) { |
148 // NOTE: We've experienced random timeouts of tests on ie9/ie10. The | 154 // NOTE: We've experienced random timeouts of tests on ie9/ie10. The |
149 // underlying issue has not been determined yet. Our current hypothesis | 155 // underlying issue has not been determined yet. Our current hypothesis |
150 // is that windows does not handle the IE processes independently. | 156 // is that windows does not handle the IE processes independently. |
151 // If we have more than one browser and kill a browser we are seeing | 157 // If we have more than one browser and kill a browser we are seeing |
152 // issues with starting up a new browser just after killing the hanging | 158 // issues with starting up a new browser just after killing the hanging |
153 // browser. | 159 // browser. |
154 maxBrowserProcesses = 1; | 160 maxBrowserProcesses = 1; |
155 } else if (conf['runtime'].startsWith('safari')) { | 161 } else if (conf['runtime'].startsWith('safari')) { |
156 // Safari does not allow us to run from a fresh profile, so we can only | 162 // Safari does not allow us to run from a fresh profile, so we can only |
157 // use one browser. Additionally, you can not start two simulators | 163 // use one browser. Additionally, you can not start two simulators |
158 // for mobile safari simultainiously. | 164 // for mobile safari simultainiously. |
159 maxBrowserProcesses = 1; | 165 maxBrowserProcesses = 1; |
160 } else if (conf['runtime'] == 'chrome' && | 166 } else if (conf['runtime'] == 'chrome' && |
161 Platform.operatingSystem == 'macos') { | 167 Platform.operatingSystem == 'macos') { |
162 // Chrome on mac results in random timeouts. | 168 // Chrome on mac results in random timeouts. |
163 // Issue: https://github.com/dart-lang/sdk/issues/23891 | 169 // Issue: https://github.com/dart-lang/sdk/issues/23891 |
164 // This change does not fix the problem. | 170 // This change does not fix the problem. |
165 maxBrowserProcesses = math.max(1, maxBrowserProcesses ~/ 2); | 171 maxBrowserProcesses = math.max(1, maxBrowserProcesses ~/ 2); |
166 } else if (conf['runtime'] != 'drt') { | 172 } else if (conf['runtime'] != 'drt') { |
167 // Even on machines with more than 16 processors, don't open more | 173 // Even on machines with more than 16 processors, don't open more |
168 // than 15 browser instances, to avoid overloading the machine. | 174 // than 15 browser instances, to avoid overloading the machine. |
169 // This is especially important when running locally on powerful | 175 // This is especially important when running locally on powerful |
170 // desktops. | 176 // desktops. |
171 maxBrowserProcesses = math.min(maxBrowserProcesses, 15); | 177 maxBrowserProcesses = math.min(maxBrowserProcesses, 15); |
172 } | 178 } |
173 | 179 |
174 // If we specifically pass in a suite only run that. | 180 // If we specifically pass in a suite only run that. |
175 if (conf['suite_dir'] != null) { | 181 if (conf['suite_dir'] != null) { |
176 var suite_path = new Path(conf['suite_dir']); | 182 var suite_path = new Path(conf['suite_dir']); |
177 testSuites.add(new PKGTestSuite(conf, suite_path)); | 183 testSuites.add(new PKGTestSuite(conf, suite_path)); |
178 } else { | 184 } else { |
179 for (String key in selectors.keys) { | 185 for (String key in selectors.keys) { |
180 if (key == 'co19') { | 186 if (key == 'co19') { |
181 testSuites.add(new Co19TestSuite(conf)); | 187 testSuites.add(new Co19TestSuite(conf)); |
182 } else if (conf['compiler'] == 'none' && | 188 } else if (conf['compiler'] == 'none' && |
183 conf['runtime'] == 'vm' && | 189 conf['runtime'] == 'vm' && |
184 key == 'vm') { | 190 key == 'vm') { |
185 // vm tests contain both cc tests (added here) and dart tests (added | 191 // vm tests contain both cc tests (added here) and dart tests (added |
186 // in [TEST_SUITE_DIRECTORIES]). | 192 // in [TEST_SUITE_DIRECTORIES]). |
187 testSuites.add(new VMTestSuite(conf)); | 193 testSuites.add(new VMTestSuite(conf)); |
188 } else if (conf['analyzer']) { | 194 } else if (conf['analyzer']) { |
189 if (key == 'analyze_library') { | 195 if (key == 'analyze_library') { |
190 testSuites.add(new AnalyzeLibraryTestSuite(conf)); | 196 testSuites.add(new AnalyzeLibraryTestSuite(conf)); |
191 } | 197 } |
192 } else if (conf['compiler'] == 'none' && | 198 } else if (conf['compiler'] == 'none' && |
193 conf['runtime'] == 'vm' && | 199 conf['runtime'] == 'vm' && |
194 key == 'pkgbuild') { | 200 key == 'pkgbuild') { |
195 if (!conf['use_repository_packages'] && | 201 if (!conf['use_repository_packages'] && |
196 !conf['use_public_packages']) { | 202 !conf['use_public_packages']) { |
197 print("You need to use either --use-repository-packages or " | 203 print("You need to use either --use-repository-packages or " |
198 "--use-public-packages with the pkgbuild test suite!"); | 204 "--use-public-packages with the pkgbuild test suite!"); |
199 exit(1); | 205 exit(1); |
200 } | 206 } |
201 if (!conf['use_sdk']) { | 207 if (!conf['use_sdk']) { |
202 print("Running the 'pkgbuild' test suite requires " | 208 print("Running the 'pkgbuild' test suite requires " |
203 "passing the '--use-sdk' to test.py"); | 209 "passing the '--use-sdk' to test.py"); |
204 exit(1); | 210 exit(1); |
205 } | 211 } |
206 testSuites.add( | 212 testSuites.add( |
207 new PkgBuildTestSuite(conf, 'pkgbuild', 'pkg/pkgbuild.status')); | 213 new PkgBuildTestSuite(conf, 'pkgbuild', 'pkg/pkgbuild.status')); |
208 } | 214 } |
209 } | 215 } |
210 | 216 |
211 for (final testSuiteDir in TEST_SUITE_DIRECTORIES) { | 217 for (final testSuiteDir in TEST_SUITE_DIRECTORIES) { |
212 final name = testSuiteDir.filename; | 218 final name = testSuiteDir.filename; |
213 if (selectors.containsKey(name)) { | 219 if (selectors.containsKey(name)) { |
214 testSuites.add( | 220 testSuites |
215 new StandardTestSuite.forDirectory(conf, testSuiteDir)); | 221 .add(new StandardTestSuite.forDirectory(conf, testSuiteDir)); |
216 } | 222 } |
217 } | 223 } |
218 } | 224 } |
219 } | 225 } |
220 | 226 |
221 void allTestsFinished() { | 227 void allTestsFinished() { |
222 for (var conf in configurations) { | 228 for (var conf in configurations) { |
223 if (conf.containsKey('_servers_')) { | 229 if (conf.containsKey('_servers_')) { |
224 conf['_servers_'].stopServers(); | 230 conf['_servers_'].stopServers(); |
225 } | 231 } |
(...skipping 19 matching lines...) Expand all Loading... |
245 eventListener.add(new StatusFileUpdatePrinter()); | 251 eventListener.add(new StatusFileUpdatePrinter()); |
246 } | 252 } |
247 eventListener.add(new SummaryPrinter()); | 253 eventListener.add(new SummaryPrinter()); |
248 eventListener.add(new FlakyLogWriter()); | 254 eventListener.add(new FlakyLogWriter()); |
249 if (printFailures) { | 255 if (printFailures) { |
250 // The buildbot has it's own failure summary since it needs to wrap it | 256 // The buildbot has it's own failure summary since it needs to wrap it |
251 // into '@@@'-annotated sections. | 257 // into '@@@'-annotated sections. |
252 var printFailureSummary = progressIndicator != 'buildbot'; | 258 var printFailureSummary = progressIndicator != 'buildbot'; |
253 eventListener.add(new TestFailurePrinter(printFailureSummary, formatter)); | 259 eventListener.add(new TestFailurePrinter(printFailureSummary, formatter)); |
254 } | 260 } |
255 eventListener.add(progressIndicatorFromName(progressIndicator, | 261 eventListener.add( |
256 startTime, | 262 progressIndicatorFromName(progressIndicator, startTime, formatter)); |
257 formatter)); | |
258 if (printTiming) { | 263 if (printTiming) { |
259 eventListener.add(new TimingPrinter(startTime)); | 264 eventListener.add(new TimingPrinter(startTime)); |
260 } | 265 } |
261 eventListener.add(new SkippedCompilationsPrinter()); | 266 eventListener.add(new SkippedCompilationsPrinter()); |
262 eventListener.add(new LeftOverTempDirPrinter()); | 267 eventListener.add(new LeftOverTempDirPrinter()); |
263 } | 268 } |
264 if (firstConf['write_test_outcome_log']) { | 269 if (firstConf['write_test_outcome_log']) { |
265 eventListener.add(new TestOutcomeLogWriter()); | 270 eventListener.add(new TestOutcomeLogWriter()); |
266 } | 271 } |
267 if (firstConf['copy_coredumps']) { | 272 if (firstConf['copy_coredumps']) { |
268 eventListener.add(new UnexpectedCrashDumpArchiver()); | 273 eventListener.add(new UnexpectedCrashDumpArchiver()); |
269 } | 274 } |
270 | 275 |
271 // The only progress indicator when listing tests should be the | 276 // The only progress indicator when listing tests should be the |
272 // the summary printer. | 277 // the summary printer. |
273 if (listTests) { | 278 if (listTests) { |
274 eventListener.add(new SummaryPrinter(jsonOnly: reportInJson)); | 279 eventListener.add(new SummaryPrinter(jsonOnly: reportInJson)); |
275 } else { | 280 } else { |
276 eventListener.add(new ExitCodeSetter()); | 281 eventListener.add(new ExitCodeSetter()); |
277 } | 282 } |
278 | 283 |
279 void startProcessQueue() { | 284 void startProcessQueue() { |
280 // [firstConf] is needed here, since the ProcessQueue needs to know the | 285 // [firstConf] is needed here, since the ProcessQueue needs to know the |
281 // settings of 'noBatch' and 'local_ip' | 286 // settings of 'noBatch' and 'local_ip' |
282 new ProcessQueue(firstConf, | 287 new ProcessQueue( |
283 maxProcesses, | 288 firstConf, |
284 maxBrowserProcesses, | 289 maxProcesses, |
285 startTime, | 290 maxBrowserProcesses, |
286 testSuites, | 291 startTime, |
287 eventListener, | 292 testSuites, |
288 allTestsFinished, | 293 eventListener, |
289 verbose, | 294 allTestsFinished, |
290 recordingPath, | 295 verbose, |
291 recordingOutputPath); | 296 recordingPath, |
| 297 recordingOutputPath); |
292 } | 298 } |
293 | 299 |
294 // Start all the HTTP servers required before starting the process queue. | 300 // Start all the HTTP servers required before starting the process queue. |
295 if (serverFutures.isEmpty) { | 301 if (serverFutures.isEmpty) { |
296 startProcessQueue(); | 302 startProcessQueue(); |
297 } else { | 303 } else { |
298 Future.wait(serverFutures).then((_) => startProcessQueue()); | 304 Future.wait(serverFutures).then((_) => startProcessQueue()); |
299 } | 305 } |
300 } | 306 } |
OLD | NEW |