Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(162)

Side by Side Diff: tools/testing/dart/test_configurations.dart

Issue 2901923003: Replace the configuration map with a typed object. (Closed)
Patch Set: Revise. Created 3 years, 6 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
« no previous file with comments | « tools/testing/dart/status_reporter.dart ('k') | tools/testing/dart/test_progress.dart » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file
2 // for details. All rights reserved. Use of this source code is governed by a 2 // for details. All rights reserved. Use of this source code is governed by a
3 // BSD-style license that can be found in the LICENSE file. 3 // BSD-style license that can be found in the LICENSE file.
4 4
5 library test_configurations; 5 import 'dart:async';
6
7 import "dart:async";
8 import 'dart:convert';
9 import 'dart:io'; 6 import 'dart:io';
10 import "dart:math" as math; 7 import 'dart:math' as math;
11 8
12 import 'android.dart'; 9 import 'android.dart';
13 import "browser_controller.dart"; 10 import 'browser_controller.dart';
14 import "co19_test_config.dart"; 11 import 'co19_test_config.dart';
15 import "http_server.dart"; 12 import 'configuration.dart';
16 import "path.dart"; 13 import 'path.dart';
17 import "test_progress.dart"; 14 import 'test_progress.dart';
18 import "test_runner.dart"; 15 import 'test_runner.dart';
19 import "test_suite.dart"; 16 import 'test_suite.dart';
20 import "utils.dart"; 17 import 'utils.dart';
21 import "vm_test_config.dart"; 18 import 'vm_test_config.dart';
22 19
23 /** 20 /**
24 * The directories that contain test suites which follow the conventions 21 * The directories that contain test suites which follow the conventions
25 * required by [StandardTestSuite]'s forDirectory constructor. 22 * required by [StandardTestSuite]'s forDirectory constructor.
26 * New test suites should follow this convention because it makes it much 23 * New test suites should follow this convention because it makes it much
27 * simpler to add them to test.dart. Existing test suites should be 24 * simpler to add them to test.dart. Existing test suites should be
28 * moved to here, if possible. 25 * moved to here, if possible.
29 */ 26 */
30 final TEST_SUITE_DIRECTORIES = [ 27 final TEST_SUITE_DIRECTORIES = [
31 new Path('pkg'), 28 new Path('pkg'),
(...skipping 19 matching lines...) Expand all
51 new Path('tests/lib_strong'), 48 new Path('tests/lib_strong'),
52 new Path('tests/standalone'), 49 new Path('tests/standalone'),
53 new Path('tests/utils'), 50 new Path('tests/utils'),
54 new Path('utils/tests/css'), 51 new Path('utils/tests/css'),
55 new Path('utils/tests/peg'), 52 new Path('utils/tests/peg'),
56 ]; 53 ];
57 54
58 // This file is created by gclient runhooks. 55 // This file is created by gclient runhooks.
59 final VS_TOOLCHAIN_FILE = new Path("build/win_toolchain.json"); 56 final VS_TOOLCHAIN_FILE = new Path("build/win_toolchain.json");
60 57
61 Future testConfigurations(List<Map<String, dynamic>> configurations) async { 58 Future testConfigurations(List<Configuration> configurations) async {
62 var startTime = new DateTime.now(); 59 var startTime = new DateTime.now();
63 // Extract global options from first configuration. 60 // Extract global options from first configuration.
64 var firstConf = configurations[0]; 61 var firstConf = configurations[0];
65 var maxProcesses = firstConf['tasks'] as int; 62 var maxProcesses = firstConf.taskCount;
66 var progressIndicator = firstConf['progress'] as String; 63 var progressIndicator = firstConf.progress;
67 BuildbotProgressIndicator.stepName = firstConf['step_name'] as String; 64 BuildbotProgressIndicator.stepName = firstConf.stepName;
68 var verbose = firstConf['verbose'] as bool; 65 var verbose = firstConf.isVerbose;
69 var printTiming = firstConf['time'] as bool; 66 var printTiming = firstConf.printTiming;
70 var listTests = firstConf['list'] as bool; 67 var listTests = firstConf.listTests;
71 68
72 var reportInJson = firstConf['report_in_json'] as bool; 69 var reportInJson = firstConf.reportInJson;
70 var recordingPath = firstConf.recordingPath;
71 var replayPath = firstConf.replayPath;
73 72
74 var recordingPath = firstConf['record_to_file'] as String; 73 Browser.resetBrowserConfiguration = firstConf.resetBrowser;
75 var recordingOutputPath = firstConf['replay_from_file'] as String;
76 74
77 Browser.resetBrowserConfiguration = 75 if (recordingPath != null && replayPath != null) {
78 firstConf['reset_browser_configuration'] as bool;
79
80 if (recordingPath != null && recordingOutputPath != null) {
81 print("Fatal: Can't have the '--record_to_file' and '--replay_from_file'" 76 print("Fatal: Can't have the '--record_to_file' and '--replay_from_file'"
82 "at the same time. Exiting ..."); 77 "at the same time. Exiting ...");
83 exit(1); 78 exit(1);
84 } 79 }
85 80
86 if (!(firstConf['append_logs'] as bool)) { 81 if (!firstConf.appendLogs) {
87 var files = [ 82 var files = [
88 new File(TestUtils.flakyFileName), 83 new File(TestUtils.flakyFileName),
89 new File(TestUtils.testOutcomeFileName) 84 new File(TestUtils.testOutcomeFileName)
90 ]; 85 ];
91 for (var file in files) { 86 for (var file in files) {
92 if (file.existsSync()) { 87 if (file.existsSync()) {
93 file.deleteSync(); 88 file.deleteSync();
94 } 89 }
95 } 90 }
96 } 91 }
97 92
98 DebugLogger.init( 93 DebugLogger.init(firstConf.writeDebugLog ? TestUtils.debugLogFilePath : null,
99 firstConf['write_debug_log'] as bool ? TestUtils.debugLogFilePath : null, 94 append: firstConf.appendLogs);
100 append: firstConf['append_logs'] as bool);
101 95
102 // Print the configurations being run by this execution of 96 // Print the configurations being run by this execution of
103 // test.dart. However, don't do it if the silent progress indicator 97 // test.dart. However, don't do it if the silent progress indicator
104 // is used. This is only needed because of the junit tests. 98 // is used. This is only needed because of the junit tests.
105 if (progressIndicator != 'silent') { 99 if (progressIndicator != Progress.silent) {
106 var outputWords = configurations.length > 1 100 var outputWords = configurations.length > 1
107 ? ['Test configurations:'] 101 ? ['Test configurations:']
108 : ['Test configuration:']; 102 : ['Test configuration:'];
109 for (Map conf in configurations) { 103
110 List settings = ['compiler', 'runtime', 'mode', 'arch'] 104 for (var configuration in configurations) {
111 .map((name) => conf[name]) 105 var settings = [
112 .toList(); 106 configuration.compiler.name,
113 if (conf['checked'] as bool) settings.add('checked'); 107 configuration.runtime.name,
114 if (conf['strong'] as bool) settings.add('strong'); 108 configuration.mode.name,
109 configuration.architecture.name
110 ];
111 if (configuration.isChecked) settings.add('checked');
112 if (configuration.isStrong) settings.add('strong');
113 if (configuration.useFastStartup) settings.add('fast-startup');
115 outputWords.add(settings.join('_')); 114 outputWords.add(settings.join('_'));
116 } 115 }
117 print(outputWords.join(' ')); 116 print(outputWords.join(' '));
118 } 117 }
119 118
120 var runningBrowserTests = configurations.any((config) { 119 var runningBrowserTests =
121 return TestUtils.isBrowserRuntime(config['runtime'] as String); 120 configurations.any((config) => config.runtime.isBrowser);
122 });
123 121
124 List<Future> serverFutures = []; 122 var serverFutures = <Future>[];
125 var testSuites = <TestSuite>[]; 123 var testSuites = <TestSuite>[];
126 var maxBrowserProcesses = maxProcesses; 124 var maxBrowserProcesses = maxProcesses;
127 if (configurations.length > 1 && 125 if (configurations.length > 1 &&
128 (configurations[0]['test_server_port'] != 0 || 126 (configurations[0].testServerPort != 0 ||
129 configurations[0]['test_server_cross_origin_port'] != 0)) { 127 configurations[0].testServerCrossOriginPort != 0)) {
130 print("If the http server ports are specified, only one configuration" 128 print("If the http server ports are specified, only one configuration"
131 " may be run at a time"); 129 " may be run at a time");
132 exit(1); 130 exit(1);
133 } 131 }
134 for (var conf in configurations) { 132
135 var selectors = conf['selectors'] as Map<String, RegExp>; 133 for (var configuration in configurations) {
136 var useContentSecurityPolicy = conf['csp'] as bool;
137 if (!listTests && runningBrowserTests) { 134 if (!listTests && runningBrowserTests) {
138 // Start global http servers that serve the entire dart repo. 135 serverFutures.add(configuration.startServers());
139 // The http server is available on window.location.port, and a second
140 // server for cross-domain tests can be found by calling
141 // getCrossOriginPortNumber().
142 var servers = new TestingServers(
143 TestUtils.buildDir(conf),
144 useContentSecurityPolicy,
145 conf['runtime'] as String,
146 null,
147 conf['package_root'] as String,
148 conf['packages'] as String);
149 serverFutures.add(servers.startServers(conf['local_ip'] as String,
150 port: conf['test_server_port'] as int,
151 crossOriginPort: conf['test_server_cross_origin_port'] as int));
152 conf['_servers_'] = servers;
153 if (verbose) {
154 serverFutures.last.then((_) {
155 var commandline = servers.httpServerCommandLine();
156 print('Started HttpServers: $commandline');
157 });
158 }
159 } 136 }
160 137
161 if ((conf['runtime'] as String).startsWith('ie')) { 138 if (configuration.runtime.isIE) {
162 // NOTE: We've experienced random timeouts of tests on ie9/ie10. The 139 // NOTE: We've experienced random timeouts of tests on ie9/ie10. The
163 // underlying issue has not been determined yet. Our current hypothesis 140 // underlying issue has not been determined yet. Our current hypothesis
164 // is that windows does not handle the IE processes independently. 141 // is that windows does not handle the IE processes independently.
165 // If we have more than one browser and kill a browser we are seeing 142 // If we have more than one browser and kill a browser we are seeing
166 // issues with starting up a new browser just after killing the hanging 143 // issues with starting up a new browser just after killing the hanging
167 // browser. 144 // browser.
168 maxBrowserProcesses = 1; 145 maxBrowserProcesses = 1;
169 } else if ((conf['runtime'] as String).startsWith('safari')) { 146 } else if (configuration.runtime.isSafari) {
170 // Safari does not allow us to run from a fresh profile, so we can only 147 // Safari does not allow us to run from a fresh profile, so we can only
171 // use one browser. Additionally, you can not start two simulators 148 // use one browser. Additionally, you can not start two simulators
172 // for mobile safari simultainiously. 149 // for mobile safari simultainiously.
173 maxBrowserProcesses = 1; 150 maxBrowserProcesses = 1;
174 } else if ((conf['runtime'] as String) == 'chrome' && 151 } else if (configuration.runtime == Runtime.chrome &&
175 Platform.operatingSystem == 'macos') { 152 Platform.operatingSystem == 'macos') {
176 // Chrome on mac results in random timeouts. 153 // Chrome on mac results in random timeouts.
177 // Issue: https://github.com/dart-lang/sdk/issues/23891 154 // Issue: https://github.com/dart-lang/sdk/issues/23891
178 // This change does not fix the problem. 155 // This change does not fix the problem.
179 maxBrowserProcesses = math.max(1, maxBrowserProcesses ~/ 2); 156 maxBrowserProcesses = math.max(1, maxBrowserProcesses ~/ 2);
180 } else if ((conf['runtime'] as String) != 'drt') { 157 } else if (configuration.runtime != Runtime.drt) {
181 // Even on machines with more than 16 processors, don't open more 158 // Even on machines with more than 16 processors, don't open more
182 // than 15 browser instances, to avoid overloading the machine. 159 // than 15 browser instances, to avoid overloading the machine.
183 // This is especially important when running locally on powerful 160 // This is especially important when running locally on powerful
184 // desktops. 161 // desktops.
185 maxBrowserProcesses = math.min(maxBrowserProcesses, 15); 162 maxBrowserProcesses = math.min(maxBrowserProcesses, 15);
186 } 163 }
187 164
188 // If we specifically pass in a suite only run that. 165 // If we specifically pass in a suite only run that.
189 if (conf['suite_dir'] != null) { 166 if (configuration.suiteDirectory != null) {
190 var suite_path = new Path(conf['suite_dir'] as String); 167 var suitePath = new Path(configuration.suiteDirectory);
191 testSuites.add(new PKGTestSuite(conf, suite_path)); 168 testSuites.add(new PKGTestSuite(configuration, suitePath));
192 } else { 169 } else {
193 for (final testSuiteDir in TEST_SUITE_DIRECTORIES) { 170 for (var testSuiteDir in TEST_SUITE_DIRECTORIES) {
194 final name = testSuiteDir.filename; 171 var name = testSuiteDir.filename;
195 if (selectors.containsKey(name)) { 172 if (configuration.selectors.containsKey(name)) {
196 testSuites 173 testSuites.add(
197 .add(new StandardTestSuite.forDirectory(conf, testSuiteDir)); 174 new StandardTestSuite.forDirectory(configuration, testSuiteDir));
198 } 175 }
199 } 176 }
200 for (String key in selectors.keys) { 177
178 for (var key in configuration.selectors.keys) {
201 if (key == 'co19') { 179 if (key == 'co19') {
202 testSuites.add(new Co19TestSuite(conf)); 180 testSuites.add(new Co19TestSuite(configuration));
203 } else if ((conf['compiler'] == 'dartk' || 181 } else if (configuration.compiler == Compiler.none &&
204 conf['compiler'] == 'none') && 182 configuration.runtime == Runtime.vm &&
205 conf['runtime'] == 'vm' &&
206 key == 'vm') { 183 key == 'vm') {
207 // vm tests contain both cc tests (added here) and dart tests (added 184 // vm tests contain both cc tests (added here) and dart tests (added
208 // in [TEST_SUITE_DIRECTORIES]). 185 // in [TEST_SUITE_DIRECTORIES]).
209 testSuites.add(new VMTestSuite(conf)); 186 testSuites.add(new VMTestSuite(configuration));
210 } else if (conf['analyzer'] as bool) { 187 } else if (configuration.compiler == Compiler.dart2analyzer) {
211 if (key == 'analyze_library') { 188 if (key == 'analyze_library') {
212 testSuites.add(new AnalyzeLibraryTestSuite(conf)); 189 testSuites.add(new AnalyzeLibraryTestSuite(configuration));
213 } 190 }
214 } 191 }
215 } 192 }
216 } 193 }
217 } 194 }
218 195
219 void allTestsFinished() { 196 void allTestsFinished() {
220 for (var conf in configurations) { 197 for (var configuration in configurations) {
221 if (conf.containsKey('_servers_')) { 198 configuration.stopServers();
222 conf['_servers_'].stopServers();
223 }
224 } 199 }
200
225 DebugLogger.close(); 201 DebugLogger.close();
226 TestUtils.deleteTempSnapshotDirectory(configurations[0]); 202 TestUtils.deleteTempSnapshotDirectory(configurations[0]);
227 } 203 }
228 204
229 var eventListener = <EventListener>[]; 205 var eventListener = <EventListener>[];
230 206
231 // We don't print progress if we list tests. 207 // We don't print progress if we list tests.
232 if (progressIndicator != 'silent' && !listTests) { 208 if (progressIndicator != Progress.silent && !listTests) {
233 var printFailures = true; 209 var printFailures = true;
234 var formatter = Formatter.normal; 210 var formatter = Formatter.normal;
235 if (progressIndicator == 'color') { 211 if (progressIndicator == Progress.color) {
236 progressIndicator = 'compact'; 212 progressIndicator = Progress.compact;
237 formatter = Formatter.color; 213 formatter = Formatter.color;
238 } 214 }
239 if (progressIndicator == 'diff') { 215 if (progressIndicator == Progress.diff) {
240 progressIndicator = 'compact'; 216 progressIndicator = Progress.compact;
241 formatter = Formatter.color; 217 formatter = Formatter.color;
242 printFailures = false; 218 printFailures = false;
243 eventListener.add(new StatusFileUpdatePrinter()); 219 eventListener.add(new StatusFileUpdatePrinter());
244 } 220 }
245 eventListener.add(new SummaryPrinter()); 221 eventListener.add(new SummaryPrinter());
246 eventListener.add(new FlakyLogWriter()); 222 eventListener.add(new FlakyLogWriter());
247 if (printFailures) { 223 if (printFailures) {
248 // The buildbot has it's own failure summary since it needs to wrap it 224 // The buildbot has it's own failure summary since it needs to wrap it
249 // into '@@@'-annotated sections. 225 // into '@@@'-annotated sections.
250 var printFailureSummary = progressIndicator != 'buildbot'; 226 var printFailureSummary = progressIndicator != Progress.buildbot;
251 eventListener.add(new TestFailurePrinter(printFailureSummary, formatter)); 227 eventListener.add(new TestFailurePrinter(printFailureSummary, formatter));
252 } 228 }
253 eventListener.add( 229 eventListener.add(ProgressIndicator.fromProgress(
254 ProgressIndicator.fromName(progressIndicator, startTime, formatter)); 230 progressIndicator, startTime, formatter));
255 if (printTiming) { 231 if (printTiming) {
256 eventListener.add(new TimingPrinter(startTime)); 232 eventListener.add(new TimingPrinter(startTime));
257 } 233 }
258 eventListener.add(new SkippedCompilationsPrinter()); 234 eventListener.add(new SkippedCompilationsPrinter());
259 } 235 }
260 if (firstConf['write_test_outcome_log'] as bool) { 236
237 if (firstConf.writeTestOutcomeLog) {
261 eventListener.add(new TestOutcomeLogWriter()); 238 eventListener.add(new TestOutcomeLogWriter());
262 } 239 }
263 if (firstConf['copy_coredumps'] as bool) { 240
241 if (firstConf.copyCoreDumps) {
264 eventListener.add(new UnexpectedCrashLogger()); 242 eventListener.add(new UnexpectedCrashLogger());
265 } 243 }
266 244
267 // The only progress indicator when listing tests should be the 245 // The only progress indicator when listing tests should be the
268 // the summary printer. 246 // the summary printer.
269 if (listTests) { 247 if (listTests) {
270 eventListener.add(new SummaryPrinter(jsonOnly: reportInJson)); 248 eventListener.add(new SummaryPrinter(jsonOnly: reportInJson));
271 } else { 249 } else {
272 eventListener.add(new ExitCodeSetter()); 250 eventListener.add(new ExitCodeSetter());
273 eventListener.add(new IgnoredTestMonitor()); 251 eventListener.add(new IgnoredTestMonitor());
274 } 252 }
275 253
276 // If any of the configurations need to access android devices we'll first 254 // If any of the configurations need to access android devices we'll first
277 // make a pool of all available adb devices. 255 // make a pool of all available adb devices.
278 AdbDevicePool adbDevicePool; 256 AdbDevicePool adbDevicePool;
279 var needsAdbDevicePool = configurations.any((Map conf) { 257 var needsAdbDevicePool = configurations.any((conf) {
280 return conf['runtime'] == 'dart_precompiled' && conf['system'] == 'android'; 258 return conf.runtime == Runtime.dartPrecompiled &&
259 conf.system == System.android;
281 }); 260 });
282 if (needsAdbDevicePool) { 261 if (needsAdbDevicePool) {
283 adbDevicePool = await AdbDevicePool.create(); 262 adbDevicePool = await AdbDevicePool.create();
284 } 263 }
285 264
286 // Start all the HTTP servers required before starting the process queue. 265 // Start all the HTTP servers required before starting the process queue.
287 if (!serverFutures.isEmpty) { 266 if (!serverFutures.isEmpty) {
288 await Future.wait(serverFutures); 267 await Future.wait(serverFutures);
289 } 268 }
290 269
291 if (Platform.isWindows) {
292 // When running tests on Windows, use cdb from depot_tools to dump
293 // stack traces of tests timing out.
294 try {
295 var text =
296 await new File(VS_TOOLCHAIN_FILE.toNativePath()).readAsString();
297 firstConf['win_sdk_path'] = JSON.decode(text)['win_sdk'];
298 } on dynamic {
299 // Ignore errors here. If win_sdk is not found, stack trace dumping
300 // for timeouts won't work.
301 }
302 }
303
304 // [firstConf] is needed here, since the ProcessQueue needs to know the 270 // [firstConf] is needed here, since the ProcessQueue needs to know the
305 // settings of 'noBatch' and 'local_ip' 271 // settings of 'noBatch' and 'local_ip'
306 new ProcessQueue( 272 new ProcessQueue(
307 firstConf, 273 firstConf,
308 maxProcesses, 274 maxProcesses,
309 maxBrowserProcesses, 275 maxBrowserProcesses,
310 startTime, 276 startTime,
311 testSuites, 277 testSuites,
312 eventListener, 278 eventListener,
313 allTestsFinished, 279 allTestsFinished,
314 verbose, 280 verbose,
315 recordingPath, 281 recordingPath,
316 recordingOutputPath, 282 replayPath,
317 adbDevicePool); 283 adbDevicePool);
318 } 284 }
OLDNEW
« no previous file with comments | « tools/testing/dart/status_reporter.dart ('k') | tools/testing/dart/test_progress.dart » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698