Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(104)

Side by Side Diff: tools/testing/dart/test_configurations.dart

Issue 2901923003: Replace the configuration map with a typed object. (Closed)
Patch Set: Created 3 years, 6 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file
2 // for details. All rights reserved. Use of this source code is governed by a 2 // for details. All rights reserved. Use of this source code is governed by a
3 // BSD-style license that can be found in the LICENSE file. 3 // BSD-style license that can be found in the LICENSE file.
4 4
5 library test_configurations; 5 import 'dart:async';
6
7 import "dart:async";
8 import 'dart:convert';
9 import 'dart:io'; 6 import 'dart:io';
10 import "dart:math" as math; 7 import 'dart:math' as math;
11 8
12 import 'android.dart'; 9 import 'android.dart';
13 import "browser_controller.dart"; 10 import 'browser_controller.dart';
14 import "co19_test_config.dart"; 11 import 'co19_test_config.dart';
15 import "http_server.dart"; 12 import 'configuration.dart';
16 import "path.dart"; 13 import 'path.dart';
17 import "test_progress.dart"; 14 import 'test_progress.dart';
18 import "test_runner.dart"; 15 import 'test_runner.dart';
19 import "test_suite.dart"; 16 import 'test_suite.dart';
20 import "utils.dart"; 17 import 'utils.dart';
21 import "vm_test_config.dart"; 18 import 'vm_test_config.dart';
22 19
23 /** 20 /**
24 * The directories that contain test suites which follow the conventions 21 * The directories that contain test suites which follow the conventions
25 * required by [StandardTestSuite]'s forDirectory constructor. 22 * required by [StandardTestSuite]'s forDirectory constructor.
26 * New test suites should follow this convention because it makes it much 23 * New test suites should follow this convention because it makes it much
27 * simpler to add them to test.dart. Existing test suites should be 24 * simpler to add them to test.dart. Existing test suites should be
28 * moved to here, if possible. 25 * moved to here, if possible.
29 */ 26 */
30 final TEST_SUITE_DIRECTORIES = [ 27 final TEST_SUITE_DIRECTORIES = [
31 new Path('pkg'), 28 new Path('pkg'),
(...skipping 19 matching lines...) Expand all
51 new Path('tests/lib_strong'), 48 new Path('tests/lib_strong'),
52 new Path('tests/standalone'), 49 new Path('tests/standalone'),
53 new Path('tests/utils'), 50 new Path('tests/utils'),
54 new Path('utils/tests/css'), 51 new Path('utils/tests/css'),
55 new Path('utils/tests/peg'), 52 new Path('utils/tests/peg'),
56 ]; 53 ];
57 54
58 // This file is created by gclient runhooks. 55 // This file is created by gclient runhooks.
59 final VS_TOOLCHAIN_FILE = new Path("build/win_toolchain.json"); 56 final VS_TOOLCHAIN_FILE = new Path("build/win_toolchain.json");
60 57
61 Future testConfigurations(List<Map<String, dynamic>> configurations) async { 58 Future testConfigurations(List<Configuration> configurations) async {
62 var startTime = new DateTime.now(); 59 var startTime = new DateTime.now();
63 // Extract global options from first configuration. 60 // Extract global options from first configuration.
64 var firstConf = configurations[0]; 61 var firstConf = configurations[0];
65 var maxProcesses = firstConf['tasks'] as int; 62 var maxProcesses = firstConf.taskCount;
66 var progressIndicator = firstConf['progress'] as String; 63 var progressIndicator = firstConf.progress;
67 BuildbotProgressIndicator.stepName = firstConf['step_name'] as String; 64 BuildbotProgressIndicator.stepName = firstConf.stepName;
68 var verbose = firstConf['verbose'] as bool; 65 var verbose = firstConf.isVerbose;
69 var printTiming = firstConf['time'] as bool; 66 var printTiming = firstConf.printTiming;
70 var listTests = firstConf['list'] as bool; 67 var listTests = firstConf.listTests;
71 68
72 var reportInJson = firstConf['report_in_json'] as bool; 69 var reportInJson = firstConf.reportInJson;
70 var recordingPath = firstConf.recordingPath;
71 var replayPath = firstConf.replayPath;
73 72
74 var recordingPath = firstConf['record_to_file'] as String; 73 Browser.resetBrowserConfiguration = firstConf.resetBrowser;
75 var recordingOutputPath = firstConf['replay_from_file'] as String;
76 74
77 Browser.resetBrowserConfiguration = 75 if (recordingPath != null && replayPath != null) {
78 firstConf['reset_browser_configuration'] as bool;
79
80 if (recordingPath != null && recordingOutputPath != null) {
81 print("Fatal: Can't have the '--record_to_file' and '--replay_from_file'" 76 print("Fatal: Can't have the '--record_to_file' and '--replay_from_file'"
82 "at the same time. Exiting ..."); 77 "at the same time. Exiting ...");
83 exit(1); 78 exit(1);
84 } 79 }
85 80
86 if (!(firstConf['append_logs'] as bool)) { 81 if (!firstConf.appendLogs) {
87 var files = [ 82 var files = [
88 new File(TestUtils.flakyFileName), 83 new File(TestUtils.flakyFileName),
89 new File(TestUtils.testOutcomeFileName) 84 new File(TestUtils.testOutcomeFileName)
90 ]; 85 ];
91 for (var file in files) { 86 for (var file in files) {
92 if (file.existsSync()) { 87 if (file.existsSync()) {
93 file.deleteSync(); 88 file.deleteSync();
94 } 89 }
95 } 90 }
96 } 91 }
97 92
98 DebugLogger.init( 93 DebugLogger.init(firstConf.writeDebugLog ? TestUtils.debugLogFilePath : null,
99 firstConf['write_debug_log'] as bool ? TestUtils.debugLogFilePath : null, 94 append: firstConf.appendLogs);
100 append: firstConf['append_logs'] as bool);
101 95
102 // Print the configurations being run by this execution of 96 // Print the configurations being run by this execution of
103 // test.dart. However, don't do it if the silent progress indicator 97 // test.dart. However, don't do it if the silent progress indicator
104 // is used. This is only needed because of the junit tests. 98 // is used. This is only needed because of the junit tests.
105 if (progressIndicator != 'silent') { 99 if (progressIndicator != Progress.silent) {
106 var outputWords = configurations.length > 1 100 var outputWords = configurations.length > 1
107 ? ['Test configurations:'] 101 ? ['Test configurations:']
108 : ['Test configuration:']; 102 : ['Test configuration:'];
109 for (Map conf in configurations) { 103
110 List settings = ['compiler', 'runtime', 'mode', 'arch'] 104 for (var configuration in configurations) {
111 .map((name) => conf[name]) 105 var settings = [
112 .toList(); 106 configuration.compiler.name,
113 if (conf['checked'] as bool) settings.add('checked'); 107 configuration.runtime.name,
114 if (conf['strong'] as bool) settings.add('strong'); 108 configuration.mode.name,
109 configuration.architecture.name
110 ];
111 if (configuration.isChecked) settings.add('checked');
112 if (configuration.isStrong) settings.add('strong');
Bill Hesse 2017/05/29 13:08:28 Please add printing of "fast-startup" if fast star
Bob Nystrom 2017/05/30 23:29:31 Done.
115 outputWords.add(settings.join('_')); 113 outputWords.add(settings.join('_'));
116 } 114 }
117 print(outputWords.join(' ')); 115 print(outputWords.join(' '));
118 } 116 }
119 117
120 var runningBrowserTests = configurations.any((config) { 118 var runningBrowserTests =
121 return TestUtils.isBrowserRuntime(config['runtime'] as String); 119 configurations.any((config) => config.runtime.isBrowser);
122 });
123 120
124 List<Future> serverFutures = []; 121 var serverFutures = <Future>[];
125 var testSuites = <TestSuite>[]; 122 var testSuites = <TestSuite>[];
126 var maxBrowserProcesses = maxProcesses; 123 var maxBrowserProcesses = maxProcesses;
127 if (configurations.length > 1 && 124 if (configurations.length > 1 &&
128 (configurations[0]['test_server_port'] != 0 || 125 (configurations[0].testServerPort != 0 ||
129 configurations[0]['test_server_cross_origin_port'] != 0)) { 126 configurations[0].testServerCrossOriginPort != 0)) {
130 print("If the http server ports are specified, only one configuration" 127 print("If the http server ports are specified, only one configuration"
131 " may be run at a time"); 128 " may be run at a time");
132 exit(1); 129 exit(1);
133 } 130 }
134 for (var conf in configurations) { 131
135 var selectors = conf['selectors'] as Map<String, RegExp>; 132 for (var configuration in configurations) {
136 var useContentSecurityPolicy = conf['csp'] as bool;
137 if (!listTests && runningBrowserTests) { 133 if (!listTests && runningBrowserTests) {
138 // Start global http servers that serve the entire dart repo. 134 serverFutures.add(configuration.startServers());
139 // The http server is available on window.location.port, and a second
140 // server for cross-domain tests can be found by calling
141 // getCrossOriginPortNumber().
142 var servers = new TestingServers(
143 TestUtils.buildDir(conf),
144 useContentSecurityPolicy,
145 conf['runtime'] as String,
146 null,
147 conf['package_root'] as String,
148 conf['packages'] as String);
149 serverFutures.add(servers.startServers(conf['local_ip'] as String,
150 port: conf['test_server_port'] as int,
151 crossOriginPort: conf['test_server_cross_origin_port'] as int));
152 conf['_servers_'] = servers;
153 if (verbose) {
154 serverFutures.last.then((_) {
155 var commandline = servers.httpServerCommandLine();
156 print('Started HttpServers: $commandline');
157 });
158 }
159 } 135 }
160 136
161 if ((conf['runtime'] as String).startsWith('ie')) { 137 if (configuration.runtime.isIE) {
162 // NOTE: We've experienced random timeouts of tests on ie9/ie10. The 138 // NOTE: We've experienced random timeouts of tests on ie9/ie10. The
163 // underlying issue has not been determined yet. Our current hypothesis 139 // underlying issue has not been determined yet. Our current hypothesis
164 // is that windows does not handle the IE processes independently. 140 // is that windows does not handle the IE processes independently.
165 // If we have more than one browser and kill a browser we are seeing 141 // If we have more than one browser and kill a browser we are seeing
166 // issues with starting up a new browser just after killing the hanging 142 // issues with starting up a new browser just after killing the hanging
167 // browser. 143 // browser.
168 maxBrowserProcesses = 1; 144 maxBrowserProcesses = 1;
169 } else if ((conf['runtime'] as String).startsWith('safari')) { 145 } else if (configuration.runtime.isSafari) {
170 // Safari does not allow us to run from a fresh profile, so we can only 146 // Safari does not allow us to run from a fresh profile, so we can only
171 // use one browser. Additionally, you can not start two simulators 147 // use one browser. Additionally, you can not start two simulators
172 // for mobile safari simultainiously. 148 // for mobile safari simultainiously.
173 maxBrowserProcesses = 1; 149 maxBrowserProcesses = 1;
174 } else if ((conf['runtime'] as String) == 'chrome' && 150 } else if (configuration.runtime == Runtime.chrome &&
175 Platform.operatingSystem == 'macos') { 151 Platform.operatingSystem == 'macos') {
176 // Chrome on mac results in random timeouts. 152 // Chrome on mac results in random timeouts.
177 // Issue: https://github.com/dart-lang/sdk/issues/23891 153 // Issue: https://github.com/dart-lang/sdk/issues/23891
178 // This change does not fix the problem. 154 // This change does not fix the problem.
179 maxBrowserProcesses = math.max(1, maxBrowserProcesses ~/ 2); 155 maxBrowserProcesses = math.max(1, maxBrowserProcesses ~/ 2);
180 } else if ((conf['runtime'] as String) != 'drt') { 156 } else if (configuration.runtime != Runtime.drt) {
181 // Even on machines with more than 16 processors, don't open more 157 // Even on machines with more than 16 processors, don't open more
182 // than 15 browser instances, to avoid overloading the machine. 158 // than 15 browser instances, to avoid overloading the machine.
183 // This is especially important when running locally on powerful 159 // This is especially important when running locally on powerful
184 // desktops. 160 // desktops.
185 maxBrowserProcesses = math.min(maxBrowserProcesses, 15); 161 maxBrowserProcesses = math.min(maxBrowserProcesses, 15);
186 } 162 }
187 163
188 // If we specifically pass in a suite only run that. 164 // If we specifically pass in a suite only run that.
189 if (conf['suite_dir'] != null) { 165 if (configuration.suiteDirectory != null) {
190 var suite_path = new Path(conf['suite_dir'] as String); 166 var suitePath = new Path(configuration.suiteDirectory);
191 testSuites.add(new PKGTestSuite(conf, suite_path)); 167 testSuites.add(new PKGTestSuite(configuration, suitePath));
192 } else { 168 } else {
193 for (final testSuiteDir in TEST_SUITE_DIRECTORIES) { 169 for (var testSuiteDir in TEST_SUITE_DIRECTORIES) {
194 final name = testSuiteDir.filename; 170 var name = testSuiteDir.filename;
195 if (selectors.containsKey(name)) { 171 if (configuration.selectors.containsKey(name)) {
196 testSuites 172 testSuites.add(
197 .add(new StandardTestSuite.forDirectory(conf, testSuiteDir)); 173 new StandardTestSuite.forDirectory(configuration, testSuiteDir));
198 } 174 }
199 } 175 }
200 for (String key in selectors.keys) { 176
177 for (var key in configuration.selectors.keys) {
201 if (key == 'co19') { 178 if (key == 'co19') {
202 testSuites.add(new Co19TestSuite(conf)); 179 testSuites.add(new Co19TestSuite(configuration));
203 } else if (conf['compiler'] == 'none' && 180 } else if (configuration.compiler == Compiler.none &&
204 conf['runtime'] == 'vm' && 181 configuration.runtime == Runtime.vm &&
205 key == 'vm') { 182 key == 'vm') {
206 // vm tests contain both cc tests (added here) and dart tests (added 183 // vm tests contain both cc tests (added here) and dart tests (added
207 // in [TEST_SUITE_DIRECTORIES]). 184 // in [TEST_SUITE_DIRECTORIES]).
208 testSuites.add(new VMTestSuite(conf)); 185 testSuites.add(new VMTestSuite(configuration));
209 } else if (conf['analyzer'] as bool) { 186 } else if (configuration.compiler == Compiler.dart2analyzer) {
210 if (key == 'analyze_library') { 187 if (key == 'analyze_library') {
211 testSuites.add(new AnalyzeLibraryTestSuite(conf)); 188 testSuites.add(new AnalyzeLibraryTestSuite(configuration));
212 } 189 }
213 } 190 }
214 } 191 }
215 } 192 }
216 } 193 }
217 194
218 void allTestsFinished() { 195 void allTestsFinished() {
219 for (var conf in configurations) { 196 for (var configuration in configurations) {
220 if (conf.containsKey('_servers_')) { 197 configuration.stopServers();
221 conf['_servers_'].stopServers();
222 }
223 } 198 }
199
224 DebugLogger.close(); 200 DebugLogger.close();
225 TestUtils.deleteTempSnapshotDirectory(configurations[0]); 201 TestUtils.deleteTempSnapshotDirectory(configurations[0]);
226 } 202 }
227 203
228 var eventListener = <EventListener>[]; 204 var eventListener = <EventListener>[];
229 205
230 // We don't print progress if we list tests. 206 // We don't print progress if we list tests.
231 if (progressIndicator != 'silent' && !listTests) { 207 if (progressIndicator != Progress.silent && !listTests) {
232 var printFailures = true; 208 var printFailures = true;
233 var formatter = Formatter.normal; 209 var formatter = Formatter.normal;
234 if (progressIndicator == 'color') { 210 if (progressIndicator == Progress.color) {
235 progressIndicator = 'compact'; 211 progressIndicator = Progress.compact;
236 formatter = Formatter.color; 212 formatter = Formatter.color;
237 } 213 }
238 if (progressIndicator == 'diff') { 214 if (progressIndicator == Progress.diff) {
239 progressIndicator = 'compact'; 215 progressIndicator = Progress.compact;
240 formatter = Formatter.color; 216 formatter = Formatter.color;
241 printFailures = false; 217 printFailures = false;
242 eventListener.add(new StatusFileUpdatePrinter()); 218 eventListener.add(new StatusFileUpdatePrinter());
243 } 219 }
244 eventListener.add(new SummaryPrinter()); 220 eventListener.add(new SummaryPrinter());
245 eventListener.add(new FlakyLogWriter()); 221 eventListener.add(new FlakyLogWriter());
246 if (printFailures) { 222 if (printFailures) {
247 // The buildbot has it's own failure summary since it needs to wrap it 223 // The buildbot has it's own failure summary since it needs to wrap it
248 // into '@@@'-annotated sections. 224 // into '@@@'-annotated sections.
249 var printFailureSummary = progressIndicator != 'buildbot'; 225 var printFailureSummary = progressIndicator != Progress.buildbot;
250 eventListener.add(new TestFailurePrinter(printFailureSummary, formatter)); 226 eventListener.add(new TestFailurePrinter(printFailureSummary, formatter));
251 } 227 }
252 eventListener.add( 228 eventListener.add(ProgressIndicator.fromProgress(
253 ProgressIndicator.fromName(progressIndicator, startTime, formatter)); 229 progressIndicator, startTime, formatter));
254 if (printTiming) { 230 if (printTiming) {
255 eventListener.add(new TimingPrinter(startTime)); 231 eventListener.add(new TimingPrinter(startTime));
256 } 232 }
257 eventListener.add(new SkippedCompilationsPrinter()); 233 eventListener.add(new SkippedCompilationsPrinter());
258 } 234 }
259 if (firstConf['write_test_outcome_log'] as bool) { 235
236 if (firstConf.writeTestOutcomeLog) {
260 eventListener.add(new TestOutcomeLogWriter()); 237 eventListener.add(new TestOutcomeLogWriter());
261 } 238 }
262 if (firstConf['copy_coredumps'] as bool) { 239
240 if (firstConf.copyCoreDumps) {
263 eventListener.add(new UnexpectedCrashLogger()); 241 eventListener.add(new UnexpectedCrashLogger());
264 } 242 }
265 243
266 // The only progress indicator when listing tests should be the 244 // The only progress indicator when listing tests should be the
267 // the summary printer. 245 // the summary printer.
268 if (listTests) { 246 if (listTests) {
269 eventListener.add(new SummaryPrinter(jsonOnly: reportInJson)); 247 eventListener.add(new SummaryPrinter(jsonOnly: reportInJson));
270 } else { 248 } else {
271 eventListener.add(new ExitCodeSetter()); 249 eventListener.add(new ExitCodeSetter());
272 eventListener.add(new IgnoredTestMonitor()); 250 eventListener.add(new IgnoredTestMonitor());
273 } 251 }
274 252
275 // If any of the configurations need to access android devices we'll first 253 // If any of the configurations need to access android devices we'll first
276 // make a pool of all available adb devices. 254 // make a pool of all available adb devices.
277 AdbDevicePool adbDevicePool; 255 AdbDevicePool adbDevicePool;
278 var needsAdbDevicePool = configurations.any((Map conf) { 256 var needsAdbDevicePool = configurations.any((conf) {
279 return conf['runtime'] == 'dart_precompiled' && conf['system'] == 'android'; 257 return conf.runtime == Runtime.dartPrecompiled &&
258 conf.system == System.android;
280 }); 259 });
281 if (needsAdbDevicePool) { 260 if (needsAdbDevicePool) {
282 adbDevicePool = await AdbDevicePool.create(); 261 adbDevicePool = await AdbDevicePool.create();
283 } 262 }
284 263
285 // Start all the HTTP servers required before starting the process queue. 264 // Start all the HTTP servers required before starting the process queue.
286 if (!serverFutures.isEmpty) { 265 if (!serverFutures.isEmpty) {
287 await Future.wait(serverFutures); 266 await Future.wait(serverFutures);
288 } 267 }
289 268
290 if (Platform.isWindows) {
291 // When running tests on Windows, use cdb from depot_tools to dump
292 // stack traces of tests timing out.
293 try {
294 var text =
295 await new File(VS_TOOLCHAIN_FILE.toNativePath()).readAsString();
296 firstConf['win_sdk_path'] = JSON.decode(text)['win_sdk'];
297 } on dynamic {
298 // Ignore errors here. If win_sdk is not found, stack trace dumping
299 // for timeouts won't work.
300 }
301 }
302
303 // [firstConf] is needed here, since the ProcessQueue needs to know the 269 // [firstConf] is needed here, since the ProcessQueue needs to know the
304 // settings of 'noBatch' and 'local_ip' 270 // settings of 'noBatch' and 'local_ip'
305 new ProcessQueue( 271 new ProcessQueue(
306 firstConf, 272 firstConf,
307 maxProcesses, 273 maxProcesses,
308 maxBrowserProcesses, 274 maxBrowserProcesses,
309 startTime, 275 startTime,
310 testSuites, 276 testSuites,
311 eventListener, 277 eventListener,
312 allTestsFinished, 278 allTestsFinished,
313 verbose, 279 verbose,
314 recordingPath, 280 recordingPath,
315 recordingOutputPath, 281 replayPath,
316 adbDevicePool); 282 adbDevicePool);
317 } 283 }
OLDNEW

Powered by Google App Engine
This is Rietveld 408576698