| OLD | NEW |
|---|
| (Empty) |
| 1 //#!/usr/bin/env dart | |
| 2 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | |
| 3 // for details. All rights reserved. Use of this source code is governed by a | |
| 4 // BSD-style license that can be found in the LICENSE file. | |
| 5 | |
| 6 /** | |
| 7 * testrunner is a program to run Dart unit tests. Unlike $DART/tools/test.dart, | |
| 8 * this program is intended for 3rd parties to be able to run unit tests in | |
| 9 * a batched fashion. As such, it adds some features and removes others. Some | |
| 10 * of the removed features are: | |
| 11 * | |
| 12 * - No support for test.status files. The assumption is that tests are | |
| 13 * expected to pass. Status file support will be added in the future. | |
| 14 * - A restricted set of runtimes. The assumption here is that the Dart | |
| 15 * libraries deal with platform dependencies, and so the primary | |
| 16 * SKUs that a user of this app would be concerned with would be | |
| 17 * Dart-native versus compiled, and client (browser) vs server. To | |
| 18 * support these, three runtimes are allowed: 'drt-dart' and 'drt-js' (for | |
| 19 * client native and client-compiled, respectively), and 'vm' | |
| 20 * (for server-side native). | |
| 21 * - No sharding of test processes. | |
| 22 * | |
| 23 * On the other hand, a number of features have been added: | |
| 24 * | |
| 25 * - The ability to filter tests by group or name. | |
| 26 * - The ability to run tests in isolates. | |
| 27 * - The ability to customize the format of the test result messages. | |
| 28 * - The ability to list the tests available. | |
| 29 * | |
| 30 * By default, testrunner will run all tests in the current directory. | |
| 31 * With a -R option, it will recurse into subdirectories. | |
| 32 * Directories can also be specified on the command line; if | |
| 33 * any are specified they will override the use of the current directory. | |
| 34 * All files that match the `--test-file-pattern` will be included; by default | |
| 35 * this is files with names that end in _test.dart. | |
| 36 * | |
| 37 * Options can be specified on the command line, via a configuration | |
| 38 * file (`--config`) or via a test.config file in the test directory, | |
| 39 * in decreasing order of priority. | |
| 40 * | |
| 41 * The three runtimes are: | |
| 42 * | |
| 43 * vm - run native Dart in the VM; i.e. using $DARTSDK/dart-sdk/bin/dart. | |
| 44 * TODO(antonm): fix the option name. | |
| 45 * drt-dart - run native Dart in content shell, the headless version of | |
| 46 * Dartium, which is located in $DARTSDK/chromium/content_shell, if | |
| 47 * you installed the SDK that is bundled with the editor, or available | |
| 48 * from http://gsdview.appspot.com/dartium-archive/continuous/ | |
| 49 * otherwise. | |
| 50 * | |
| 51 * TODO(antonm): fix the option name. | |
| 52 * drt-js - run Dart compiled to Javascript in content shell. | |
| 53 * | |
| 54 * testrunner supports simple DOM render tests. These can use expected values | |
| 55 * for the render output from content shell, either are textual DOM | |
| 56 * descriptions (`--layout-tests`) or pixel renderings (`--pixel-tests`). | |
| 57 * When running layout tests, testrunner will see if there is a file with | |
| 58 * a .png or a .txt extension in a directory with the same name as the | |
| 59 * test file (without extension) and with the test name as the file name. | |
| 60 * For example, if there is a test file foo_test.dart with tests 'test1' | |
| 61 * and 'test2', it will look for foo_test/test1.txt and foo_test/test2.txt | |
| 62 * for text render layout files. If these exist it will do additional checks | |
| 63 * of the rendered layout; if not, the test will fail. | |
| 64 * | |
| 65 * Layout file (re)generation can be done using `--regenerate`. This will | |
| 66 * create or update the layout files (and implicitly pass the tests). | |
| 67 * | |
| 68 * The wrapping and execution of test files is handled by test_pipeline.dart, | |
| 69 * which is run in an isolate. The `--pipeline` argument can be used to | |
| 70 * specify a different script for running a test file pipeline, allowing | |
| 71 * customization of the pipeline. | |
| 72 * | |
| 73 * Wrapper files are created for tests in the tmp directory, which can be | |
| 74 * overridden with --tempdir. These files are not removed after the tests | |
| 75 * are complete, primarily to reduce the amount of times pub must be | |
| 76 * executed. You can use --clean-files to force file cleanup. The temp | |
| 77 * directories will have pubspec.yaml files auto-generated unless the | |
| 78 * original test file directories have such files; in that case the existing | |
| 79 * files will be copied. Whenever a new pubspec file is copied or | |
| 80 * created pub will be run (but not otherwise - so if you want to do | |
| 81 * the equivelent of pub update you should use --clean-files and the rerun | |
| 82 * the tests). | |
| 83 * | |
| 84 * TODO(gram): if the user has a pubspec.yaml file, we should inspect the | |
| 85 * pubspec.lock file and give useful errors: | |
| 86 * - if the lock file doesn't exit, then run pub install | |
| 87 * - if it exists and it doesn't have the required packages (unittest or | |
| 88 * browser), ask the user to add them and run pub install again. | |
| 89 */ | |
| 90 | |
| 91 library testrunner; | |
| 92 import 'dart:async'; | |
| 93 import 'dart:io'; | |
| 94 import 'dart:isolate'; | |
| 95 import 'dart:math'; | |
| 96 import 'package:args/args.dart'; | |
| 97 | |
| 98 part 'options.dart'; | |
| 99 part 'utils.dart'; | |
| 100 | |
| 101 /** The set of [PipelineRunner]s to execute. */ | |
| 102 List _tasks; | |
| 103 | |
| 104 /** The maximum number of pipelines that can run concurrently. */ | |
| 105 int _maxTasks; | |
| 106 | |
| 107 /** The number of pipelines currently running. */ | |
| 108 int _numTasks; | |
| 109 | |
| 110 /** The index of the next pipeline runner to execute. */ | |
| 111 int _nextTask; | |
| 112 | |
| 113 /** The sink to use for high-value messages, like test results. */ | |
| 114 IOSink _outSink; | |
| 115 | |
| 116 /** The sink to use for low-value messages, like verbose output. */ | |
| 117 IOSink _logSink; | |
| 118 | |
| 119 /** | |
| 120 * The last temp test directory we accessed; we use this to know if we | |
| 121 * need to check the pub configuration. | |
| 122 */ | |
| 123 String _testDir; | |
| 124 | |
| 125 /** | |
| 126 * The user can specify output streams on the command line, using 'none', | |
| 127 * 'stdout', 'stderr', or a file path; [getSink] will take such a name | |
| 128 * and return an appropriate [IOSink]. | |
| 129 */ | |
| 130 IOSink getSink(String name) { | |
| 131 if (name == null || name == 'none') { | |
| 132 return null; | |
| 133 } | |
| 134 if (name == 'stdout') { | |
| 135 return stdout; | |
| 136 } | |
| 137 if (name == 'stderr') { | |
| 138 return stderr; | |
| 139 } | |
| 140 var f = new File(name); | |
| 141 return f.openWrite(); | |
| 142 } | |
| 143 | |
| 144 /** | |
| 145 * Given a [List] of [testFiles], either print the list or create | |
| 146 * and execute pipelines for the files. | |
| 147 */ | |
| 148 void processTests(Map config, List testFiles) { | |
| 149 _outSink = getSink(config['out']); | |
| 150 _logSink = getSink(config['log']); | |
| 151 if (config['list-files']) { | |
| 152 if (_outSink != null) { | |
| 153 for (var i = 0; i < testFiles.length; i++) { | |
| 154 _outSink.write(testFiles[i]); | |
| 155 _outSink.write('\n'); | |
| 156 } | |
| 157 } | |
| 158 } else { | |
| 159 _maxTasks = min(config['tasks'], testFiles.length); | |
| 160 _numTasks = 0; | |
| 161 _nextTask = 0; | |
| 162 spawnTasks(config, testFiles); | |
| 163 } | |
| 164 } | |
| 165 | |
| 166 /** | |
| 167 * Create or update a pubspec for the target test directory. We use the | |
| 168 * source directory pubspec if available; otherwise we create a minimal one. | |
| 169 * We return a Future if we are running pub install, or null otherwise. | |
| 170 */ | |
| 171 Future doPubConfig(Path sourcePath, String sourceDir, | |
| 172 Path targetPath, String targetDir, | |
| 173 String pub, String runtime) { | |
| 174 // Make sure the target directory exists. | |
| 175 var d = new Directory(targetDir); | |
| 176 if (!d.existsSync()) { | |
| 177 d.createSync(recursive: true); | |
| 178 } | |
| 179 | |
| 180 // If the source has no pubspec, but the dest does, leave | |
| 181 // things as they are. If neither do, create one in dest. | |
| 182 | |
| 183 var sourcePubSpecName = new Path(sourceDir).append("pubspec.yaml"). | |
| 184 toNativePath(); | |
| 185 var targetPubSpecName = new Path(targetDir).append("pubspec.yaml"). | |
| 186 toNativePath(); | |
| 187 var sourcePubSpec = new File(sourcePubSpecName); | |
| 188 var targetPubSpec = new File(targetPubSpecName); | |
| 189 | |
| 190 if (!sourcePubSpec.existsSync()) { | |
| 191 if (targetPubSpec.existsSync()) { | |
| 192 return null; | |
| 193 } else { | |
| 194 // Create one. | |
| 195 if (runtime == 'vm') { | |
| 196 writeFile(targetPubSpecName, | |
| 197 "name: testrunner\ndependencies:\n unittest: any\n"); | |
| 198 } else { | |
| 199 writeFile(targetPubSpecName, | |
| 200 "name: testrunner\ndependencies:\n unittest: any\n browser: any\n"); | |
| 201 } | |
| 202 } | |
| 203 } else { | |
| 204 if (targetPubSpec.existsSync()) { | |
| 205 // If there is a source one, and it is older than the target, | |
| 206 // leave the target as is. | |
| 207 if (sourcePubSpec.lastModifiedSync().millisecondsSinceEpoch < | |
| 208 targetPubSpec.lastModifiedSync().millisecondsSinceEpoch) { | |
| 209 return null; | |
| 210 } | |
| 211 } | |
| 212 // Source exists and is newer than target or there is no target; | |
| 213 // copy the source to the target. If there is a pubspec.lock file, | |
| 214 // copy that too. | |
| 215 var s = sourcePubSpec.readAsStringSync(); | |
| 216 targetPubSpec.writeAsStringSync(s); | |
| 217 var sourcePubLock = new File(sourcePubSpecName.replaceAll(".yaml", ".lock"))
; | |
| 218 if (sourcePubLock.existsSync()) { | |
| 219 var targetPubLock = | |
| 220 new File(targetPubSpecName.replaceAll(".yaml", ".lock")); | |
| 221 s = sourcePubLock.readAsStringSync(); | |
| 222 targetPubLock.writeAsStringSync(s); | |
| 223 } | |
| 224 } | |
| 225 // A new target pubspec was created so run pub install. | |
| 226 return _processHelper(pub, [ 'install' ], workingDir: targetDir); | |
| 227 } | |
| 228 | |
| 229 /** Execute as many tasks as possible up to the maxTasks limit. */ | |
| 230 void spawnTasks(Map config, List testFiles) { | |
| 231 var verbose = config['verbose']; | |
| 232 // If we were running in the VM and the immediate flag was set, we have | |
| 233 // already printed the important messages (i.e. prefixed with ###), | |
| 234 // so we should skip them now. | |
| 235 var skipNonVerbose = config['immediate'] && config['runtime'] == 'vm'; | |
| 236 while (_numTasks < _maxTasks && _nextTask < testFiles.length) { | |
| 237 ++_numTasks; | |
| 238 var testfile = testFiles[_nextTask++]; | |
| 239 config['testfile'] = testfile; | |
| 240 ReceivePort port = new ReceivePort(); | |
| 241 port.receive((msg, _) { | |
| 242 List stdout = msg[0]; | |
| 243 List stderr = msg[1]; | |
| 244 List log = msg[2]; | |
| 245 int exitCode = msg[3]; | |
| 246 writelog(stdout, _outSink, _logSink, verbose, skipNonVerbose); | |
| 247 writelog(stderr, _outSink, _logSink, true, skipNonVerbose); | |
| 248 writelog(log, _outSink, _logSink, verbose, skipNonVerbose); | |
| 249 port.close(); | |
| 250 --_numTasks; | |
| 251 if (exitCode == 0 || !config['stop-on-failure']) { | |
| 252 spawnTasks(config, testFiles); | |
| 253 } | |
| 254 if (_numTasks == 0) { | |
| 255 // No outstanding tasks; we're all done. | |
| 256 // We could later print a summary report here. | |
| 257 } | |
| 258 }); | |
| 259 // Get the names of the source and target test files and containing | |
| 260 // directories. | |
| 261 var testPath = new Path(testfile); | |
| 262 var sourcePath = testPath.directoryPath; | |
| 263 var sourceDir = sourcePath.toNativePath(); | |
| 264 | |
| 265 var targetPath = new Path(config["tempdir"]); | |
| 266 var normalizedTarget = testPath.directoryPath.toNativePath() | |
| 267 .replaceAll(Platform.pathSeparator, '_') | |
| 268 .replaceAll(':', '_'); | |
| 269 targetPath = targetPath.append("${normalizedTarget}_${config['runtime']}"); | |
| 270 var targetDir = targetPath.toNativePath(); | |
| 271 | |
| 272 config['targetDir'] = targetDir; | |
| 273 // If this is a new target dir, we need to redo the pub check. | |
| 274 var f = null; | |
| 275 if (targetDir != _testDir) { | |
| 276 f = doPubConfig(sourcePath, sourceDir, targetPath, targetDir, | |
| 277 config['pub'], config['runtime']); | |
| 278 _testDir = targetDir; | |
| 279 } | |
| 280 var response = new ReceivePort(); | |
| 281 spawnUri(config['pipeline'], [], response) | |
| 282 .then((_) => f) | |
| 283 .then((_) => response.first) | |
| 284 .then((s) { s.send([config, port.sendPort]); }); | |
| 285 if (f != null) break; // Don't do any more until pub is done. | |
| 286 } | |
| 287 } | |
| 288 | |
| 289 /** | |
| 290 * Our tests are configured so that critical messages have a '###' prefix. | |
| 291 * [writelog] takes the output from a pipeline execution and writes it to | |
| 292 * our output sinks. It will strip the '###' if necessary on critical | |
| 293 * messages; other messages will only be written if verbose output was | |
| 294 * specified. | |
| 295 */ | |
| 296 void writelog(List messages, IOSink out, IOSink log, | |
| 297 bool includeVerbose, bool skipNonVerbose) { | |
| 298 for (var i = 0; i < messages.length; i++) { | |
| 299 var msg = messages[i]; | |
| 300 if (msg.startsWith('###')) { | |
| 301 if (!skipNonVerbose && out != null) { | |
| 302 out.write(msg.substring(3)); | |
| 303 out.write('\n'); | |
| 304 } | |
| 305 } else if (msg.startsWith('CONSOLE MESSAGE:')) { | |
| 306 if (!skipNonVerbose && out != null) { | |
| 307 int idx = msg.indexOf('###'); | |
| 308 if (idx > 0) { | |
| 309 out.write(msg.substring(idx + 3)); | |
| 310 out.write('\n'); | |
| 311 } | |
| 312 } | |
| 313 } else if (includeVerbose && log != null) { | |
| 314 log.write(msg); | |
| 315 log.write('\n'); | |
| 316 } | |
| 317 } | |
| 318 } | |
| 319 | |
| 320 normalizeFilter(List filter) { | |
| 321 // We want the filter to be a quoted string or list of quoted | |
| 322 // strings. | |
| 323 for (var i = 0; i < filter.length; i++) { | |
| 324 var f = filter[i]; | |
| 325 if (f[0] != "'" && f[0] != '"') { | |
| 326 filter[i] = "'$f'"; // TODO(gram): Quote embedded quotes. | |
| 327 } | |
| 328 } | |
| 329 return filter; | |
| 330 } | |
| 331 | |
| 332 void sanitizeConfig(Map config, ArgParser parser) { | |
| 333 config['layout'] = config['layout-text'] || config['layout-pixel']; | |
| 334 config['verbose'] = (config['log'] != 'none' && !config['list-groups']); | |
| 335 config['timeout'] = int.parse(config['timeout']); | |
| 336 config['tasks'] = int.parse(config['tasks']); | |
| 337 | |
| 338 var dartsdk = config['dartsdk']; | |
| 339 var pathSep = Platform.pathSeparator; | |
| 340 | |
| 341 if (dartsdk == null) { | |
| 342 var runner = Platform.executable; | |
| 343 var idx = runner.indexOf('dart-sdk'); | |
| 344 if (idx < 0) { | |
| 345 print("Please use --dartsdk option or run using the dart executable " | |
| 346 "from the Dart SDK"); | |
| 347 exit(0); | |
| 348 } | |
| 349 dartsdk = runner.substring(0, idx); | |
| 350 } | |
| 351 if (Platform.operatingSystem == 'macos') { | |
| 352 config['dart2js'] = | |
| 353 '$dartsdk${pathSep}dart-sdk${pathSep}bin${pathSep}dart2js'; | |
| 354 config['dart'] = '$dartsdk${pathSep}dart-sdk${pathSep}bin${pathSep}dart'; | |
| 355 config['pub'] = '$dartsdk${pathSep}dart-sdk${pathSep}bin${pathSep}pub'; | |
| 356 config['drt'] = | |
| 357 '$dartsdk/chromium/Content Shell.app/Contents/MacOS/Content Shell'; | |
| 358 } else if (Platform.operatingSystem == 'linux') { | |
| 359 config['dart2js'] = | |
| 360 '$dartsdk${pathSep}dart-sdk${pathSep}bin${pathSep}dart2js'; | |
| 361 config['dart'] = '$dartsdk${pathSep}dart-sdk${pathSep}bin${pathSep}dart'; | |
| 362 config['pub'] = '$dartsdk${pathSep}dart-sdk${pathSep}bin${pathSep}pub'; | |
| 363 config['drt'] = '$dartsdk${pathSep}chromium${pathSep}content_shell'; | |
| 364 } else { | |
| 365 config['dart2js'] = | |
| 366 '$dartsdk${pathSep}dart-sdk${pathSep}bin${pathSep}dart2js.bat'; | |
| 367 config['dart'] = '$dartsdk${pathSep}dart-sdk${pathSep}bin${pathSep}dart.exe'
; | |
| 368 config['pub'] = '$dartsdk${pathSep}dart-sdk${pathSep}bin${pathSep}pub.bat'; | |
| 369 config['drt'] = '$dartsdk${pathSep}chromium${pathSep}content_shell.exe'; | |
| 370 } | |
| 371 | |
| 372 for (var prog in [ 'drt', 'dart', 'pub', 'dart2js' ]) { | |
| 373 config[prog] = makePathAbsolute(config[prog]); | |
| 374 } | |
| 375 config['runnerDir'] = runnerDirectory; | |
| 376 config['include'] = normalizeFilter(config['include']); | |
| 377 config['exclude'] = normalizeFilter(config['exclude']); | |
| 378 } | |
| 379 | |
| 380 main(List<String> arguments) { | |
| 381 var optionsParser = getOptionParser(); | |
| 382 var options = loadConfiguration(optionsParser, arguments); | |
| 383 if (isSane(options)) { | |
| 384 if (options['list-options']) { | |
| 385 printOptions(optionsParser, options, false, stdout); | |
| 386 } else if (options['list-all-options']) { | |
| 387 printOptions(optionsParser, options, true, stdout); | |
| 388 } else { | |
| 389 var config = new Map(); | |
| 390 for (var option in options.options) { | |
| 391 config[option] = options[option]; | |
| 392 } | |
| 393 var rest = []; | |
| 394 // Process the remmaining command line args. If they look like | |
| 395 // options then split them up and add them to the map; they may be for | |
| 396 // custom pipelines. | |
| 397 for (var other in options.rest) { | |
| 398 var idx; | |
| 399 if (other.startsWith('--') && (idx = other.indexOf('=')) > 0) { | |
| 400 var optName = other.substring(2, idx); | |
| 401 var optValue = other.substring(idx+1); | |
| 402 config[optName] = optValue; | |
| 403 } else { | |
| 404 rest.add(other); | |
| 405 } | |
| 406 } | |
| 407 | |
| 408 sanitizeConfig(config, optionsParser); | |
| 409 | |
| 410 // Build the list of tests and then execute them. | |
| 411 List dirs = rest; | |
| 412 if (dirs.length == 0) { | |
| 413 dirs.add('.'); // Use current working directory as default. | |
| 414 } | |
| 415 var f = buildFileList(dirs, | |
| 416 new RegExp(config['test-file-pattern']), config['recurse']); | |
| 417 if (config['sort']) f.sort(); | |
| 418 processTests(config, f); | |
| 419 } | |
| 420 } | |
| 421 } | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 59093003:
Remove unmaintained utils/testrunner (Closed)
Base URL: https://dart.googlecode.com/svn/branches/bleeding_edge/dart