| OLD | NEW |
| 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 part of testrunner; | 5 part of testrunner; |
| 6 | 6 |
| 7 /** Create and return an options parser for the test runner. */ | 7 /** Create and return an options parser for the test runner. */ |
| 8 ArgParser getOptionParser() { | 8 ArgParser getOptionParser() { |
| 9 var parser = new ArgParser(); | 9 var parser = new ArgParser(); |
| 10 | 10 |
| 11 parser.addOption('help', abbr: '?', | 11 parser.addOption('help', abbr: '?', |
| 12 help: 'Show usage information.'); | 12 help: 'Show usage information.'); |
| 13 | 13 |
| 14 parser.addOption('runtime', abbr: 'r', defaultsTo: 'vm', | 14 parser.addOption('runtime', abbr: 'r', defaultsTo: 'vm', |
| 15 help: 'Where the tests should be run.', | 15 help: 'Where the tests should be run.', |
| 16 allowed: ['vm', 'drt-dart', 'drt-js'], | 16 allowed: ['vm', 'drt-dart', 'drt-js'], |
| 17 allowedHelp: { | 17 allowedHelp: { |
| 18 'vm': 'Run Dart code natively on the standalone dart vm.', | 18 'vm': 'Run Dart code natively on the standalone dart vm.', |
| 19 'drt-dart': 'Run Dart code natively in the headless version of\n' | 19 'drt-dart': 'Run Dart code natively in the headless version of\n' |
| 20 'Chrome, DumpRenderTree.', | 20 'Chrome, DumpRenderTree.', |
| 21 'drt-js': 'Run Dart compiled to JavaScript in the headless version\n' | 21 'drt-js': 'Run Dart compiled to JavaScript in the headless version\n' |
| 22 'of Chrome, DumpRenderTree.' | 22 'of Chrome, DumpRenderTree.' |
| 23 }); | 23 }); |
| 24 | 24 |
| 25 parser.addFlag('checked', defaultsTo: false, | 25 parser.addFlag('checked', defaultsTo: false, |
| 26 help: 'Run tests in checked mode.'); | 26 help: 'Run tests in checked mode.'); |
| 27 | 27 |
| 28 parser.addFlag('sort', defaultsTo: false, |
| 29 help: 'Sort test files before running.'); |
| 30 |
| 28 parser.addFlag('layout-text', defaultsTo: false, | 31 parser.addFlag('layout-text', defaultsTo: false, |
| 29 help: 'Run text layout tests.'); | 32 help: 'Run text layout tests.'); |
| 30 | 33 |
| 31 parser.addFlag('layout-pixel', defaultsTo: false, | 34 parser.addFlag('layout-pixel', defaultsTo: false, |
| 32 help: 'Run pixel layout tests.'); | 35 help: 'Run pixel layout tests.'); |
| 33 | 36 |
| 34 parser.addOption('timeout', abbr: 't', | 37 parser.addOption('timeout', abbr: 't', |
| 35 help: 'Timeout in seconds', defaultsTo: '60'); | 38 help: 'Timeout in seconds', defaultsTo: '60'); |
| 36 | 39 |
| 37 parser.addOption('tasks', abbr: 'j', | 40 parser.addOption('tasks', abbr: 'j', |
| (...skipping 33 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 71 | 74 |
| 72 parser.addFlag('list-files', defaultsTo: false, | 75 parser.addFlag('list-files', defaultsTo: false, |
| 73 help: 'List test files only, do not run them.'); | 76 help: 'List test files only, do not run them.'); |
| 74 | 77 |
| 75 parser.addFlag('list-tests', defaultsTo: false, | 78 parser.addFlag('list-tests', defaultsTo: false, |
| 76 help: 'List tests only, do not run them.'); | 79 help: 'List tests only, do not run them.'); |
| 77 | 80 |
| 78 parser.addFlag('list-groups', defaultsTo: false, | 81 parser.addFlag('list-groups', defaultsTo: false, |
| 79 help: 'List test groups only, do not run tests.'); | 82 help: 'List test groups only, do not run tests.'); |
| 80 | 83 |
| 81 parser.addFlag('keep-files', defaultsTo: false, | 84 parser.addFlag('clean-files', defaultsTo: false, |
| 82 help: 'Keep the generated files in the temporary directory.'); | 85 help: 'Remove the generated files from the temporary directory.'); |
| 83 | 86 |
| 84 parser.addFlag('list-options', defaultsTo: false, | 87 parser.addFlag('list-options', defaultsTo: false, |
| 85 help: 'Print non-default option settings, usable as a test.config.'); | 88 help: 'Print non-default option settings, usable as a test.config.'); |
| 86 | 89 |
| 87 parser.addFlag('list-all-options', defaultsTo: false, | 90 parser.addFlag('list-all-options', defaultsTo: false, |
| 88 help: 'Print all option settings, usable as a test.config.'); | 91 help: 'Print all option settings, usable as a test.config.'); |
| 89 | 92 |
| 90 parser.addFlag('time', | 93 parser.addFlag('time', |
| 91 help: 'Print timing information after running tests', | 94 help: 'Print timing information after running tests', |
| 92 defaultsTo: false); | 95 defaultsTo: false); |
| 93 | 96 |
| 94 parser.addFlag('stop-on-failure', defaultsTo: false, | 97 parser.addFlag('stop-on-failure', defaultsTo: false, |
| 95 help: 'Stop execution upon first failure.'); | 98 help: 'Stop execution after first file with failures.'); |
| 96 | 99 |
| 97 parser.addFlag('isolate', defaultsTo: false, | 100 parser.addFlag('isolate', defaultsTo: false, |
| 98 help: 'Runs each test in a separate isolate.'); | 101 help: 'Runs each test in a separate isolate.'); |
| 99 | 102 |
| 100 parser.addOption('configfile', help: 'Path to an argument file to load.'); | 103 parser.addOption('configfile', help: 'Path to an argument file to load.'); |
| 101 | 104 |
| 102 parser.addOption('dartsdk', help: 'Path to dart SDK.'); | 105 parser.addOption('dartsdk', help: 'Path to dart SDK.'); |
| 103 | 106 |
| 104 // The defaults here should be the name of the executable, with | 107 var tmp; |
| 105 // the assumption that it is available on the PATH. | 108 if (Platform.operatingSystem == 'windows') { |
| 106 parser.addOption('dart2js', help: 'Path to dart2js executable.', | 109 tmp = "c:\\tmp\\test"; |
| 107 defaultsTo: 'dart2js'); | 110 } else { |
| 108 parser.addOption('dart', help: 'Path to dart executable.', | 111 tmp = "/tmp/test"; |
| 109 defaultsTo: 'dart'); | 112 } |
| 110 parser.addOption('drt', help: 'Path to DumpRenderTree executable.', | |
| 111 defaultsTo: 'drt'); | |
| 112 | |
| 113 parser.addOption('tempdir', help: 'Directory to store temp files.', | 113 parser.addOption('tempdir', help: 'Directory to store temp files.', |
| 114 defaultsTo: '${Platform.pathSeparator}tmp' | 114 defaultsTo: tmp); |
| 115 '${Platform.pathSeparator}testrunner'); | |
| 116 | 115 |
| 117 parser.addOption('test-file-pattern', | 116 parser.addOption('test-file-pattern', |
| 118 help: 'A regular expression that test file names must match ' | 117 help: 'A regular expression that test file names must match ' |
| 119 'to be considered', defaultsTo: '_test.dart\$'); | 118 'to be considered', defaultsTo: '_test.dart\$'); |
| 120 | 119 |
| 121 parser.addOption('include', | 120 parser.addOption('include', |
| 122 help: 'Only run tests from the specified group(s).', | 121 help: 'Only run tests from the specified group(s).', |
| 123 allowMultiple: true); | 122 allowMultiple: true); |
| 124 | 123 |
| 125 parser.addOption('exclude', | 124 parser.addOption('exclude', |
| (...skipping 14 matching lines...) Expand all Loading... |
| 140 help: 'Regenerate layout test expectation files.', | 139 help: 'Regenerate layout test expectation files.', |
| 141 defaultsTo: false); | 140 defaultsTo: false); |
| 142 | 141 |
| 143 parser.addFlag('server', help: 'Run an HTTP server.', defaultsTo: false); | 142 parser.addFlag('server', help: 'Run an HTTP server.', defaultsTo: false); |
| 144 | 143 |
| 145 parser.addOption('port', help: 'Port to use for HTTP server'); | 144 parser.addOption('port', help: 'Port to use for HTTP server'); |
| 146 | 145 |
| 147 parser.addOption('root', | 146 parser.addOption('root', |
| 148 help: 'Root directory for HTTP server for static files'); | 147 help: 'Root directory for HTTP server for static files'); |
| 149 | 148 |
| 150 parser.addOption('unittest', help: '#import path for unit test library.'); | |
| 151 | |
| 152 parser.addOption('pipeline', | 149 parser.addOption('pipeline', |
| 153 help: 'Pipeline script to use to run each test file.', | 150 help: 'Pipeline script to use to run each test file.', |
| 154 defaultsTo: 'run_pipeline.dart'); | 151 defaultsTo: 'run_pipeline.dart'); |
| 155 | 152 |
| 156 return parser; | 153 return parser; |
| 157 } | 154 } |
| 158 | 155 |
| 159 /** Print a value option, quoting it if it has embedded spaces. */ | 156 /** Print a value option, quoting it if it has embedded spaces. */ |
| 160 _printValueOption(String name, value, OutputStream stream) { | 157 _printValueOption(String name, value, IOSink dest) { |
| 161 if (value.indexOf(' ') >= 0) { | 158 if (value.indexOf(' ') >= 0) { |
| 162 stream.writeString("--$name='$value'\n"); | 159 dest.write("--$name='$value'\n"); |
| 163 } else { | 160 } else { |
| 164 stream.writeString("--$name=$value\n"); | 161 dest.write("--$name=$value\n"); |
| 165 } | 162 } |
| 166 } | 163 } |
| 167 | 164 |
| 168 /** Print the current option values. */ | 165 /** Print the current option values. */ |
| 169 printOptions(ArgParser parser, ArgResults arguments, | 166 printOptions(ArgParser parser, ArgResults arguments, |
| 170 bool includeDefaults, OutputStream stream) { | 167 bool includeDefaults, IOSink dest) { |
| 171 if (stream == null) return; | 168 if (dest == null) return; |
| 172 for (var name in arguments.options) { | 169 for (var name in arguments.options) { |
| 173 if (!name.startsWith('list-')) { | 170 if (!name.startsWith('list-')) { |
| 174 var value = arguments[name]; | 171 var value = arguments[name]; |
| 175 var defaultValue = parser.getDefault(name); | 172 var defaultValue = parser.getDefault(name); |
| 176 if (value is bool) { | 173 if (value is bool) { |
| 177 if (includeDefaults || (value != defaultValue)) { | 174 if (includeDefaults || (value != defaultValue)) { |
| 178 stream.writeString('--${value ? "" : "no-"}$name\n'); | 175 dest.write('--${value ? "" : "no-"}$name\n'); |
| 179 } | 176 } |
| 180 } else if (value is List) { | 177 } else if (value is List) { |
| 181 if (value.length > 0) { | 178 if (value.length > 0) { |
| 182 for (var v in value) { | 179 for (var v in value) { |
| 183 _printValueOption(name, v, stream); | 180 _printValueOption(name, v, dest); |
| 184 } | 181 } |
| 185 } | 182 } |
| 186 } else if (value != null && (includeDefaults || value != defaultValue)) { | 183 } else if (value != null && (includeDefaults || value != defaultValue)) { |
| 187 _printValueOption(name, value, stream); | 184 _printValueOption(name, value, dest); |
| 188 } | 185 } |
| 189 } | 186 } |
| 190 } | 187 } |
| 191 } | 188 } |
| 192 | 189 |
| 193 /** | 190 /** |
| 194 * Get the test runner configuration. This loads options from multiple | 191 * Get the test runner configuration. This loads options from multiple |
| 195 * sources, in increasing order of priority: a test.config file in the | 192 * sources, in increasing order of priority: a test.config file in the |
| 196 * current directory, a test config file specified with --configfile on | 193 * current directory, a test config file specified with --configfile on |
| 197 * the command line, and other arguments specified on the command line. | 194 * the command line, and other arguments specified on the command line. |
| 198 */ | 195 */ |
| 199 ArgResults loadConfiguration(optionsParser) { | 196 ArgResults loadConfiguration(optionsParser) { |
| 200 var options = new List(); | 197 var options = new List(); |
| 201 // We first load options from a test.config file in the working directory. | 198 // We first load options from a test.config file in the working directory. |
| 202 options.addAll(getFileContents('test.config', false). | 199 options.addAll(getFileContents('test.config', false). |
| 203 where((e) => e.trim().length > 0 && e[0] != '#')); | 200 where((e) => e.trim().length > 0 && e[0] != '#')); |
| 204 // Next we look to see if the command line included a -testconfig argument, | 201 // Next we look to see if the command line included a -testconfig argument, |
| 205 // and if so, load options from that file too; where these are not | 202 // and if so, load options from that file too; where these are not |
| 206 // multi-valued they will take precedence over the ones in test.config. | 203 // multi-valued they will take precedence over the ones in test.config. |
| 207 var commandLineArgs = new Options().arguments; | 204 var commandLineArgs = new Options().arguments; |
| 208 var cfgarg = '--configfile'; | 205 var cfgarg = '--configfile'; |
| 206 var cfgarge = '--configfile='; |
| 209 for (var i = 0; i < commandLineArgs.length; i++) { | 207 for (var i = 0; i < commandLineArgs.length; i++) { |
| 210 if (commandLineArgs[i].startsWith(cfgarg)) { | 208 if (commandLineArgs[i].startsWith(cfgarg)) { |
| 211 if (commandLineArgs[i] == cfgarg) { | 209 if (commandLineArgs[i] == cfgarg) { |
| 212 if (i == commandLineArgs.length - 1) { | 210 if (i == commandLineArgs.length - 1) { |
| 213 throw new Exception('Missing argument to $cfgarg'); | 211 throw new Exception('Missing argument to $cfgarg'); |
| 214 } | 212 } |
| 215 options.addAll(getFileContents(commandLineArgs[++i], true). | 213 options.addAll(getFileContents(commandLineArgs[++i], true). |
| 216 where((e) => e.trim().length > 0 && e[0] != '#')); | 214 where((e) => e.trim().length > 0 && e[0] != '#')); |
| 217 } else if (commandLineArgs[i].startsWith('$cfgarg=')) { | 215 } else if (commandLineArgs[i].startsWith(cfgarge)) { |
| 218 options.addAll( | 216 options.addAll( |
| 219 getFileContents(commandLineArgs[i].substring(cfgarg.length), true). | 217 getFileContents(commandLineArgs[i].substring(cfgarge.length), true). |
| 220 where((e) => e.trim().length > 0 && e[0] != '#')); | 218 where((e) => e.trim().length > 0 && e[0] != '#')); |
| 221 } else { | 219 } else { |
| 222 throw new Exception('Missing argument to $cfgarg'); | 220 throw new Exception('Missing argument to $cfgarg'); |
| 223 } | 221 } |
| 224 } | 222 } |
| 225 } | 223 } |
| 226 // Finally, we add options from the command line. These have the highest | 224 // Finally, we add options from the command line. These have the highest |
| 227 // precedence of all. | 225 // precedence of all. |
| 228 options.addAll(commandLineArgs); | 226 options.addAll(commandLineArgs); |
| 229 // Now try parse the whole collection of options, and if this fails, | 227 // Now try parse the whole collection of options, and if this fails, |
| (...skipping 10 matching lines...) Expand all Loading... |
| 240 | 238 |
| 241 /** Perform some sanity checking of the configuration. */ | 239 /** Perform some sanity checking of the configuration. */ |
| 242 bool isSane(ArgResults config) { | 240 bool isSane(ArgResults config) { |
| 243 if (config == null) { | 241 if (config == null) { |
| 244 return false; | 242 return false; |
| 245 } | 243 } |
| 246 if (config['runtime'] == null) { | 244 if (config['runtime'] == null) { |
| 247 print('Missing required option --runtime'); | 245 print('Missing required option --runtime'); |
| 248 return false; | 246 return false; |
| 249 } | 247 } |
| 250 if (config['unittest'] == null) { | |
| 251 print('Missing required option --unittest'); | |
| 252 return false; | |
| 253 } | |
| 254 if (config['include'].length > 0 && | 248 if (config['include'].length > 0 && |
| 255 config['exclude'].length > 0) { | 249 config['exclude'].length > 0) { |
| 256 print('--include and --exclude are mutually exclusive.'); | 250 print('--include and --exclude are mutually exclusive.'); |
| 257 return false; | 251 return false; |
| 258 } | 252 } |
| 259 if ((config['layout-text'] || config['layout-pixel']) && | 253 if ((config['layout-text'] || config['layout-pixel']) && |
| 260 config['runtime'] == 'vm') { | 254 config['runtime'] == 'vm') { |
| 261 print('Layout tests must use --runtime values of "drt-dart" or "drt-js"'); | 255 print('Layout tests must use --runtime values of "drt-dart" or "drt-js"'); |
| 262 return false; | 256 return false; |
| 263 } | 257 } |
| 264 return true; | 258 return true; |
| 265 } | 259 } |
| OLD | NEW |