OLD | NEW |
1 // Copyright (c) 2015, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2015, the Dart project authors. Please see the AUTHORS file |
2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
4 | 4 |
5 import 'dart:io'; | 5 import 'dart:io'; |
6 import 'dart:math' as math; | |
7 | 6 |
8 import 'package:args/args.dart'; | |
9 import 'package:path/path.dart' as p; | 7 import 'package:path/path.dart' as p; |
10 | 8 |
11 import '../frontend/timeout.dart'; | 9 import '../frontend/timeout.dart'; |
12 import '../backend/metadata.dart'; | 10 import '../backend/metadata.dart'; |
13 import '../backend/test_platform.dart'; | 11 import '../backend/test_platform.dart'; |
14 import '../utils.dart'; | |
15 import '../util/io.dart'; | 12 import '../util/io.dart'; |
16 | 13 import 'configuration/args.dart' as args; |
17 /// The default number of test suites to run at once. | 14 import 'configuration/load.dart'; |
18 /// | 15 import 'configuration/values.dart'; |
19 /// This defaults to half the available processors, since presumably some of | |
20 /// them will be used for the OS and other processes. | |
21 final _defaultConcurrency = math.max(1, Platform.numberOfProcessors ~/ 2); | |
22 | 16 |
23 /// A class that encapsulates the command-line configuration of the test runner. | 17 /// A class that encapsulates the command-line configuration of the test runner. |
24 class Configuration { | 18 class Configuration { |
25 /// The parser used to parse the command-line arguments. | |
26 static final ArgParser _parser = (() { | |
27 var parser = new ArgParser(allowTrailingOptions: true); | |
28 | |
29 var allPlatforms = TestPlatform.all.toList(); | |
30 if (!Platform.isMacOS) allPlatforms.remove(TestPlatform.safari); | |
31 if (!Platform.isWindows) allPlatforms.remove(TestPlatform.internetExplorer); | |
32 | |
33 parser.addFlag("help", abbr: "h", negatable: false, | |
34 help: "Shows this usage information."); | |
35 parser.addFlag("version", negatable: false, | |
36 help: "Shows the package's version."); | |
37 parser.addOption("package-root", hide: true); | |
38 | |
39 parser.addSeparator("======== Selecting Tests"); | |
40 parser.addOption("name", | |
41 abbr: 'n', | |
42 help: 'A substring of the name of the test to run.\n' | |
43 'Regular expression syntax is supported.'); | |
44 parser.addOption("plain-name", | |
45 abbr: 'N', | |
46 help: 'A plain-text substring of the name of the test to run.'); | |
47 // TODO(nweiz): Support the full platform-selector syntax for choosing which | |
48 // tags to run. In the shorter term, disallow non-"identifier" tags. | |
49 parser.addOption("tags", | |
50 abbr: 't', | |
51 help: 'Run only tests with all of the specified tags.', | |
52 allowMultiple: true); | |
53 parser.addOption("tag", hide: true, allowMultiple: true); | |
54 parser.addOption("exclude-tags", | |
55 abbr: 'x', | |
56 help: "Don't run tests with any of the specified tags.", | |
57 allowMultiple: true); | |
58 parser.addOption("exclude-tag", hide: true, allowMultiple: true); | |
59 | |
60 parser.addSeparator("======== Running Tests"); | |
61 parser.addOption("platform", | |
62 abbr: 'p', | |
63 help: 'The platform(s) on which to run the tests.', | |
64 allowed: allPlatforms.map((platform) => platform.identifier).toList(), | |
65 defaultsTo: 'vm', | |
66 allowMultiple: true); | |
67 parser.addOption("concurrency", | |
68 abbr: 'j', | |
69 help: 'The number of concurrent test suites run.\n' | |
70 '(defaults to $_defaultConcurrency)', | |
71 valueHelp: 'threads'); | |
72 parser.addOption("pub-serve", | |
73 help: 'The port of a pub serve instance serving "test/".', | |
74 valueHelp: 'port'); | |
75 | |
76 // Note: although we list the 30s default timeout as though it were a | |
77 // default value for this argument, it's actually encoded in the [Invoker]'s | |
78 // call to [Timeout.apply]. | |
79 parser.addOption("timeout", | |
80 help: 'The default test timeout. For example: 15s, 2x, none\n' | |
81 '(defaults to 30s)'); | |
82 parser.addFlag("pause-after-load", | |
83 help: 'Pauses for debugging before any tests execute.\n' | |
84 'Implies --concurrency=1 and --timeout=none.\n' | |
85 'Currently only supported for browser tests.', | |
86 negatable: false); | |
87 | |
88 parser.addSeparator("======== Output"); | |
89 parser.addOption("reporter", | |
90 abbr: 'r', | |
91 help: 'The runner used to print test results.', | |
92 allowed: ['compact', 'expanded', 'json'], | |
93 defaultsTo: Platform.isWindows ? 'expanded' : 'compact', | |
94 allowedHelp: { | |
95 'compact': 'A single line, updated continuously.', | |
96 'expanded': 'A separate line for each update.', | |
97 'json': 'A machine-readable format (see https://goo.gl/0HRhdZ).' | |
98 }); | |
99 parser.addFlag("verbose-trace", negatable: false, | |
100 help: 'Whether to emit stack traces with core library frames.'); | |
101 parser.addFlag("js-trace", negatable: false, | |
102 help: 'Whether to emit raw JavaScript stack traces for browser tests.'); | |
103 parser.addFlag("color", defaultsTo: null, | |
104 help: 'Whether to use terminal colors.\n(auto-detected by default)'); | |
105 | |
106 return parser; | |
107 })(); | |
108 | |
109 /// The usage string for the command-line arguments. | 19 /// The usage string for the command-line arguments. |
110 static String get usage => _parser.usage; | 20 static String get usage => args.usage; |
111 | 21 |
112 /// Whether `--help` was passed. | 22 /// Whether `--help` was passed. |
113 final bool help; | 23 bool get help => _help ?? false; |
| 24 final bool _help; |
114 | 25 |
115 /// Whether `--version` was passed. | 26 /// Whether `--version` was passed. |
116 final bool version; | 27 bool get version => _version ?? false; |
| 28 final bool _version; |
117 | 29 |
118 /// Whether stack traces should be presented as-is or folded to remove | 30 /// Whether stack traces should be presented as-is or folded to remove |
119 /// irrelevant packages. | 31 /// irrelevant packages. |
120 final bool verboseTrace; | 32 bool get verboseTrace => _verboseTrace ?? false; |
| 33 final bool _verboseTrace; |
121 | 34 |
122 /// Whether JavaScript stack traces should be left as-is or converted to | 35 /// Whether JavaScript stack traces should be left as-is or converted to |
123 /// Dart-like traces. | 36 /// Dart-like traces. |
124 final bool jsTrace; | 37 bool get jsTrace => _jsTrace ?? false; |
| 38 final bool _jsTrace; |
125 | 39 |
126 /// Whether to pause for debugging after loading each test suite. | 40 /// Whether to pause for debugging after loading each test suite. |
127 final bool pauseAfterLoad; | 41 bool get pauseAfterLoad => _pauseAfterLoad ?? false; |
| 42 final bool _pauseAfterLoad; |
128 | 43 |
129 /// The package root for resolving "package:" URLs. | 44 /// The package root for resolving "package:" URLs. |
130 final String packageRoot; | 45 String get packageRoot => _packageRoot ?? p.join(p.current, 'packages'); |
| 46 final String _packageRoot; |
131 | 47 |
132 /// The name of the reporter to use to display results. | 48 /// The name of the reporter to use to display results. |
133 final String reporter; | 49 String get reporter => _reporter ?? defaultReporter; |
| 50 final String _reporter; |
134 | 51 |
135 /// The URL for the `pub serve` instance from which to load tests, or `null` | 52 /// The URL for the `pub serve` instance from which to load tests, or `null` |
136 /// if tests should be loaded from the filesystem. | 53 /// if tests should be loaded from the filesystem. |
137 final Uri pubServeUrl; | 54 final Uri pubServeUrl; |
138 | 55 |
139 /// The default test timeout. | 56 /// The default test timeout. |
| 57 /// |
| 58 /// When [merge]d, this combines with the other configuration's timeout using |
| 59 /// [Timeout.merge]. |
140 final Timeout timeout; | 60 final Timeout timeout; |
141 | 61 |
142 /// Whether to use command-line color escapes. | 62 /// Whether to use command-line color escapes. |
143 final bool color; | 63 bool get color => _color ?? canUseSpecialChars; |
| 64 final bool _color; |
144 | 65 |
145 /// How many tests to run concurrently. | 66 /// How many tests to run concurrently. |
146 final int concurrency; | 67 int get concurrency => |
| 68 pauseAfterLoad ? 1 : (_concurrency ?? defaultConcurrency); |
| 69 final int _concurrency; |
147 | 70 |
148 /// The from which to load tests. | 71 /// The paths from which to load tests. |
149 final List<String> paths; | 72 List<String> get paths => _paths ?? ["test"]; |
| 73 final List<String> _paths; |
150 | 74 |
151 /// Whether the load paths were passed explicitly or the default was used. | 75 /// Whether the load paths were passed explicitly or the default was used. |
152 final bool explicitPaths; | 76 bool get explicitPaths => _paths != null; |
153 | 77 |
154 /// The pattern to match against test names to decide which to run, or `null` | 78 /// The pattern to match against test names to decide which to run, or `null` |
155 /// if all tests should be run. | 79 /// if all tests should be run. |
156 final Pattern pattern; | 80 final Pattern pattern; |
157 | 81 |
158 /// The set of platforms on which to run tests. | 82 /// The set of platforms on which to run tests. |
159 final List<TestPlatform> platforms; | 83 List<TestPlatform> get platforms => _platforms ?? [TestPlatform.vm]; |
| 84 final List<TestPlatform> _platforms; |
160 | 85 |
161 /// Restricts the set of tests to a set of tags | 86 /// Restricts the set of tests to a set of tags. |
| 87 /// |
| 88 /// If this is empty, it applies no restrictions. |
| 89 /// |
| 90 /// When [merge]d, this is unioned with the other configuration's tags. |
162 final Set<String> tags; | 91 final Set<String> tags; |
163 | 92 |
164 /// Does not run tests with tags from this set | 93 /// Does not run tests with tags from this set. |
| 94 /// |
| 95 /// If this is empty, it applies no restrictions. |
| 96 /// |
| 97 /// When [merge]d, this is unioned with the other configuration's excluded |
| 98 /// tags. |
165 final Set<String> excludeTags; | 99 final Set<String> excludeTags; |
166 | 100 |
167 /// The global test metadata derived from this configuration. | 101 /// The global test metadata derived from this configuration. |
168 Metadata get metadata => | 102 Metadata get metadata => |
169 new Metadata(timeout: timeout, verboseTrace: verboseTrace); | 103 new Metadata(timeout: timeout, verboseTrace: verboseTrace); |
170 | 104 |
171 /// Parses the configuration from [args]. | 105 /// Parses the configuration from [args]. |
172 /// | 106 /// |
173 /// Throws a [FormatException] if [args] are invalid. | 107 /// Throws a [FormatException] if [args] are invalid. |
174 factory Configuration.parse(List<String> args) { | 108 factory Configuration.parse(List<String> arguments) => args.parse(arguments); |
175 var options = _parser.parse(args); | |
176 | 109 |
177 var pattern; | 110 /// Loads the configuration from [path]. |
178 if (options['name'] != null) { | 111 /// |
179 if (options["plain-name"] != null) { | 112 /// Throws an [IOException] if [path] does not exist or cannot be read. Throws |
180 throw new FormatException( | 113 /// a [FormatException] if its contents are invalid. |
181 "--name and --plain-name may not both be passed."); | 114 factory Configuration.load(String path) => load(path); |
182 } | |
183 | 115 |
184 pattern = _wrapFormatException( | 116 Configuration({bool help, bool version, bool verboseTrace, bool jsTrace, |
185 options, 'name', (value) => new RegExp(value)); | 117 bool pauseAfterLoad, bool color, String packageRoot, String reporter, |
186 } else if (options['plain-name'] != null) { | 118 int pubServePort, int concurrency, Timeout timeout, this.pattern, |
187 pattern = options['plain-name']; | 119 Iterable<TestPlatform> platforms, Iterable<String> paths, |
188 } | 120 Iterable<String> tags, Iterable<String> excludeTags}) |
189 | 121 : _help = help, |
190 var tags = new Set(); | 122 _version = version, |
191 tags.addAll(options['tags'] ?? []); | 123 _verboseTrace = verboseTrace, |
192 tags.addAll(options['tag'] ?? []); | 124 _jsTrace = jsTrace, |
193 | 125 _pauseAfterLoad = pauseAfterLoad, |
194 var excludeTags = new Set(); | 126 _color = color, |
195 excludeTags.addAll(options['exclude-tags'] ?? []); | 127 _packageRoot = packageRoot, |
196 excludeTags.addAll(options['exclude-tag'] ?? []); | 128 _reporter = reporter, |
197 | |
198 var tagIntersection = tags.intersection(excludeTags); | |
199 if (tagIntersection.isNotEmpty) { | |
200 throw new FormatException( | |
201 'The ${pluralize('tag', tagIntersection.length)} ' | |
202 '${toSentence(tagIntersection)} ' | |
203 '${pluralize('was', tagIntersection.length, plural: 'were')} ' | |
204 'both included and excluded.'); | |
205 } | |
206 | |
207 return new Configuration( | |
208 help: options['help'], | |
209 version: options['version'], | |
210 verboseTrace: options['verbose-trace'], | |
211 jsTrace: options['js-trace'], | |
212 pauseAfterLoad: options['pause-after-load'], | |
213 color: options['color'], | |
214 packageRoot: options['package-root'], | |
215 reporter: options['reporter'], | |
216 pubServePort: _wrapFormatException(options, 'pub-serve', int.parse), | |
217 concurrency: _wrapFormatException(options, 'concurrency', int.parse, | |
218 orElse: () => _defaultConcurrency), | |
219 timeout: _wrapFormatException(options, 'timeout', | |
220 (value) => new Timeout.parse(value), | |
221 orElse: () => new Timeout.factor(1)), | |
222 pattern: pattern, | |
223 platforms: options['platform'].map(TestPlatform.find), | |
224 paths: options.rest.isEmpty ? null : options.rest, | |
225 tags: tags, | |
226 excludeTags: excludeTags); | |
227 } | |
228 | |
229 /// Runs [parse] on the value of the option [name], and wraps any | |
230 /// [FormatException] it throws with additional information. | |
231 static _wrapFormatException(ArgResults options, String name, parse(value), | |
232 {orElse()}) { | |
233 var value = options[name]; | |
234 if (value == null) return orElse == null ? null : orElse(); | |
235 | |
236 try { | |
237 return parse(value); | |
238 } on FormatException catch (error) { | |
239 throw new FormatException('Couldn\'t parse --$name "${options[name]}": ' | |
240 '${error.message}'); | |
241 } | |
242 } | |
243 | |
244 Configuration({this.help: false, this.version: false, | |
245 this.verboseTrace: false, this.jsTrace: false, | |
246 bool pauseAfterLoad: false, bool color, String packageRoot, | |
247 String reporter, int pubServePort, int concurrency, Timeout timeout, | |
248 this.pattern, Iterable<TestPlatform> platforms, | |
249 Iterable<String> paths, Set<String> tags, Set<String> excludeTags}) | |
250 : pauseAfterLoad = pauseAfterLoad, | |
251 color = color == null ? canUseSpecialChars : color, | |
252 packageRoot = packageRoot == null | |
253 ? p.join(p.current, 'packages') | |
254 : packageRoot, | |
255 reporter = reporter == null ? 'compact' : reporter, | |
256 pubServeUrl = pubServePort == null | 129 pubServeUrl = pubServePort == null |
257 ? null | 130 ? null |
258 : Uri.parse("http://localhost:$pubServePort"), | 131 : Uri.parse("http://localhost:$pubServePort"), |
259 concurrency = pauseAfterLoad | 132 _concurrency = concurrency, |
260 ? 1 | 133 timeout = (pauseAfterLoad ?? false) |
261 : (concurrency == null ? _defaultConcurrency : concurrency), | |
262 timeout = pauseAfterLoad | |
263 ? Timeout.none | 134 ? Timeout.none |
264 : (timeout == null ? new Timeout.factor(1) : timeout), | 135 : (timeout == null ? new Timeout.factor(1) : timeout), |
265 platforms = platforms == null ? [TestPlatform.vm] : platforms.toList(), | 136 _platforms = _list(platforms), |
266 paths = paths == null ? ["test"] : paths.toList(), | 137 _paths = _list(paths), |
267 explicitPaths = paths != null, | 138 tags = tags?.toSet() ?? new Set(), |
268 this.tags = tags, | 139 excludeTags = excludeTags?.toSet() ?? new Set(); |
269 this.excludeTags = excludeTags; | 140 |
| 141 /// Returns a [input] as a list or `null`. |
| 142 /// |
| 143 /// If [input] is `null` or empty, this returns `null`. Otherwise, it returns |
| 144 /// `input.toList()`. |
| 145 static List _list(Iterable input) { |
| 146 if (input == null) return null; |
| 147 input = input.toList(); |
| 148 if (input.isEmpty) return null; |
| 149 return input; |
| 150 } |
| 151 |
| 152 /// Merges this with [other]. |
| 153 /// |
| 154 /// For most fields, if both configurations have values set, [other]'s value |
| 155 /// takes precedence. However, certain fields are merged together instead. |
| 156 /// This is indicated in those fields' documentation. |
| 157 Configuration merge(Configuration other) => new Configuration( |
| 158 help: other._help ?? _help, |
| 159 version: other._version ?? _version, |
| 160 verboseTrace: other._verboseTrace ?? _verboseTrace, |
| 161 jsTrace: other._jsTrace ?? _jsTrace, |
| 162 pauseAfterLoad: other._pauseAfterLoad ?? _pauseAfterLoad, |
| 163 color: other._color ?? _color, |
| 164 packageRoot: other._packageRoot ?? _packageRoot, |
| 165 reporter: other._reporter ?? _reporter, |
| 166 pubServePort: (other.pubServeUrl ?? pubServeUrl)?.port, |
| 167 concurrency: other._concurrency ?? _concurrency, |
| 168 timeout: timeout.merge(other.timeout), |
| 169 pattern: other.pattern ?? pattern, |
| 170 platforms: other._platforms ?? _platforms, |
| 171 paths: other._paths ?? _paths, |
| 172 tags: other.tags.union(tags), |
| 173 excludeTags: other.excludeTags.union(excludeTags)); |
270 } | 174 } |
OLD | NEW |