OLD | NEW |
1 // Copyright (c) 2016, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2016, the Dart project authors. Please see the AUTHORS file |
2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
4 | 4 |
5 /// An entrypoint used to run portions of front_end and measure its performance. | 5 /// An entrypoint used to run portions of front_end and measure its performance. |
6 library front_end.tool.perf; | 6 library front_end.tool.perf; |
7 | 7 |
8 import 'dart:async'; | 8 import 'dart:async'; |
9 import 'dart:io' show exit, stderr; | 9 import 'dart:io' show exit; |
10 | 10 |
11 import 'package:analyzer/dart/ast/ast.dart'; | 11 import 'package:analyzer/dart/ast/ast.dart'; |
12 import 'package:analyzer/error/listener.dart'; | 12 import 'package:analyzer/error/listener.dart'; |
13 import 'package:analyzer/file_system/file_system.dart' show ResourceUriResolver; | 13 import 'package:analyzer/file_system/file_system.dart' show ResourceUriResolver; |
14 import 'package:analyzer/file_system/physical_file_system.dart' | 14 import 'package:analyzer/file_system/physical_file_system.dart' |
15 show PhysicalResourceProvider; | 15 show PhysicalResourceProvider; |
16 import 'package:analyzer/source/package_map_resolver.dart'; | 16 import 'package:analyzer/source/package_map_resolver.dart'; |
17 import 'package:analyzer/src/context/builder.dart'; | 17 import 'package:analyzer/src/context/builder.dart'; |
18 import 'package:analyzer/src/dart/sdk/sdk.dart' show FolderBasedDartSdk; | 18 import 'package:analyzer/src/dart/sdk/sdk.dart' show FolderBasedDartSdk; |
19 import 'package:analyzer/src/generated/parser.dart'; | 19 import 'package:analyzer/src/generated/parser.dart'; |
20 import 'package:analyzer/src/generated/source.dart'; | 20 import 'package:analyzer/src/generated/source.dart'; |
21 import 'package:analyzer/src/generated/source_io.dart'; | 21 import 'package:analyzer/src/generated/source_io.dart'; |
22 import 'package:analyzer/src/summary/format.dart'; | 22 import 'package:analyzer/src/summary/format.dart'; |
23 import 'package:analyzer/src/summary/idl.dart'; | 23 import 'package:analyzer/src/summary/idl.dart'; |
24 import 'package:analyzer/src/summary/link.dart'; | 24 import 'package:analyzer/src/summary/link.dart'; |
25 import 'package:analyzer/src/summary/summarize_ast.dart'; | 25 import 'package:analyzer/src/summary/summarize_ast.dart'; |
26 import 'package:kernel/analyzer/loader.dart'; | |
27 import 'package:kernel/kernel.dart'; | |
28 import 'package:package_config/discovery.dart'; | |
29 | |
30 import 'package:front_end/compiler_options.dart'; | 26 import 'package:front_end/compiler_options.dart'; |
31 import 'package:front_end/kernel_generator.dart'; | 27 import 'package:front_end/kernel_generator.dart'; |
32 import 'package:front_end/src/scanner/reader.dart'; | 28 import 'package:front_end/src/scanner/reader.dart'; |
33 import 'package:front_end/src/scanner/scanner.dart'; | 29 import 'package:front_end/src/scanner/scanner.dart'; |
34 import 'package:front_end/src/scanner/token.dart'; | 30 import 'package:front_end/src/scanner/token.dart'; |
35 | 31 import 'package:kernel/kernel.dart'; |
36 /// Cumulative total number of chars scanned. | 32 import 'package:package_config/discovery.dart'; |
37 int scanTotalChars = 0; | |
38 | |
39 /// Cumulative time spent scanning. | |
40 Stopwatch scanTimer = new Stopwatch(); | |
41 | |
42 /// Cumulative time spent parsing. | |
43 Stopwatch parseTimer = new Stopwatch(); | |
44 | |
45 /// Cumulative time spent building unlinked summaries. | |
46 Stopwatch unlinkedSummarizeTimer = new Stopwatch(); | |
47 | |
48 /// Cumulative time spent prelinking summaries. | |
49 Stopwatch prelinkSummaryTimer = new Stopwatch(); | |
50 | |
51 /// Factory to load and resolve app, packages, and sdk sources. | |
52 SourceFactory sources; | |
53 | 33 |
54 main(List<String> args) async { | 34 main(List<String> args) async { |
55 // TODO(sigmund): provide sdk folder as well. | 35 // TODO(sigmund): provide sdk folder as well. |
56 if (args.length < 2) { | 36 if (args.length < 2) { |
57 print('usage: perf.dart <bench-id> <entry.dart>'); | 37 print('usage: perf.dart <bench-id> <entry.dart>'); |
58 exit(1); | 38 exit(1); |
59 } | 39 } |
60 var totalTimer = new Stopwatch()..start(); | 40 var totalTimer = new Stopwatch()..start(); |
61 | 41 |
62 var bench = args[0]; | 42 var bench = args[0]; |
(...skipping 58 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
121 print('unsupported bench-id: $bench. Please specify one of the following: ' | 101 print('unsupported bench-id: $bench. Please specify one of the following: ' |
122 '${handlers.keys.join(", ")}'); | 102 '${handlers.keys.join(", ")}'); |
123 exit(1); | 103 exit(1); |
124 } | 104 } |
125 await handler(); | 105 await handler(); |
126 | 106 |
127 totalTimer.stop(); | 107 totalTimer.stop(); |
128 report("total", totalTimer.elapsedMicroseconds); | 108 report("total", totalTimer.elapsedMicroseconds); |
129 } | 109 } |
130 | 110 |
131 /// Sets up analyzer to be able to load and resolve app, packages, and sdk | 111 /// Cumulative time spent parsing. |
132 /// sources. | 112 Stopwatch parseTimer = new Stopwatch(); |
133 Future setup(Uri entryUri) async { | 113 |
134 var provider = PhysicalResourceProvider.INSTANCE; | 114 /// Cumulative time spent prelinking summaries. |
135 var packageMap = new ContextBuilder(provider, null, null) | 115 Stopwatch prelinkSummaryTimer = new Stopwatch(); |
136 .convertPackagesToMap(await findPackages(entryUri)); | 116 |
137 sources = new SourceFactory([ | 117 /// Cumulative time spent scanning. |
138 new ResourceUriResolver(provider), | 118 Stopwatch scanTimer = new Stopwatch(); |
139 new PackageMapUriResolver(provider, packageMap), | 119 |
140 new DartUriResolver( | 120 /// Cumulative total number of chars scanned. |
141 new FolderBasedDartSdk(provider, provider.getFolder("sdk"))), | 121 int scanTotalChars = 0; |
142 ]); | 122 |
| 123 /// Factory to load and resolve app, packages, and sdk sources. |
| 124 SourceFactory sources; |
| 125 |
| 126 /// Cumulative time spent building unlinked summaries. |
| 127 Stopwatch unlinkedSummarizeTimer = new Stopwatch(); |
| 128 |
| 129 /// Add to [files] all sources reachable from [start]. |
| 130 void collectSources(Source start, Set<Source> files) { |
| 131 if (!files.add(start)) return; |
| 132 var unit = parseDirectives(start); |
| 133 for (var directive in unit.directives) { |
| 134 if (directive is UriBasedDirective) { |
| 135 var next = sources.resolveUri(start, directive.uri.stringValue); |
| 136 collectSources(next, files); |
| 137 } |
| 138 } |
143 } | 139 } |
144 | 140 |
145 /// Load and scans all files we need to process: files reachable from the | 141 Future<Program> generateKernel(Uri entryUri, |
146 /// entrypoint and all core libraries automatically included by the VM. | 142 {bool useSdkSummary: false, bool compileSdk: true}) async { |
147 Set<Source> scanReachableFiles(Uri entryUri) { | 143 var dartkTimer = new Stopwatch()..start(); |
148 var files = new Set<Source>(); | 144 // TODO(sigmund): add a constructor with named args to compiler options. |
149 var loadTimer = new Stopwatch()..start(); | 145 var options = new CompilerOptions() |
150 collectSources(sources.forUri2(entryUri), files); | 146 ..strongMode = false |
151 | 147 ..compileSdk = compileSdk |
152 var libs = [ | 148 ..packagesFilePath = '.packages' |
153 "dart:async", | 149 ..onError = ((e) => print('${e.message}')); |
154 "dart:collection", | 150 if (useSdkSummary) { |
155 "dart:convert", | 151 // TODO(sigmund): adjust path based on the benchmark runner architecture. |
156 "dart:core", | 152 // Possibly let the runner make the file available at an architecture |
157 "dart:developer", | 153 // independent location. |
158 "dart:_internal", | 154 options.sdkSummary = 'out/ReleaseX64/dart-sdk/lib/_internal/spec.sum'; |
159 "dart:isolate", | 155 } else { |
160 "dart:math", | 156 options.sdkPath = 'sdk'; |
161 "dart:mirrors", | |
162 "dart:typed_data", | |
163 "dart:io" | |
164 ]; | |
165 | |
166 for (var lib in libs) { | |
167 collectSources(sources.forUri(lib), files); | |
168 } | 157 } |
169 | 158 Program program = await kernelForProgram(entryUri, options); |
170 loadTimer.stop(); | 159 dartkTimer.stop(); |
171 | 160 var suffix = useSdkSummary ? "_sum" : ""; |
172 print('input size: ${scanTotalChars} chars'); | 161 report("kernel_gen_e2e${suffix}", dartkTimer.elapsedMicroseconds); |
173 var loadTime = loadTimer.elapsedMicroseconds - scanTimer.elapsedMicroseconds; | 162 return program; |
174 report("load", loadTime); | |
175 report("scan", scanTimer.elapsedMicroseconds); | |
176 return files; | |
177 } | 163 } |
178 | 164 |
179 /// Scans every file in [files] and reports the time spent doing so. | 165 /// Generates unlinkmed summaries for all files in [files], and returns them in |
180 void scanFiles(Set<Source> files) { | 166 /// an [UnlinkedSummaries] container. |
| 167 UnlinkedSummaries generateUnlinkedSummaries(Set<Source> files) { |
| 168 var unlinkedSummaries = new UnlinkedSummaries(); |
| 169 for (var source in files) { |
| 170 unlinkedSummaries.summariesByUri[source.uri.toString()] = |
| 171 unlinkedSummarize(source); |
| 172 } |
| 173 return unlinkedSummaries; |
| 174 } |
| 175 |
| 176 /// Produces linked summaries for every file in [files] and reports the time |
| 177 /// spent doing so. |
| 178 void linkedSummarizeFiles(Set<Source> files) { |
181 // The code below will record again how many chars are scanned and how long it | 179 // The code below will record again how many chars are scanned and how long it |
182 // takes to scan them, even though we already did so in [scanReachableFiles]. | 180 // takes to scan them, even though we already did so in [scanReachableFiles]. |
183 // Recording and reporting this twice is unnecessary, but we do so for now to | 181 // Recording and reporting this twice is unnecessary, but we do so for now to |
184 // validate that the results are consistent. | |
185 scanTimer = new Stopwatch(); | |
186 var old = scanTotalChars; | |
187 scanTotalChars = 0; | |
188 for (var source in files) { | |
189 tokenize(source); | |
190 } | |
191 | |
192 // Report size and scanning time again. See discussion above. | |
193 if (old != scanTotalChars) print('input size changed? ${old} chars'); | |
194 report("scan", scanTimer.elapsedMicroseconds); | |
195 } | |
196 | |
197 /// Parses every file in [files] and reports the time spent doing so. | |
198 void parseFiles(Set<Source> files) { | |
199 // The code below will record again how many chars are scanned and how long it | |
200 // takes to scan them, even though we already did so in [scanReachableFiles]. | |
201 // Recording and reporting this twice is unnecessary, but we do so for now to | |
202 // validate that the results are consistent. | |
203 scanTimer = new Stopwatch(); | |
204 var old = scanTotalChars; | |
205 scanTotalChars = 0; | |
206 parseTimer = new Stopwatch(); | |
207 for (var source in files) { | |
208 parseFull(source); | |
209 } | |
210 | |
211 // Report size and scanning time again. See discussion above. | |
212 if (old != scanTotalChars) print('input size changed? ${old} chars'); | |
213 report("scan", scanTimer.elapsedMicroseconds); | |
214 report("parse", parseTimer.elapsedMicroseconds); | |
215 } | |
216 | |
217 /// Produces unlinked summaries for every file in [files] and reports the time | |
218 /// spent doing so. | |
219 void unlinkedSummarizeFiles(Set<Source> files) { | |
220 // The code below will record again how many chars are scanned and how long it | |
221 // takes to scan them, even though we already did so in [scanReachableFiles]. | |
222 // Recording and reporting this twice is unnecessary, but we do so for now to | |
223 // validate that the results are consistent. | 182 // validate that the results are consistent. |
224 scanTimer = new Stopwatch(); | 183 scanTimer = new Stopwatch(); |
225 var old = scanTotalChars; | 184 var old = scanTotalChars; |
226 scanTotalChars = 0; | 185 scanTotalChars = 0; |
227 parseTimer = new Stopwatch(); | 186 parseTimer = new Stopwatch(); |
228 unlinkedSummarizeTimer = new Stopwatch(); | 187 unlinkedSummarizeTimer = new Stopwatch(); |
229 generateUnlinkedSummaries(files); | 188 var unlinkedSummaries = generateUnlinkedSummaries(files); |
| 189 prelinkSummaryTimer = new Stopwatch(); |
| 190 Map<String, LinkedLibraryBuilder> prelinkedLibraries = |
| 191 prelinkSummaries(files, unlinkedSummaries); |
| 192 var linkTimer = new Stopwatch()..start(); |
| 193 LinkedLibrary getDependency(String uri) { |
| 194 // getDependency should never be called because all dependencies are present |
| 195 // in [prelinkedLibraries]. |
| 196 print('Warning: getDependency called for: $uri'); |
| 197 return null; |
| 198 } |
| 199 |
| 200 bool strong = true; |
| 201 relink(prelinkedLibraries, getDependency, unlinkedSummaries.getUnit, strong); |
| 202 linkTimer.stop(); |
230 | 203 |
231 if (old != scanTotalChars) print('input size changed? ${old} chars'); | 204 if (old != scanTotalChars) print('input size changed? ${old} chars'); |
232 report("scan", scanTimer.elapsedMicroseconds); | 205 report("scan", scanTimer.elapsedMicroseconds); |
233 report("parse", parseTimer.elapsedMicroseconds); | 206 report("parse", parseTimer.elapsedMicroseconds); |
234 report('unlinked summarize', unlinkedSummarizeTimer.elapsedMicroseconds); | 207 report('unlinked summarize', unlinkedSummarizeTimer.elapsedMicroseconds); |
235 report( | 208 report( |
236 'unlinked summarize + parse', | 209 'unlinked summarize + parse', |
237 unlinkedSummarizeTimer.elapsedMicroseconds + | 210 unlinkedSummarizeTimer.elapsedMicroseconds + |
238 parseTimer.elapsedMicroseconds); | 211 parseTimer.elapsedMicroseconds); |
| 212 report('prelink', prelinkSummaryTimer.elapsedMicroseconds); |
| 213 report('link', linkTimer.elapsedMicroseconds); |
239 } | 214 } |
240 | 215 |
241 /// Simple container for a mapping from URI string to an unlinked summary. | 216 /// Uses the diet-parser to parse only directives in [source]. |
242 class UnlinkedSummaries { | 217 CompilationUnit parseDirectives(Source source) { |
243 final summariesByUri = <String, UnlinkedUnit>{}; | 218 var token = tokenize(source); |
244 | 219 var parser = new Parser(source, AnalysisErrorListener.NULL_LISTENER); |
245 /// Get the unlinked summary for the given URI, and report a warning if it | 220 return parser.parseDirectives(token); |
246 /// can't be found. | |
247 UnlinkedUnit getUnit(String uri) { | |
248 var result = summariesByUri[uri]; | |
249 if (result == null) { | |
250 print('Warning: no summary found for: $uri'); | |
251 } | |
252 return result; | |
253 } | |
254 } | 221 } |
255 | 222 |
256 /// Generates unlinkmed summaries for all files in [files], and returns them in | 223 /// Parses every file in [files] and reports the time spent doing so. |
257 /// an [UnlinkedSummaries] container. | 224 void parseFiles(Set<Source> files) { |
258 UnlinkedSummaries generateUnlinkedSummaries(Set<Source> files) { | |
259 var unlinkedSummaries = new UnlinkedSummaries(); | |
260 for (var source in files) { | |
261 unlinkedSummaries.summariesByUri[source.uri.toString()] = | |
262 unlinkedSummarize(source); | |
263 } | |
264 return unlinkedSummaries; | |
265 } | |
266 | |
267 /// Produces prelinked summaries for every file in [files] and reports the time | |
268 /// spent doing so. | |
269 void prelinkedSummarizeFiles(Set<Source> files) { | |
270 // The code below will record again how many chars are scanned and how long it | 225 // The code below will record again how many chars are scanned and how long it |
271 // takes to scan them, even though we already did so in [scanReachableFiles]. | 226 // takes to scan them, even though we already did so in [scanReachableFiles]. |
272 // Recording and reporting this twice is unnecessary, but we do so for now to | 227 // Recording and reporting this twice is unnecessary, but we do so for now to |
273 // validate that the results are consistent. | 228 // validate that the results are consistent. |
274 scanTimer = new Stopwatch(); | 229 scanTimer = new Stopwatch(); |
275 var old = scanTotalChars; | 230 var old = scanTotalChars; |
276 scanTotalChars = 0; | 231 scanTotalChars = 0; |
277 parseTimer = new Stopwatch(); | 232 parseTimer = new Stopwatch(); |
278 unlinkedSummarizeTimer = new Stopwatch(); | 233 for (var source in files) { |
279 var unlinkedSummaries = generateUnlinkedSummaries(files); | 234 parseFull(source); |
280 prelinkSummaryTimer = new Stopwatch(); | 235 } |
281 prelinkSummaries(files, unlinkedSummaries); | |
282 | 236 |
| 237 // Report size and scanning time again. See discussion above. |
283 if (old != scanTotalChars) print('input size changed? ${old} chars'); | 238 if (old != scanTotalChars) print('input size changed? ${old} chars'); |
284 report("scan", scanTimer.elapsedMicroseconds); | 239 report("scan", scanTimer.elapsedMicroseconds); |
285 report("parse", parseTimer.elapsedMicroseconds); | 240 report("parse", parseTimer.elapsedMicroseconds); |
286 report('unlinked summarize', unlinkedSummarizeTimer.elapsedMicroseconds); | |
287 report( | |
288 'unlinked summarize + parse', | |
289 unlinkedSummarizeTimer.elapsedMicroseconds + | |
290 parseTimer.elapsedMicroseconds); | |
291 report('prelink', prelinkSummaryTimer.elapsedMicroseconds); | |
292 } | 241 } |
293 | 242 |
294 /// Produces linked summaries for every file in [files] and reports the time | 243 /// Parse the full body of [source] and return it's compilation unit. |
| 244 CompilationUnit parseFull(Source source) { |
| 245 var token = tokenize(source); |
| 246 parseTimer.start(); |
| 247 var parser = new Parser(source, AnalysisErrorListener.NULL_LISTENER); |
| 248 var unit = parser.parseCompilationUnit(token); |
| 249 parseTimer.stop(); |
| 250 return unit; |
| 251 } |
| 252 |
| 253 /// Produces prelinked summaries for every file in [files] and reports the time |
295 /// spent doing so. | 254 /// spent doing so. |
296 void linkedSummarizeFiles(Set<Source> files) { | 255 void prelinkedSummarizeFiles(Set<Source> files) { |
297 // The code below will record again how many chars are scanned and how long it | 256 // The code below will record again how many chars are scanned and how long it |
298 // takes to scan them, even though we already did so in [scanReachableFiles]. | 257 // takes to scan them, even though we already did so in [scanReachableFiles]. |
299 // Recording and reporting this twice is unnecessary, but we do so for now to | 258 // Recording and reporting this twice is unnecessary, but we do so for now to |
300 // validate that the results are consistent. | 259 // validate that the results are consistent. |
301 scanTimer = new Stopwatch(); | 260 scanTimer = new Stopwatch(); |
302 var old = scanTotalChars; | 261 var old = scanTotalChars; |
303 scanTotalChars = 0; | 262 scanTotalChars = 0; |
304 parseTimer = new Stopwatch(); | 263 parseTimer = new Stopwatch(); |
305 unlinkedSummarizeTimer = new Stopwatch(); | 264 unlinkedSummarizeTimer = new Stopwatch(); |
306 var unlinkedSummaries = generateUnlinkedSummaries(files); | 265 var unlinkedSummaries = generateUnlinkedSummaries(files); |
307 prelinkSummaryTimer = new Stopwatch(); | 266 prelinkSummaryTimer = new Stopwatch(); |
308 Map<String, LinkedLibraryBuilder> prelinkedLibraries = | 267 prelinkSummaries(files, unlinkedSummaries); |
309 prelinkSummaries(files, unlinkedSummaries); | |
310 var linkTimer = new Stopwatch()..start(); | |
311 LinkedLibrary getDependency(String uri) { | |
312 // getDependency should never be called because all dependencies are present | |
313 // in [prelinkedLibraries]. | |
314 print('Warning: getDependency called for: $uri'); | |
315 return null; | |
316 } | |
317 | |
318 bool strong = true; | |
319 relink(prelinkedLibraries, getDependency, unlinkedSummaries.getUnit, strong); | |
320 linkTimer.stop(); | |
321 | 268 |
322 if (old != scanTotalChars) print('input size changed? ${old} chars'); | 269 if (old != scanTotalChars) print('input size changed? ${old} chars'); |
323 report("scan", scanTimer.elapsedMicroseconds); | 270 report("scan", scanTimer.elapsedMicroseconds); |
324 report("parse", parseTimer.elapsedMicroseconds); | 271 report("parse", parseTimer.elapsedMicroseconds); |
325 report('unlinked summarize', unlinkedSummarizeTimer.elapsedMicroseconds); | 272 report('unlinked summarize', unlinkedSummarizeTimer.elapsedMicroseconds); |
326 report( | 273 report( |
327 'unlinked summarize + parse', | 274 'unlinked summarize + parse', |
328 unlinkedSummarizeTimer.elapsedMicroseconds + | 275 unlinkedSummarizeTimer.elapsedMicroseconds + |
329 parseTimer.elapsedMicroseconds); | 276 parseTimer.elapsedMicroseconds); |
330 report('prelink', prelinkSummaryTimer.elapsedMicroseconds); | 277 report('prelink', prelinkSummaryTimer.elapsedMicroseconds); |
331 report('link', linkTimer.elapsedMicroseconds); | |
332 } | 278 } |
333 | 279 |
334 /// Prelinks all the summaries for [files], using [unlinkedSummaries] to obtain | 280 /// Prelinks all the summaries for [files], using [unlinkedSummaries] to obtain |
335 /// their unlinked summaries. | 281 /// their unlinked summaries. |
336 /// | 282 /// |
337 /// The return value is suitable for passing to the summary linker. | 283 /// The return value is suitable for passing to the summary linker. |
338 Map<String, LinkedLibraryBuilder> prelinkSummaries( | 284 Map<String, LinkedLibraryBuilder> prelinkSummaries( |
339 Set<Source> files, UnlinkedSummaries unlinkedSummaries) { | 285 Set<Source> files, UnlinkedSummaries unlinkedSummaries) { |
340 prelinkSummaryTimer.start(); | 286 prelinkSummaryTimer.start(); |
341 Set<String> libraryUris = | 287 Set<String> libraryUris = |
342 files.map((source) => source.uri.toString()).toSet(); | 288 files.map((source) => source.uri.toString()).toSet(); |
343 | 289 |
344 String getDeclaredVariable(String s) => null; | 290 String getDeclaredVariable(String s) => null; |
345 var prelinkedLibraries = | 291 var prelinkedLibraries = |
346 setupForLink(libraryUris, unlinkedSummaries.getUnit, getDeclaredVariable); | 292 setupForLink(libraryUris, unlinkedSummaries.getUnit, getDeclaredVariable); |
347 prelinkSummaryTimer.stop(); | 293 prelinkSummaryTimer.stop(); |
348 return prelinkedLibraries; | 294 return prelinkedLibraries; |
349 } | 295 } |
350 | 296 |
351 /// Add to [files] all sources reachable from [start]. | 297 /// Report that metric [name] took [time] micro-seconds to process |
352 void collectSources(Source start, Set<Source> files) { | 298 /// [scanTotalChars] characters. |
353 if (!files.add(start)) return; | 299 void report(String name, int time) { |
354 var unit = parseDirectives(start); | 300 var sb = new StringBuffer(); |
355 for (var directive in unit.directives) { | 301 sb.write('$name: $time us, ${time ~/ 1000} ms'); |
356 if (directive is UriBasedDirective) { | 302 sb.write(', ${scanTotalChars * 1000 ~/ time} chars/ms'); |
357 var next = sources.resolveUri(start, directive.uri.stringValue); | 303 print('$sb'); |
358 collectSources(next, files); | |
359 } | |
360 } | |
361 } | 304 } |
362 | 305 |
363 /// Uses the diet-parser to parse only directives in [source]. | 306 /// Scans every file in [files] and reports the time spent doing so. |
364 CompilationUnit parseDirectives(Source source) { | 307 void scanFiles(Set<Source> files) { |
365 var token = tokenize(source); | 308 // The code below will record again how many chars are scanned and how long it |
366 var parser = new Parser(source, AnalysisErrorListener.NULL_LISTENER); | 309 // takes to scan them, even though we already did so in [scanReachableFiles]. |
367 return parser.parseDirectives(token); | 310 // Recording and reporting this twice is unnecessary, but we do so for now to |
| 311 // validate that the results are consistent. |
| 312 scanTimer = new Stopwatch(); |
| 313 var old = scanTotalChars; |
| 314 scanTotalChars = 0; |
| 315 for (var source in files) { |
| 316 tokenize(source); |
| 317 } |
| 318 |
| 319 // Report size and scanning time again. See discussion above. |
| 320 if (old != scanTotalChars) print('input size changed? ${old} chars'); |
| 321 report("scan", scanTimer.elapsedMicroseconds); |
368 } | 322 } |
369 | 323 |
370 /// Parse the full body of [source] and return it's compilation unit. | 324 /// Load and scans all files we need to process: files reachable from the |
371 CompilationUnit parseFull(Source source) { | 325 /// entrypoint and all core libraries automatically included by the VM. |
372 var token = tokenize(source); | 326 Set<Source> scanReachableFiles(Uri entryUri) { |
373 parseTimer.start(); | 327 var files = new Set<Source>(); |
374 var parser = new Parser(source, AnalysisErrorListener.NULL_LISTENER); | 328 var loadTimer = new Stopwatch()..start(); |
375 var unit = parser.parseCompilationUnit(token); | 329 collectSources(sources.forUri2(entryUri), files); |
376 parseTimer.stop(); | 330 |
377 return unit; | 331 var libs = [ |
| 332 "dart:async", |
| 333 "dart:collection", |
| 334 "dart:convert", |
| 335 "dart:core", |
| 336 "dart:developer", |
| 337 "dart:_internal", |
| 338 "dart:isolate", |
| 339 "dart:math", |
| 340 "dart:mirrors", |
| 341 "dart:typed_data", |
| 342 "dart:io" |
| 343 ]; |
| 344 |
| 345 for (var lib in libs) { |
| 346 collectSources(sources.forUri(lib), files); |
| 347 } |
| 348 |
| 349 loadTimer.stop(); |
| 350 |
| 351 print('input size: ${scanTotalChars} chars'); |
| 352 var loadTime = loadTimer.elapsedMicroseconds - scanTimer.elapsedMicroseconds; |
| 353 report("load", loadTime); |
| 354 report("scan", scanTimer.elapsedMicroseconds); |
| 355 return files; |
378 } | 356 } |
379 | 357 |
380 UnlinkedUnitBuilder unlinkedSummarize(Source source) { | 358 /// Sets up analyzer to be able to load and resolve app, packages, and sdk |
381 var unit = parseFull(source); | 359 /// sources. |
382 unlinkedSummarizeTimer.start(); | 360 Future setup(Uri entryUri) async { |
383 var unlinkedUnit = serializeAstUnlinked(unit); | 361 var provider = PhysicalResourceProvider.INSTANCE; |
384 unlinkedSummarizeTimer.stop(); | 362 var packageMap = new ContextBuilder(provider, null, null) |
385 return unlinkedUnit; | 363 .convertPackagesToMap(await findPackages(entryUri)); |
| 364 sources = new SourceFactory([ |
| 365 new ResourceUriResolver(provider), |
| 366 new PackageMapUriResolver(provider, packageMap), |
| 367 new DartUriResolver( |
| 368 new FolderBasedDartSdk(provider, provider.getFolder("sdk"))), |
| 369 ]); |
386 } | 370 } |
387 | 371 |
388 /// Scan [source] and return the first token produced by the scanner. | 372 /// Scan [source] and return the first token produced by the scanner. |
389 Token tokenize(Source source) { | 373 Token tokenize(Source source) { |
390 scanTimer.start(); | 374 scanTimer.start(); |
391 var contents = source.contents.data; | 375 var contents = source.contents.data; |
392 scanTotalChars += contents.length; | 376 scanTotalChars += contents.length; |
393 // TODO(sigmund): is there a way to scan from a random-access-file without | 377 // TODO(sigmund): is there a way to scan from a random-access-file without |
394 // first converting to String? | 378 // first converting to String? |
395 var scanner = new _Scanner(contents); | 379 var scanner = new _Scanner(contents); |
396 var token = scanner.tokenize(); | 380 var token = scanner.tokenize(); |
397 scanTimer.stop(); | 381 scanTimer.stop(); |
398 return token; | 382 return token; |
399 } | 383 } |
400 | 384 |
| 385 UnlinkedUnitBuilder unlinkedSummarize(Source source) { |
| 386 var unit = parseFull(source); |
| 387 unlinkedSummarizeTimer.start(); |
| 388 var unlinkedUnit = serializeAstUnlinked(unit); |
| 389 unlinkedSummarizeTimer.stop(); |
| 390 return unlinkedUnit; |
| 391 } |
| 392 |
| 393 /// Produces unlinked summaries for every file in [files] and reports the time |
| 394 /// spent doing so. |
| 395 void unlinkedSummarizeFiles(Set<Source> files) { |
| 396 // The code below will record again how many chars are scanned and how long it |
| 397 // takes to scan them, even though we already did so in [scanReachableFiles]. |
| 398 // Recording and reporting this twice is unnecessary, but we do so for now to |
| 399 // validate that the results are consistent. |
| 400 scanTimer = new Stopwatch(); |
| 401 var old = scanTotalChars; |
| 402 scanTotalChars = 0; |
| 403 parseTimer = new Stopwatch(); |
| 404 unlinkedSummarizeTimer = new Stopwatch(); |
| 405 generateUnlinkedSummaries(files); |
| 406 |
| 407 if (old != scanTotalChars) print('input size changed? ${old} chars'); |
| 408 report("scan", scanTimer.elapsedMicroseconds); |
| 409 report("parse", parseTimer.elapsedMicroseconds); |
| 410 report('unlinked summarize', unlinkedSummarizeTimer.elapsedMicroseconds); |
| 411 report( |
| 412 'unlinked summarize + parse', |
| 413 unlinkedSummarizeTimer.elapsedMicroseconds + |
| 414 parseTimer.elapsedMicroseconds); |
| 415 } |
| 416 |
| 417 /// Simple container for a mapping from URI string to an unlinked summary. |
| 418 class UnlinkedSummaries { |
| 419 final summariesByUri = <String, UnlinkedUnit>{}; |
| 420 |
| 421 /// Get the unlinked summary for the given URI, and report a warning if it |
| 422 /// can't be found. |
| 423 UnlinkedUnit getUnit(String uri) { |
| 424 var result = summariesByUri[uri]; |
| 425 if (result == null) { |
| 426 print('Warning: no summary found for: $uri'); |
| 427 } |
| 428 return result; |
| 429 } |
| 430 } |
| 431 |
401 class _Scanner extends Scanner { | 432 class _Scanner extends Scanner { |
402 _Scanner(String contents) : super(new CharSequenceReader(contents)) { | 433 _Scanner(String contents) : super(new CharSequenceReader(contents)) { |
403 preserveComments = false; | 434 preserveComments = false; |
404 } | 435 } |
405 | 436 |
406 @override | 437 @override |
407 void reportError(errorCode, int offset, List<Object> arguments) { | 438 void reportError(errorCode, int offset, List<Object> arguments) { |
408 // ignore errors. | 439 // ignore errors. |
409 } | 440 } |
410 } | 441 } |
411 | |
412 /// Report that metric [name] took [time] micro-seconds to process | |
413 /// [scanTotalChars] characters. | |
414 void report(String name, int time) { | |
415 var sb = new StringBuffer(); | |
416 sb.write('$name: $time us, ${time ~/ 1000} ms'); | |
417 sb.write(', ${scanTotalChars * 1000 ~/ time} chars/ms'); | |
418 print('$sb'); | |
419 } | |
420 | |
421 Future<Program> generateKernel(Uri entryUri, | |
422 {bool useSdkSummary: false, bool compileSdk: true}) async { | |
423 var dartkTimer = new Stopwatch()..start(); | |
424 // TODO(sigmund): add a constructor with named args to compiler options. | |
425 var options = new CompilerOptions() | |
426 ..strongMode = false | |
427 ..compileSdk = compileSdk | |
428 ..packagesFilePath = '.packages' | |
429 ..onError = ((e) => print('${e.message}')); | |
430 if (useSdkSummary) { | |
431 // TODO(sigmund): adjust path based on the benchmark runner architecture. | |
432 // Possibly let the runner make the file available at an architecture | |
433 // independent location. | |
434 options.sdkSummary = 'out/ReleaseX64/dart-sdk/lib/_internal/spec.sum'; | |
435 } else { | |
436 options.sdkPath = 'sdk'; | |
437 } | |
438 Program program = await kernelForProgram(entryUri, options); | |
439 dartkTimer.stop(); | |
440 var suffix = useSdkSummary ? "_sum" : ""; | |
441 report("kernel_gen_e2e${suffix}", dartkTimer.elapsedMicroseconds); | |
442 return program; | |
443 } | |
OLD | NEW |