Minor updates to frontend perf tool: warm up & derive .packages directly.

pkg/analyzer_cli/tool/perf.dart

 // Copyright (c) 2016, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 /// An entrypoint used to run portions of analyzer and measure its performance.
 library analyzer_cli.tool.perf;
 
+import 'dart:async';
 import 'dart:io' show exit;
 
 import 'package:analyzer/dart/ast/ast.dart';
 import 'package:analyzer/dart/ast/token.dart';
 import 'package:analyzer/error/listener.dart';
 import 'package:analyzer/file_system/file_system.dart' show ResourceUriResolver;
 import 'package:analyzer/file_system/physical_file_system.dart'
     show PhysicalResourceProvider;
 import 'package:analyzer/source/package_map_resolver.dart';
 import 'package:analyzer/src/context/builder.dart';
 import 'package:analyzer/src/dart/scanner/reader.dart';
 import 'package:analyzer/src/dart/scanner/scanner.dart';
 import 'package:analyzer/src/dart/sdk/sdk.dart' show FolderBasedDartSdk;
 import 'package:analyzer/src/generated/parser.dart';
 import 'package:analyzer/src/generated/source.dart';
 import 'package:analyzer/src/generated/source_io.dart';
 import 'package:package_config/discovery.dart';
 
 /// Cummulative total number of chars scanned.
 int scanTotalChars = 0;
 
 /// Cummulative time spent scanning.
 Stopwatch scanTimer = new Stopwatch();
 
 /// Factory to load and resolve app, packages, and sdk sources.
 SourceFactory sources;
 
-main(args) {
+main(List<String> args) async {
   // TODO(sigmund): provide sdk folder as well.
-  if (args.length < 3) {
-    print('usage: perf.dart <bench-id> <package-root> <entry.dart>');
+  if (args.length < 2) {
+    print('usage: perf.dart <bench-id> <entry.dart>');

[Paul Berry, 2016/10/10 19:20:40]

Consider using the args package so that we can have a more stable command-line
API to this tool.

[Siggi Cherem (dart-lang), 2016/10/10 19:30:02]

Let's see how this evolves, my hope is that we can keep this as 2 inputs
"bench-id" + entry point, but if we need to start adding more options, we can do
that. 

     exit(1);
   }
   var totalTimer = new Stopwatch()..start();
 
   var bench = args[0];
-  var packageRoot = Uri.base.resolve(args[1]);
-  var entryUri = Uri.base.resolve(args[2]);
+  var entryUri = Uri.base.resolve(args[1]);
 
-  setup(packageRoot);
+  await setup(entryUri);
+
   if (bench == 'scan') {
-    scanReachableFiles(entryUri);
+    Set<Source> files = scanReachableFiles(entryUri);
+    // TODO(sigmund): consider replacing the warmup with instrumented snapshots.
+    for (int i = 0; i < 10; i++) scanFiles(files);

[Paul Berry, 2016/10/10 19:20:40]

It looks like the way you're doing the warmup is to run the operation 10x and
take the sum of all of runs.  What I'd recommend doing instead is:  Run the
operation 10x, then start a timer, then run 10x more, then stop the timer.  This
allows the warmup time to be completely excluded from the benchmark, rather than
merely diluted.

[Siggi Cherem (dart-lang), 2016/10/10 19:30:01]

Sorry - it is not obvious to see how I'm using this without the benchmark
harness around this. This script currently prints the time it takes on each
iteration individually, so what I'm doing is running it 10 times, then storing
the time of the 10th iteration alone.

[Paul Berry, 2016/10/10 20:39:38]

Ah, ok.  I didn't realize that.  In which case what you're doing seems fine to
me.

   } else if (bench == 'parse') {
     Set<Source> files = scanReachableFiles(entryUri);
-    parseFiles(files);
+    // TODO(sigmund): consider replacing the warmup with instrumented snapshots.
+    for (int i = 0; i < 10; i++) parseFiles(files);

[Paul Berry, 2016/10/10 19:20:40]

Similar comment here.

[Siggi Cherem (dart-lang), 2016/10/10 19:30:02]

Similar here :)

   } else {
     print('unsupported bench-id: $bench. Please specify "scan" or "parse"');
     // TODO(sigmund): implement the remaining benchmarks.
     exit(1);
   }
 
   totalTimer.stop();
   report("total", totalTimer.elapsedMicroseconds);
 }
 
 /// Sets up analyzer to be able to load and resolve app, packages, and sdk
 /// sources.
-void setup(Uri packageRoot) {
+Future setup(Uri entryUri) async {
   var provider = PhysicalResourceProvider.INSTANCE;
   var packageMap = new ContextBuilder(provider, null, null)
-      .convertPackagesToMap(getPackagesDirectory(packageRoot));
+      .convertPackagesToMap(await findPackages(entryUri));
   sources = new SourceFactory([
     new ResourceUriResolver(provider),
     new PackageMapUriResolver(provider, packageMap),
     new DartUriResolver(
         new FolderBasedDartSdk(provider, provider.getFolder("sdk"))),
   ]);
 }
 
 /// Load and scans all files we need to process: files reachable from the
 /// entrypoint and all core libraries automatically included by the VM.
 Set<Source> scanReachableFiles(Uri entryUri) {
   var files = new Set<Source>();
   var loadTimer = new Stopwatch()..start();
   collectSources(sources.forUri2(entryUri), files);
-  collectSources(sources.forUri("dart:async"), files);
-  collectSources(sources.forUri("dart:collection"), files);
-  collectSources(sources.forUri("dart:convert"), files);
-  collectSources(sources.forUri("dart:core"), files);
-  collectSources(sources.forUri("dart:developer"), files);
-  collectSources(sources.forUri("dart:_internal"), files);
-  collectSources(sources.forUri("dart:isolate"), files);
-  collectSources(sources.forUri("dart:math"), files);
-  collectSources(sources.forUri("dart:mirrors"), files);
-  collectSources(sources.forUri("dart:typed_data"), files);
+
+  var libs = [
+    "dart:async",
+    "dart:collection",
+    "dart:convert",
+    "dart:core",
+    "dart:developer",
+    "dart:_internal",
+    "dart:isolate",
+    "dart:math",
+    "dart:mirrors",
+    "dart:typed_data",
+    "dart:io"
+  ];
+
+  for (var lib in libs) {
+    collectSources(sources.forUri(lib), files);
+  }
+
   loadTimer.stop();
 
   print('input size: ${scanTotalChars} chars');
   var loadTime = loadTimer.elapsedMicroseconds - scanTimer.elapsedMicroseconds;
   report("load", loadTime);
   report("scan", scanTimer.elapsedMicroseconds);
   return files;
 }
 
+/// Scans every file in [files] and reports the time spent doing so.
+void scanFiles(Set<Source> files) {
+  // The code below will record again how many chars are scanned and how long it
+  // takes to scan them, even though we already did so in [scanReachableFiles].
+  // Recording and reporting this twice is unnecessary, but we do so for now to
+  // validate that the results are consistent.
+  scanTimer = new Stopwatch();
+  var old = scanTotalChars;
+  scanTotalChars = 0;
+  for (var source in files) {
+    tokenize(source);
+  }
+
+  // Report size and scanning time again. See discussion above.
+  if (old != scanTotalChars) print('input size changed? ${old} chars');
+  report("scan", scanTimer.elapsedMicroseconds);
+}
+
 /// Parses every file in [files] and reports the time spent doing so.
 void parseFiles(Set<Source> files) {
   // The code below will record again how many chars are scanned and how long it
   // takes to scan them, even though we already did so in [scanReachableFiles].
   // Recording and reporting this twice is unnecessary, but we do so for now to
   // validate that the results are consistent.
   scanTimer = new Stopwatch();
   var old = scanTotalChars;
   scanTotalChars = 0;
   var parseTimer = new Stopwatch()..start();
@@ skipping 51 matching lines @@
 }
 
 /// Report that metric [name] took [time] micro-seconds to process
 /// [scanTotalChars] characters.
 void report(String name, int time) {
   var sb = new StringBuffer();
   sb.write('$name: $time us, ${time ~/ 1000} ms');
   sb.write(', ${scanTotalChars * 1000 ~/ time} chars/ms');
   print('$sb');
 }