Minor updates to frontend perf tool: warm up & derive .packages directly.

pkg/analyzer_cli/tool/perf.dart

Index: pkg/analyzer_cli/tool/perf.dart
diff --git a/pkg/analyzer_cli/tool/perf.dart b/pkg/analyzer_cli/tool/perf.dart
index 3c97e23e4f4288d8612d3cc01a65a1013f08501a..845beefb7c7b5a0b3371a950bc51b33c938b3977 100644
--- a/pkg/analyzer_cli/tool/perf.dart
+++ b/pkg/analyzer_cli/tool/perf.dart
@@ -5,6 +5,7 @@
 /// An entrypoint used to run portions of analyzer and measure its performance.
 library analyzer_cli.tool.perf;
 
+import 'dart:async';
 import 'dart:io' show exit;
 
 import 'package:analyzer/dart/ast/ast.dart';
@@ -32,24 +33,27 @@ Stopwatch scanTimer = new Stopwatch();
 /// Factory to load and resolve app, packages, and sdk sources.
 SourceFactory sources;
 
-main(args) {
+main(List<String> args) async {
   // TODO(sigmund): provide sdk folder as well.
-  if (args.length < 3) {
-    print('usage: perf.dart <bench-id> <package-root> <entry.dart>');
+  if (args.length < 2) {
+    print('usage: perf.dart <bench-id> <entry.dart>');

[Paul Berry, 2016/10/10 19:20:40]

Consider using the args package so that we can have a more stable command-line
API to this tool.

[Siggi Cherem (dart-lang), 2016/10/10 19:30:02]

Let's see how this evolves, my hope is that we can keep this as 2 inputs
"bench-id" + entry point, but if we need to start adding more options, we can do
that. 

     exit(1);
   }
   var totalTimer = new Stopwatch()..start();
 
   var bench = args[0];
-  var packageRoot = Uri.base.resolve(args[1]);
-  var entryUri = Uri.base.resolve(args[2]);
+  var entryUri = Uri.base.resolve(args[1]);
+
+  await setup(entryUri);
 
-  setup(packageRoot);
   if (bench == 'scan') {
-    scanReachableFiles(entryUri);
+    Set<Source> files = scanReachableFiles(entryUri);
+    // TODO(sigmund): consider replacing the warmup with instrumented snapshots.
+    for (int i = 0; i < 10; i++) scanFiles(files);

[Paul Berry, 2016/10/10 19:20:40]

It looks like the way you're doing the warmup is to run the operation 10x and
take the sum of all of runs.  What I'd recommend doing instead is:  Run the
operation 10x, then start a timer, then run 10x more, then stop the timer.  This
allows the warmup time to be completely excluded from the benchmark, rather than
merely diluted.

[Siggi Cherem (dart-lang), 2016/10/10 19:30:01]

Sorry - it is not obvious to see how I'm using this without the benchmark
harness around this. This script currently prints the time it takes on each
iteration individually, so what I'm doing is running it 10 times, then storing
the time of the 10th iteration alone.

[Paul Berry, 2016/10/10 20:39:38]

Ah, ok.  I didn't realize that.  In which case what you're doing seems fine to
me.

   } else if (bench == 'parse') {
     Set<Source> files = scanReachableFiles(entryUri);
-    parseFiles(files);
+    // TODO(sigmund): consider replacing the warmup with instrumented snapshots.
+    for (int i = 0; i < 10; i++) parseFiles(files);

[Paul Berry, 2016/10/10 19:20:40]

Similar comment here.

[Siggi Cherem (dart-lang), 2016/10/10 19:30:02]

Similar here :)

   } else {
     print('unsupported bench-id: $bench. Please specify "scan" or "parse"');
     // TODO(sigmund): implement the remaining benchmarks.
@@ -62,10 +66,10 @@ main(args) {
 
 /// Sets up analyzer to be able to load and resolve app, packages, and sdk
 /// sources.
-void setup(Uri packageRoot) {
+Future setup(Uri entryUri) async {
   var provider = PhysicalResourceProvider.INSTANCE;
   var packageMap = new ContextBuilder(provider, null, null)
-      .convertPackagesToMap(getPackagesDirectory(packageRoot));
+      .convertPackagesToMap(await findPackages(entryUri));
   sources = new SourceFactory([
     new ResourceUriResolver(provider),
     new PackageMapUriResolver(provider, packageMap),
@@ -80,16 +84,25 @@ Set<Source> scanReachableFiles(Uri entryUri) {
   var files = new Set<Source>();
   var loadTimer = new Stopwatch()..start();
   collectSources(sources.forUri2(entryUri), files);
-  collectSources(sources.forUri("dart:async"), files);
-  collectSources(sources.forUri("dart:collection"), files);
-  collectSources(sources.forUri("dart:convert"), files);
-  collectSources(sources.forUri("dart:core"), files);
-  collectSources(sources.forUri("dart:developer"), files);
-  collectSources(sources.forUri("dart:_internal"), files);
-  collectSources(sources.forUri("dart:isolate"), files);
-  collectSources(sources.forUri("dart:math"), files);
-  collectSources(sources.forUri("dart:mirrors"), files);
-  collectSources(sources.forUri("dart:typed_data"), files);
+
+  var libs = [
+    "dart:async",
+    "dart:collection",
+    "dart:convert",
+    "dart:core",
+    "dart:developer",
+    "dart:_internal",
+    "dart:isolate",
+    "dart:math",
+    "dart:mirrors",
+    "dart:typed_data",
+    "dart:io"
+  ];
+
+  for (var lib in libs) {
+    collectSources(sources.forUri(lib), files);
+  }
+
   loadTimer.stop();
 
   print('input size: ${scanTotalChars} chars');
@@ -99,6 +112,24 @@ Set<Source> scanReachableFiles(Uri entryUri) {
   return files;
 }
 
+/// Scans every file in [files] and reports the time spent doing so.
+void scanFiles(Set<Source> files) {
+  // The code below will record again how many chars are scanned and how long it
+  // takes to scan them, even though we already did so in [scanReachableFiles].
+  // Recording and reporting this twice is unnecessary, but we do so for now to
+  // validate that the results are consistent.
+  scanTimer = new Stopwatch();
+  var old = scanTotalChars;
+  scanTotalChars = 0;
+  for (var source in files) {
+    tokenize(source);
+  }
+
+  // Report size and scanning time again. See discussion above.
+  if (old != scanTotalChars) print('input size changed? ${old} chars');
+  report("scan", scanTimer.elapsedMicroseconds);
+}
+
 /// Parses every file in [files] and reports the time spent doing so.
 void parseFiles(Set<Source> files) {
   // The code below will record again how many chars are scanned and how long it