Improved AS benchmarking metrics.

pkg/analysis_server/benchmark/perf/analysis_timing_tests.dart

Index: pkg/analysis_server/benchmark/perf/analysis_timing_tests.dart
diff --git a/pkg/analysis_server/benchmark/perf/analysis_timing_tests.dart b/pkg/analysis_server/benchmark/perf/analysis_timing_tests.dart
index 6f7a3c67277d0be10b1a4b5f7e631595a1e88ed8..a738396333c490cb8ff9fec005636b7a4df24e6e 100644
--- a/pkg/analysis_server/benchmark/perf/analysis_timing_tests.dart
+++ b/pkg/analysis_server/benchmark/perf/analysis_timing_tests.dart
@@ -31,38 +31,25 @@ main(List<String> arguments) {
   }
   source = args[SOURCE_OPTION];
   priorityFile = args[PRIORITY_FILE_OPTION];
-  testName = args[TEST_NAME_OPTION] ?? DEFAULT_TEST;
-
-  switch (testName) {
-    case 'analysis':
-      defineReflectiveTests(AnalysisTimingIntegrationTest);
-      break;
-    case 'highlighting':
-      defineReflectiveTests(HighlightingTimingIntegrationTest);
-      break;
-    case 'navigation':
-      defineReflectiveTests(NavigationTimingIntegrationTest);
-      break;
-    case 'outline':
-      defineReflectiveTests(OutlineTimingIntegrationTest);
-      break;
-    default:
-      print('unrecognized test name $testName');
-      exit(1);
-  }
+  var metricNameParam = args[METRIC_NAME_OPTION] ?? DEFAULT_METRIC;
+
+  metricNames.addAll(metricNameParam);
+
+  defineReflectiveTests(TimingTest);
 }
 
-const DEFAULT_TEST = 'analysis';
+const DEFAULT_METRIC = 'analysis';
+const METRIC_NAME_OPTION = 'metric';
 const PRIORITY_FILE_OPTION = 'priority';
 const SOURCE_OPTION = 'source';
-const TEST_NAME_OPTION = 'test';
 
+final metricNames = <String>[];
 String priorityFile;
 String source;
-String testName;
 
 ArgParser _createArgParser() => new ArgParser()
-  ..addOption(TEST_NAME_OPTION, help: 'test name (defaults to `analysis`)')
+  ..addOption(METRIC_NAME_OPTION,
+      help: 'metric name (defaults to `analysis`)', allowMultiple: true)
   ..addOption(SOURCE_OPTION, help: 'full path to source directory for analysis')
   ..addOption(PRIORITY_FILE_OPTION,
       help: '(optional) full path to a priority file');
@@ -75,81 +62,76 @@ class AbstractTimingTest extends AbstractAnalysisServerPerformanceTest {
       });
 }
 
-@reflectiveTest
-class AnalysisTimingIntegrationTest extends AbstractTimingTest {
-  test_detect_analysis_done() {
-    stopwatch.start();
-    setAnalysisRoot();
-    if (priorityFile != null) {
-      sendAnalysisSetPriorityFiles([priorityFile]);
-    }
-    return analysisFinished.then((_) {
-      print('analysis completed in ${stopwatch.elapsed}');
-      stopwatch.reset();
-    });
-  }
-}
-
-@reflectiveTest
-class HighlightingTimingIntegrationTest extends PriorityFileTimer {
-  @override
-  String get description => 'highlighting';
-
-  @override
-  Stream get eventStream => onAnalysisHighlights;
-
-  @override
-  AnalysisService get service => AnalysisService.HIGHLIGHTS;
+class Metric {
+  List<Duration> timings = <Duration>[];
+  Stream eventStream;
+  AnalysisService service;
+  String name;
+  Metric(this.name, this.service, this.eventStream);
+  String toString() => '$name: $service, ${eventStream.runtimeType}, $timings';
 }
 
 @reflectiveTest
-class NavigationTimingIntegrationTest extends PriorityFileTimer {
-  @override
-  String get description => 'navigation';
-
-  @override
-  Stream get eventStream => onAnalysisNavigation;
-
-  @override
-  AnalysisService get service => AnalysisService.NAVIGATION;
-}
+class TimingTest extends AbstractTimingTest {
+  List<Metric> _metrics;
+
+  List<Metric> get metrics =>
+      _metrics ??= metricNames.map((name) => getMetric(name)).toList();
+
+  Metric getMetric(String name) {
+    switch (name) {
+      case 'folding':
+        return new Metric(name, AnalysisService.FOLDING, onAnalysisFolding);
+      case 'highlighting':
+        return new Metric(
+            name, AnalysisService.HIGHLIGHTS, onAnalysisHighlights);
+      case 'implemented':
+        return new Metric(
+            name, AnalysisService.IMPLEMENTED, onAnalysisImplemented);
+      case 'navigation':
+        return new Metric(
+            name, AnalysisService.NAVIGATION, onAnalysisNavigation);
+      case 'outline':
+        return new Metric(name, AnalysisService.OUTLINE, onAnalysisOutline);
+      case 'occurences':
+        return new Metric(
+            name, AnalysisService.OCCURRENCES, onAnalysisOccurrences);
+      case 'overrides':
+        return new Metric(name, AnalysisService.OVERRIDES, onAnalysisOverrides);
+    }
+    print('no metric found for $name');
+    exit(1);
+    return null; // Won't get here.
+  }
 
-@reflectiveTest
-class OutlineTimingIntegrationTest extends PriorityFileTimer {
-  @override
-  String get description => 'outline';
+  Future test_timing() {
+    //debugStdio();
 
-  @override
-  Stream get eventStream => onAnalysisOutline;
+    expect(metrics, isNotEmpty);
+    expect(priorityFile, isNotNull,
+        reason: 'A priority file must be specified for '
+            '${metrics.first.name} testing.');
 
-  @override
-  AnalysisService get service => AnalysisService.OUTLINE;
-}
+    stopwatch.start();
 
-abstract class PriorityFileTimer extends AbstractTimingTest {
-  String get description;
-  Stream get eventStream;
-  AnalysisService get service;
+    metrics.forEach((Metric m) => m.eventStream.listen((_) {
+          m.timings.add(
+              new Duration(milliseconds: stopwatch.elapsed.inMilliseconds));
+        }));
 
-  Future test_timing() {
-    expect(priorityFile, isNotNull,
-        reason: 'A priority file must be specified for $description testing.');
-    stopwatch.start();
+    var subscriptions = <AnalysisService, List<String>>{};
+    metrics.forEach((Metric m) => subscriptions[m.service] = [priorityFile]);
 
-    Duration elapsed;
-    eventStream.listen((_) {
-      elapsed = stopwatch.elapsed;
-    });
+    sendAnalysisSetSubscriptions(subscriptions);
 
+    // Set root after subscribing to avoid empty notifications.
     setAnalysisRoot();
-    sendAnalysisSetSubscriptions({
-      service: [priorityFile]
-    });
 
     sendAnalysisSetPriorityFiles([priorityFile]);
 
     return analysisFinished.then((_) {
-      print('$description completed in ${elapsed}');
+      print('analysis completed in ${stopwatch.elapsed}');
+      metrics.forEach((Metric m) => print('${m.name} timings: ${m.timings}'));
       stopwatch.reset();
     });
   }