Improved AS benchmarking metrics.

pkg/analysis_server/benchmark/perf/analysis_timing_tests.dart

 // Copyright (c) 2015, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 library server.performance.analysis.timing;
 
 import 'dart:async';
 import 'dart:io';
 
 import 'package:analysis_server/src/protocol.dart';
@@ skipping 13 matching lines @@
 main(List<String> arguments) {
   initializeTestEnvironment();
   ArgParser parser = _createArgParser();
   var args = parser.parse(arguments);
   if (args[SOURCE_OPTION] == null) {
     print('path to source directory must be specified');
     exit(1);
   }
   source = args[SOURCE_OPTION];
   priorityFile = args[PRIORITY_FILE_OPTION];
-  testName = args[TEST_NAME_OPTION] ?? DEFAULT_TEST;
+  var metricNameParam = args[METRIC_NAME_OPTION] ?? DEFAULT_METRIC;
 
-  switch (testName) {
-    case 'analysis':
-      defineReflectiveTests(AnalysisTimingIntegrationTest);
-      break;
-    case 'highlighting':
-      defineReflectiveTests(HighlightingTimingIntegrationTest);
-      break;
-    case 'navigation':
-      defineReflectiveTests(NavigationTimingIntegrationTest);
-      break;
-    case 'outline':
-      defineReflectiveTests(OutlineTimingIntegrationTest);
-      break;
-    default:
-      print('unrecognized test name $testName');
-      exit(1);
-  }
+  metricNames.addAll(metricNameParam);
+
+  defineReflectiveTests(TimingTest);
 }
 
-const DEFAULT_TEST = 'analysis';
+const DEFAULT_METRIC = 'analysis';
+const METRIC_NAME_OPTION = 'metric';
 const PRIORITY_FILE_OPTION = 'priority';
 const SOURCE_OPTION = 'source';
-const TEST_NAME_OPTION = 'test';
 
+final metricNames = <String>[];
 String priorityFile;
 String source;
-String testName;
 
 ArgParser _createArgParser() => new ArgParser()
-  ..addOption(TEST_NAME_OPTION, help: 'test name (defaults to `analysis`)')
+  ..addOption(METRIC_NAME_OPTION,
+      help: 'metric name (defaults to `analysis`)', allowMultiple: true)
   ..addOption(SOURCE_OPTION, help: 'full path to source directory for analysis')
   ..addOption(PRIORITY_FILE_OPTION,
       help: '(optional) full path to a priority file');
 
 class AbstractTimingTest extends AbstractAnalysisServerPerformanceTest {
   @override
   Future setUp() => super.setUp().then((_) {
         sourceDirectory = new Directory(source);
         subscribeToStatusNotifications();
       });
 }
 
-@reflectiveTest
-class AnalysisTimingIntegrationTest extends AbstractTimingTest {
-  test_detect_analysis_done() {
-    stopwatch.start();
-    setAnalysisRoot();
-    if (priorityFile != null) {
-      sendAnalysisSetPriorityFiles([priorityFile]);
-    }
-    return analysisFinished.then((_) {
-      print('analysis completed in ${stopwatch.elapsed}');
-      stopwatch.reset();
-    });
-  }
+class Metric {
+  List<Duration> timings = <Duration>[];
+  Stream eventStream;
+  AnalysisService service;
+  String name;
+  Metric(this.name, this.service, this.eventStream);
+  String toString() => '$name: $service, ${eventStream.runtimeType}, $timings';
 }
 
 @reflectiveTest
-class HighlightingTimingIntegrationTest extends PriorityFileTimer {
-  @override
-  String get description => 'highlighting';
+class TimingTest extends AbstractTimingTest {
+  List<Metric> _metrics;
 
-  @override
-  Stream get eventStream => onAnalysisHighlights;
+  List<Metric> get metrics =>
+      _metrics ??= metricNames.map((name) => getMetric(name)).toList();
 
-  @override
-  AnalysisService get service => AnalysisService.HIGHLIGHTS;
-}
-
-@reflectiveTest
-class NavigationTimingIntegrationTest extends PriorityFileTimer {
-  @override
-  String get description => 'navigation';
-
-  @override
-  Stream get eventStream => onAnalysisNavigation;
-
-  @override
-  AnalysisService get service => AnalysisService.NAVIGATION;
-}
-
-@reflectiveTest
-class OutlineTimingIntegrationTest extends PriorityFileTimer {
-  @override
-  String get description => 'outline';
-
-  @override
-  Stream get eventStream => onAnalysisOutline;
-
-  @override
-  AnalysisService get service => AnalysisService.OUTLINE;
-}
-
-abstract class PriorityFileTimer extends AbstractTimingTest {
-  String get description;
-  Stream get eventStream;
-  AnalysisService get service;
+  Metric getMetric(String name) {
+    switch (name) {
+      case 'folding':
+        return new Metric(name, AnalysisService.FOLDING, onAnalysisFolding);
+      case 'highlighting':
+        return new Metric(
+            name, AnalysisService.HIGHLIGHTS, onAnalysisHighlights);
+      case 'implemented':
+        return new Metric(
+            name, AnalysisService.IMPLEMENTED, onAnalysisImplemented);
+      case 'navigation':
+        return new Metric(
+            name, AnalysisService.NAVIGATION, onAnalysisNavigation);
+      case 'outline':
+        return new Metric(name, AnalysisService.OUTLINE, onAnalysisOutline);
+      case 'occurences':
+        return new Metric(
+            name, AnalysisService.OCCURRENCES, onAnalysisOccurrences);
+      case 'overrides':
+        return new Metric(name, AnalysisService.OVERRIDES, onAnalysisOverrides);
+    }
+    print('no metric found for $name');
+    exit(1);
+    return null; // Won't get here.
+  }
 
   Future test_timing() {
+    //debugStdio();
+
+    expect(metrics, isNotEmpty);
     expect(priorityFile, isNotNull,
-        reason: 'A priority file must be specified for $description testing.');
+        reason: 'A priority file must be specified for '
+            '${metrics.first.name} testing.');
+
     stopwatch.start();
 
-    Duration elapsed;
-    eventStream.listen((_) {
-      elapsed = stopwatch.elapsed;
-    });
+    metrics.forEach((Metric m) => m.eventStream.listen((_) {
+          m.timings.add(
+              new Duration(milliseconds: stopwatch.elapsed.inMilliseconds));
+        }));
 
+    var subscriptions = <AnalysisService, List<String>>{};
+    metrics.forEach((Metric m) => subscriptions[m.service] = [priorityFile]);
+
+    sendAnalysisSetSubscriptions(subscriptions);
+
+    // Set root after subscribing to avoid empty notifications.
     setAnalysisRoot();
-    sendAnalysisSetSubscriptions({
-      service: [priorityFile]
-    });
 
     sendAnalysisSetPriorityFiles([priorityFile]);
 
     return analysisFinished.then((_) {
-      print('$description completed in ${elapsed}');
+      print('analysis completed in ${stopwatch.elapsed}');
+      metrics.forEach((Metric m) => print('${m.name} timings: ${m.timings}'));
       stopwatch.reset();
     });
   }
 }