refactor timing tests to not use the test framework

pkg/analysis_server/benchmark/perf/analysis_timing_tests.dart

 // Copyright (c) 2015, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 library server.performance.analysis.timing;
 
 import 'dart:async';
 import 'dart:io';
 
 import 'package:analysis_server/plugin/protocol/protocol.dart';
 import 'package:args/args.dart';
-import 'package:test_reflective_loader/test_reflective_loader.dart';
 import 'package:unittest/unittest.dart';
 
 import '../../test/utils.dart';
 import 'performance_tests.dart';
 
 /**
  * Pass in the directory of the source to be analyzed as option `--source`,
  * optionally specify a priority file with `--priority` and the specific
  * test to run with `--metric`.  If no test is specified, the default is
  * `analysis`.
  */
 main(List<String> arguments) {
   initializeTestEnvironment();
   ArgParser parser = _createArgParser();
   var args = parser.parse(arguments);
   if (args[SOURCE_OPTION] == null) {
     print('path to source directory must be specified');
     exit(1);
   }
   source = args[SOURCE_OPTION];
   priorityFile = args[PRIORITY_FILE_OPTION];
   metricNames.addAll(args[METRIC_NAME_OPTION]);
   unittestConfiguration.timeout = new Duration(minutes: 20);
 
+  var test;
+
   if (metricNames.isEmpty) {
-    defineReflectiveTests(AnalysisTimingTest);
+    test = new AnalysisTimingTest();
   } else {
-    defineReflectiveTests(SubscriptionTimingTest);
+    test = new SubscriptionTimingTest();
   }
+
+  Future.wait([test.test_timing()]);
 }
 
 const DEFAULT_METRIC = 'analysis';
 const METRIC_NAME_OPTION = 'metric';
 const PRIORITY_FILE_OPTION = 'priority';
 const SOURCE_OPTION = 'source';
 
 final metricNames = <String>[];
 String priorityFile;
 String source;
 
 ArgParser _createArgParser() => new ArgParser()
   ..addOption(METRIC_NAME_OPTION,
       help: 'metric name (defaults to `analysis`)', allowMultiple: true)
   ..addOption(SOURCE_OPTION, help: 'full path to source directory for analysis')
   ..addOption(PRIORITY_FILE_OPTION,
       help: '(optional) full path to a priority file');
 
-class AbstractTimingTest extends AbstractAnalysisServerPerformanceTest {
-  @override
-  Future setUp() => super.setUp().then((_) {
-        sourceDirectory = new Directory(source);
-        subscribeToStatusNotifications();
-      });
-}
+/**
+ * AnalysisTimingTest measures the time taken by the analsyis server to fully analyze
+ * the given directory. Measurement is started after setting the analysis root, and
+ * analysis is considered complete on receiving the `"isAnalyzing": false` message
+ * from the analysis server.
+ */
+class AnalysisTimingTest extends AbstractTimingTest {
+  Future test_timing() async {
+    // Set root after subscribing to avoid empty notifications.
+    await init(source);
 
-@reflectiveTest
-class AnalysisTimingTest extends AbstractTimingTest {
-  Future test_timing() {
-    // Set root after subscribing to avoid empty notifications.
     setAnalysisRoot();
+    stopwatch.start();
+    await analysisFinished;
+    print('analysis completed in ${stopwatch.elapsed}');
+    stopwatch.reset();

[Brian Wilkerson, 2015/10/19 22:50:32]

Does 'reset' also stop the clock? If not, we should probably just stop it. I'm
not sure what the value of resetting it is in any case. (And I'd actually stop
it before the call to 'print', but I'm not sure there's a good reason for doing
so.)

[keertip, 2015/10/19 23:10:11]

Will remove it. This is cruft from when we were trying to run the test multiple
times, and it made sense to reset the stopwatch.

 
-    stopwatch.start();
-    return analysisFinished.then((_) {
-      print('analysis completed in ${stopwatch.elapsed}');
-      stopwatch.reset();
-    });
+    await shutdown();
   }
 }
 
 class Metric {
   List<Duration> timings = <Duration>[];
   Stream eventStream;
   AnalysisService service;
   String name;
   Metric(this.name, this.service, this.eventStream);
   String toString() => '$name: $service, ${eventStream.runtimeType}, $timings';
 }
 
-@reflectiveTest
+/**
+ * SubscriptionTimingTest measures the time taken by the analysis server to return
+ * information for navigation, semantic highlighting, outline, get occurances,
+ * overrides, folding and implemented. These timings are wrt to the specified priority file
+ * - the file that is currently opened and has focus in the editor. Measure the time from
+ * when the client subscribes for the notifications till there is a response from the server.
+ * Does not wait for analysis to be complete before subscribing for notifications.
+ */
 class SubscriptionTimingTest extends AbstractTimingTest {
   List<Metric> _metrics;
 
   List<Metric> get metrics =>
       _metrics ??= metricNames.map((name) => getMetric(name)).toList();
 
   Metric getMetric(String name) {
     switch (name) {
       case 'folding':
         return new Metric(name, AnalysisService.FOLDING, onAnalysisFolding);
@@ skipping 12 matching lines @@
         return new Metric(
             name, AnalysisService.OCCURRENCES, onAnalysisOccurrences);
       case 'overrides':
         return new Metric(name, AnalysisService.OVERRIDES, onAnalysisOverrides);
     }
     print('no metric found for $name');
     exit(1);
     return null; // Won't get here.
   }
 
-  Future test_timing() {
+  Future test_timing() async {
 //   debugStdio();
 
     expect(metrics, isNotEmpty);
     expect(priorityFile, isNotNull,
         reason: 'A priority file must be specified for '
             '${metrics.first.name} testing.');
 
+    await init(source);
     stopwatch.start();
 
     metrics.forEach((Metric m) => m.eventStream.listen((_) {
           m.timings.add(
               new Duration(milliseconds: stopwatch.elapsed.inMilliseconds));
         }));
 
     var subscriptions = <AnalysisService, List<String>>{};
     metrics.forEach((Metric m) => subscriptions[m.service] = [priorityFile]);
 
     sendAnalysisSetSubscriptions(subscriptions);
 
     // Set root after subscribing to avoid empty notifications.
     setAnalysisRoot();
 
     sendAnalysisSetPriorityFiles([priorityFile]);
 
-    return analysisFinished.then((_) {
-      print('analysis completed in ${stopwatch.elapsed}');
-      metrics.forEach((Metric m) => print('${m.name} timings: ${m.timings}'));
-      stopwatch.reset();
-    });
+    await analysisFinished;
+    print('analysis completed in ${stopwatch.elapsed}');
+    metrics.forEach((Metric m) => print('${m.name} timings: ${m.timings}'));
+    stopwatch.reset();
+
+    await shutdown();
   }
 }