performance measurement improvements

pkg/analysis_server/test/performance/driver.dart

 // Copyright (c) 2015, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 library server.driver;
 
 import 'dart:async';
-import 'dart:math' show max;
+import 'dart:math' show max, sqrt;
 
+import 'package:analyzer/src/generated/engine.dart' as engine;
 import 'package:logging/logging.dart';
 
 import '../integration/integration_test_methods.dart';
 import '../integration/integration_tests.dart';
 import 'operation.dart';
 
 final SPACE = ' '.codeUnitAt(0);
 
 void _printColumn(StringBuffer sb, String text, int keyLen,
     {bool rightJustified: false}) {
@@ skipping 69 matching lines @@
    * error response, the future will be completed with an error.
    */
   Future send(String method, Map<String, dynamic> params) {
     return server.send(method, params);
   }
 
   /**
    * Launch the analysis server.
    * Return a [Future] that completes when analysis server has started.
    */
-  Future startServer() async {
+  Future startServer({int diagnosticPort}) async {
     logger.log(Level.FINE, 'starting server');
     initializeInttestMixin();
     server = new Server();
     Completer serverConnected = new Completer();
     onServerConnected.listen((_) {
       logger.log(Level.FINE, 'connected to server');
       serverConnected.complete();
     });
     running = true;
-    return server.start(/*profileServer: true*/).then((params) {
+    return server
+        .start(diagnosticPort: diagnosticPort /*profileServer: true*/)
+        .then((params) {
       server.listenToOutput(dispatchNotification);
       server.exitCode.then((_) {
         logger.log(Level.FINE, 'server stopped');
         running = false;
         _resultsReady();
       });
       return serverConnected.future;
     });
   }
 
@@ skipping 42 matching lines @@
     Duration maxTime = elapsedTimes[0];
     Duration minTime = elapsedTimes[0];
     int totalTimeMicros = 0;
     for (Duration elapsed in elapsedTimes) {
       ++count;
       int timeMicros = elapsed.inMicroseconds;
       maxTime = maxTime.compareTo(elapsed) > 0 ? maxTime : elapsed;
       minTime = minTime.compareTo(elapsed) < 0 ? minTime : elapsed;
       totalTimeMicros += timeMicros;
     }
-    int averageTimeMicros = (totalTimeMicros / count).round();
+    int meanTime = (totalTimeMicros / count).round();
+    List<Duration> sorted = elapsedTimes.toList()..sort();
+    Duration time90th = sorted[(sorted.length * 0.90).round() - 1];
+    Duration time99th = sorted[(sorted.length * 0.99).round() - 1];
+    int differenceFromMeanSquared = 0;
+    for (Duration elapsed in elapsedTimes) {
+      int timeMicros = elapsed.inMicroseconds;
+      int differenceFromMean = timeMicros - meanTime;
+      differenceFromMeanSquared += differenceFromMean * differenceFromMean;
+    }
+    double variance = differenceFromMeanSquared / count;
+    int standardDeviation = sqrt(variance).round();
+
     StringBuffer sb = new StringBuffer();
     _printColumn(sb, tag, keyLen);
     _printColumn(sb, count.toString(), 6, rightJustified: true);
     _printColumn(sb, errorCount.toString(), 6, rightJustified: true);
     _printColumn(sb, unexpectedResultCount.toString(), 6, rightJustified: true);
+    _printDuration(sb, new Duration(microseconds: meanTime));
+    _printDuration(sb, time90th);
+    _printDuration(sb, time99th);
+    _printColumn(sb, standardDeviation.toString(), 15, rightJustified: true);
     _printDuration(sb, minTime);
-    _printDuration(sb, new Duration(microseconds: averageTimeMicros));
     _printDuration(sb, maxTime);
     _printDuration(sb, new Duration(microseconds: totalTimeMicros));
     print(sb.toString());
   }
 
   void record(bool success, Duration elapsed) {
     if (!success) {
       ++errorCount;
     }
     elapsedTimes.add(elapsed);
@@ skipping 16 matching lines @@
  */
 class Results {
   Map<String, Measurement> measurements = new Map<String, Measurement>();
 
   /**
    * Display results on stdout.
    */
   void printResults() {
     print('');
     print('==================================================================');
+    if (engine.AnalysisEngine.instance.useTaskModel) {
+      print('New task model');
+    } else {
+      print('Old task model');
+    }
     print('');
     List<String> keys = measurements.keys.toList()..sort();
     int keyLen = keys.fold(0, (int len, String key) => max(len, key.length));
     _printGroupHeader('Request/Response', keyLen);
     int totalCount = 0;
     int totalErrorCount = 0;
     int totalUnexpectedResultCount = 0;
     for (String tag in keys) {
       Measurement m = measurements[tag];
       if (!m.notification) {
         m.printSummary(keyLen);
         totalCount += m.count;
         totalErrorCount += m.errorCount;
         totalUnexpectedResultCount += m.unexpectedResultCount;
       }
     }
-    _printTotals(keyLen, totalCount, totalErrorCount, totalUnexpectedResultCount);
+    _printTotals(
+        keyLen, totalCount, totalErrorCount, totalUnexpectedResultCount);
     print('');
     _printGroupHeader('Notifications', keyLen);
     for (String tag in keys) {
       Measurement m = measurements[tag];
       if (m.notification) {
         m.printSummary(keyLen);
       }
     }
     /// TODO(danrubel) *** print warnings if driver caches are not empty ****
     print('');
@@ skipping 15 matching lines @@
     }
     measurement.record(success, elapsed);
   }
 
   void recordUnexpectedResults(String tag) {
     measurements[tag].recordUnexpectedResults();
   }
 
   void _printGroupHeader(String groupName, int keyLen) {
     StringBuffer sb = new StringBuffer();
-        _printColumn(sb, groupName, keyLen);
-        _printColumn(sb, 'count', 6, rightJustified: true);
-        _printColumn(sb, 'error', 6, rightJustified: true);
-        _printColumn(sb, 'uxr(1)', 6, rightJustified: true);
-        sb.write('  ');
-        _printColumn(sb, 'minimum', 15);
-        _printColumn(sb, 'average', 15);
-        _printColumn(sb, 'maximum', 15);
-        _printColumn(sb, 'total', 15);
-        print(sb.toString());
+    _printColumn(sb, groupName, keyLen);
+    _printColumn(sb, 'count', 6, rightJustified: true);
+    _printColumn(sb, 'error', 6, rightJustified: true);
+    _printColumn(sb, 'uxr(1)', 6, rightJustified: true);
+    sb.write('  ');
+    _printColumn(sb, 'mean', 15);
+    _printColumn(sb, '90th', 15);
+    _printColumn(sb, '99th', 15);
+    _printColumn(sb, 'std-dev', 15);
+    _printColumn(sb, 'minimum', 15);
+    _printColumn(sb, 'maximum', 15);
+    _printColumn(sb, 'total', 15);
+    print(sb.toString());
   }
 
-  void _printTotals(int keyLen, int totalCount, int totalErrorCount, int totalUnexpectedResultCount) {
+  void _printTotals(int keyLen, int totalCount, int totalErrorCount,
+      int totalUnexpectedResultCount) {
     StringBuffer sb = new StringBuffer();
     _printColumn(sb, 'Totals', keyLen);
     _printColumn(sb, totalCount.toString(), 6, rightJustified: true);
     _printColumn(sb, totalErrorCount.toString(), 6, rightJustified: true);
     _printColumn(sb, totalUnexpectedResultCount.toString(), 6,
         rightJustified: true);
     print(sb.toString());
   }
 }