Handle session logs and add event subset selection

pkg/analysis_server/tool/instrumentation/log/log.dart

 // Copyright (c) 2016, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 /**
  * A representation of the contents of an instrumentation log.
  */
-library analysis_server.tool.instrumentation.log;
+library analysis_server.tool.instrumentation.log.log;
 
-import 'dart:collection';
 import 'dart:convert';
 
 import 'package:analyzer/instrumentation/instrumentation.dart';
 
 /**
+ * A boolean-valued function of one argument.
+ */
+typedef bool Predicate<T>(T value);
+
+/**
+ * A description of a group of log entries.
+ */
+class EntryGroup {
+  /**
+   * A list of all of the instances of this class.
+   */
+  static final List<EntryGroup> groups = <EntryGroup>[
+    new EntryGroup._(
+        'nonTask', 'Non-task', (LogEntry entry) => entry is! TaskEntry),
+    new EntryGroup._(
+        'errors',
+        'Errors',
+        (LogEntry entry) =>
+            entry is ErrorEntry ||
+            entry is ExceptionEntry ||
+            (entry is NotificationEntry && entry.isServerError)),
+    new EntryGroup._('malformed', 'Malformed',
+        (LogEntry entry) => entry is MalformedLogEntry),
+    new EntryGroup._('all', 'All', (LogEntry entry) => true),
+  ];
+
+  /**
+   * The unique id of the group.
+   */
+  final String id;
+
+  /**
+   * The human-readable name of the group.
+   */
+  final String name;
+
+  /**
+   * The filter used to determine which entries belong to the group. The filter
+   * should return `true` for members and `false` for non-members.
+   */
+  final Predicate<LogEntry> filter;
+
+  /**
+   * Initialize a newly created entry group with the given state.
+   */
+  EntryGroup._(this.id, this.name, this.filter);
+
+  /**
+   * Given a list of [entries], return all of the entries in the list that are
+   * members of this group.
+   */
+  List<LogEntry> computeMembers(List<LogEntry> entries) {
+    return entries.where(filter).toList();
+  }
+
+  /**
+   * Return the entry group with the given [id], or `null` if there is no group
+   * with the given id.
+   */
+  static EntryGroup withId(String id) {
+    for (EntryGroup group in groups) {
+      if (group.id == id) {
+        return group;
+      }
+    }
+    return null;
+  }
+}
+
+/**
  * A range of log entries, represented by the index of the first and last
  * entries in the range.
  */
 class EntryRange {
   /**
    * The index of the first entry in the range.
    */
   int firstIndex;
 
   /**
@@ skipping 74 matching lines @@
    * The paths of the log files containing the entries.
    */
   List<String> logFilePaths;
 
   /**
    * The entries in the instrumentation log.
    */
   List<LogEntry> logEntries;
 
   /**
-   * The entries in the instrumentation log that are not instances of
-   * [TaskEntry].
+   * A table mapping the entry groups that have been computed to the list of
+   * entries in that group.
    */
-  List<LogEntry> nonTaskEntries;
+  Map<EntryGroup, List<LogEntry>> entryGroups = <EntryGroup, List<LogEntry>>{};
 
   /**
    * A table mapping entries that are paired with another entry to the entry
    * with which they are paired.
    */
-  Map<LogEntry, LogEntry> _pairedEntries = new HashMap<LogEntry, LogEntry>();
+  Map<LogEntry, LogEntry> _pairedEntries = <LogEntry, LogEntry>{};
 
   /**
    * A table mapping the id's of requests to the entry representing the request.
    */
-  Map<String, RequestEntry> _requestMap = new HashMap<String, RequestEntry>();
+  Map<String, RequestEntry> _requestMap = <String, RequestEntry>{};
 
   /**
    * A table mapping the id's of responses to the entry representing the
    * response.
    */
-  Map<String, ResponseEntry> _responseMap =
-      new HashMap<String, ResponseEntry>();
+  Map<String, ResponseEntry> _responseMap = <String, ResponseEntry>{};
 
   /**
    * A table mapping the ids of completion events to the events with those ids.
    */
   Map<String, List<NotificationEntry>> _completionMap =
-      new HashMap<String, List<NotificationEntry>>();
+      <String, List<NotificationEntry>>{};
 
   /**
    * The ranges of entries that are between analysis start and analysis end
    * notifications.
    */
   List<EntryRange> analysisRanges;
 
   /**
    * Initialize a newly created instrumentation log by parsing each of the lines
    * in the [logContent] into a separate entry. The log contents should be the
    * contents of the files whose paths are in the given list of [logFilePaths].
    */
   InstrumentationLog(this.logFilePaths, List<String> logContent) {
     _parseLogContent(logContent);
   }
 
   /**
    * Return a list of the completion events associated with the given [id].
    */
   List<NotificationEntry> completionEventsWithId(String id) =>
       _completionMap[id];
 
   /**
+   * Return the log entries that are contained in the given [group].
+   */
+  List<LogEntry> entriesInGroup(EntryGroup group) =>
+      entryGroups.putIfAbsent(group, () => group.computeMembers(logEntries));
+
+  /**
    * Return the entry that is paired with the given [entry], or `null` if there
    * is no entry paired with it.
    */
   LogEntry pairedEntry(LogEntry entry) => _pairedEntries[entry];
 
   /**
    * Return the response that corresponds to the given request.
    */
   RequestEntry requestFor(ResponseEntry entry) => _requestMap[entry.id];
 
   /**
    * Return the response that corresponds to the given request.
    */
   ResponseEntry responseFor(RequestEntry entry) => _responseMap[entry.id];
 
   /**
    * Return a list containing all of the task entries between the start of
    * analysis notification at the given [startIndex] and the matching end of
    * analysis notification (or the end of the log if the log does not contain a
    * corresponding end notification.
    */
   List<TaskEntry> taskEntriesFor(int startIndex) {
     List<TaskEntry> taskEntries = <TaskEntry>[];
-    NotificationEntry startEntry = nonTaskEntries[startIndex];
+    NotificationEntry startEntry = logEntries[startIndex];
     LogEntry endEntry = pairedEntry(startEntry);
     int lastIndex = endEntry == null ? logEntries.length : endEntry.index;
     for (int i = startEntry.index + 1; i < lastIndex; i++) {
       LogEntry entry = logEntries[i];
       if (entry is TaskEntry) {
         taskEntries.add(entry);
       }
     }
     return taskEntries;
   }
 
   /**
+   * Return `true` if the given [logContent] appears to be from session data.
+   */
+  bool _isSessionData(List<String> logContent) {
+    if (logContent.length < 2) {
+      return false;
+    }
+    String firstLine = logContent[0];
+    return firstLine.startsWith('-----') && logContent[1].startsWith('~') ||
+        firstLine.startsWith('~');
+  }
+
+  /**
    * Merge any multi-line entries into a single line so that every element in
    * the given [logContent] is a single entry.
    */
   void _mergeEntries(List<String> logContent) {
     bool isStartOfEntry(String line) {
       return line.startsWith(LogEntry.entryRegExp);
     }
 
     String merge(String line, List<String> extraLines) {
       StringBuffer buffer = new StringBuffer();
@@ skipping 19 matching lines @@
     if (extraLines.isNotEmpty) {
       throw new StateError(
           '${extraLines.length} non-entry lines before any entry');
     }
   }
 
   /**
    * Parse the given [logContent] into a list of log entries.
    */
   void _parseLogContent(List<String> logContent) {
-    _mergeEntries(logContent);
+    if (_isSessionData(logContent)) {
+      if (logContent[0].startsWith('-----')) {
+        logContent.removeAt(0);
+      }
+      int lastIndex = logContent.length - 1;
+      if (logContent[lastIndex].startsWith('extraction complete')) {
+        logContent.removeAt(lastIndex);
+      }
+    } else {
+      _mergeEntries(logContent);
+    }
     logEntries = <LogEntry>[];
-    nonTaskEntries = <LogEntry>[];
     analysisRanges = <EntryRange>[];
     NotificationEntry analysisStartEntry = null;
     int analysisStartIndex = -1;
     NotificationEntry pubStartEntry = null;
     for (String line in logContent) {
       LogEntry entry = new LogEntry.from(logEntries.length, line);
       if (entry != null) {
         logEntries.add(entry);
-        if (entry is! TaskEntry) {
-          nonTaskEntries.add(entry);
-        }
         if (entry is RequestEntry) {
           _requestMap[entry.id] = entry;
         } else if (entry is ResponseEntry) {
           _responseMap[entry.id] = entry;
           RequestEntry request = _requestMap[entry.id];
           _pairedEntries[entry] = request;
           _pairedEntries[request] = entry;
         } else if (entry is NotificationEntry) {
           if (entry.isServerStatus) {
             var analysisStatus = entry.param('analysis');
@@ skipping 172 matching lines @@
    */
   static final RegExp entryRegExp = new RegExp('[0-9]+\\:');
 
   /**
    * A table mapping kinds to the names of those kinds.
    */
   static final Map<String, String> kindMap = {
     'Err': 'Error',
     'Ex': 'Exception',
     'Log': 'Log message',
+    'Mal': 'Malformed entry',
     'Noti': 'Notification',
     'Read': 'Read file',
     'Req': 'Request',
     'Res': 'Response',
     'Perf': 'Performance data',
     'SPResult': 'Subprocess result',
     'SPStart': 'Subprocess start',
     'Task': 'Task',
     'Ver': 'Version information',
     'Watch': 'Watch event',
@@ skipping 20 matching lines @@
    */
   LogEntry(this.index, this.timeStamp);
 
   /**
    * Create a log entry from the given encoded form of the [entry].
    */
   factory LogEntry.from(int index, String entry) {
     if (entry.isEmpty) {
       return null;
     }
-    List<String> components = _parseComponents(entry);
-    int timeStamp;
     try {
-      timeStamp = int.parse(components[0]);
+      List<String> components = _parseComponents(entry);
+      int timeStamp;
+      String component = components[0];
+      if (component.startsWith('~')) {
+        component = component.substring(1);
+      }
+      timeStamp = int.parse(component);
+      String entryKind = components[1];
+      if (entryKind == InstrumentationService.TAG_ANALYSIS_TASK) {
+        return new TaskEntry(index, timeStamp, components[2], components[3]);
+      } else if (entryKind == InstrumentationService.TAG_ERROR) {
+        return new ErrorEntry(
+            index, timeStamp, entryKind, components.sublist(2));
+      } else if (entryKind == InstrumentationService.TAG_EXCEPTION) {
+        return new ExceptionEntry(
+            index, timeStamp, entryKind, components.sublist(2));
+      } else if (entryKind == InstrumentationService.TAG_FILE_READ) {
+        // Fall through
+      } else if (entryKind == InstrumentationService.TAG_LOG_ENTRY) {
+        // Fall through
+      } else if (entryKind == InstrumentationService.TAG_NOTIFICATION) {
+        Map requestData = JSON.decode(components[2]);
+        return new NotificationEntry(index, timeStamp, requestData);
+      } else if (entryKind == InstrumentationService.TAG_PERFORMANCE) {
+        // Fall through
+      } else if (entryKind == InstrumentationService.TAG_REQUEST) {
+        Map requestData = JSON.decode(components[2]);
+        return new RequestEntry(index, timeStamp, requestData);
+      } else if (entryKind == InstrumentationService.TAG_RESPONSE) {
+        Map responseData = JSON.decode(components[2]);
+        return new ResponseEntry(index, timeStamp, responseData);
+      } else if (entryKind == InstrumentationService.TAG_SUBPROCESS_START) {
+        // Fall through
+      } else if (entryKind == InstrumentationService.TAG_SUBPROCESS_RESULT) {
+        // Fall through
+      } else if (entryKind == InstrumentationService.TAG_VERSION) {
+        // Fall through
+      } else if (entryKind == InstrumentationService.TAG_WATCH_EVENT) {
+        // Fall through
+      }
+      return new GenericEntry(
+          index, timeStamp, entryKind, components.sublist(2));
     } catch (exception) {
-      print('Invalid time stamp in "${components[0]}"; entry = "$entry"');
-      return null;
+      LogEntry logEntry = new MalformedLogEntry(index, entry);
+      logEntry.recordProblem(exception.toString());
+      return logEntry;
     }
-    String entryKind = components[1];
-    if (entryKind == InstrumentationService.TAG_ANALYSIS_TASK) {
-      return new TaskEntry(index, timeStamp, components[2], components[3]);
-    } else if (entryKind == InstrumentationService.TAG_ERROR) {
-      return new ErrorEntry(index, timeStamp, entryKind, components.sublist(2));
-    } else if (entryKind == InstrumentationService.TAG_EXCEPTION) {
-      return new ExceptionEntry(
-          index, timeStamp, entryKind, components.sublist(2));
-    } else if (entryKind == InstrumentationService.TAG_FILE_READ) {
-      // Fall through
-    } else if (entryKind == InstrumentationService.TAG_LOG_ENTRY) {
-      // Fall through
-    } else if (entryKind == InstrumentationService.TAG_NOTIFICATION) {
-      Map requestData = JSON.decode(components[2]);
-      return new NotificationEntry(index, timeStamp, requestData);
-    } else if (entryKind == InstrumentationService.TAG_PERFORMANCE) {
-      // Fall through
-    } else if (entryKind == InstrumentationService.TAG_REQUEST) {
-      Map requestData = JSON.decode(components[2]);
-      return new RequestEntry(index, timeStamp, requestData);
-    } else if (entryKind == InstrumentationService.TAG_RESPONSE) {
-      Map responseData = JSON.decode(components[2]);
-      return new ResponseEntry(index, timeStamp, responseData);
-    } else if (entryKind == InstrumentationService.TAG_SUBPROCESS_START) {
-      // Fall through
-    } else if (entryKind == InstrumentationService.TAG_SUBPROCESS_RESULT) {
-      // Fall through
-    } else if (entryKind == InstrumentationService.TAG_VERSION) {
-      // Fall through
-    } else if (entryKind == InstrumentationService.TAG_WATCH_EVENT) {
-      // Fall through
-    }
-    return new GenericEntry(index, timeStamp, entryKind, components.sublist(2));
   }
 
   /**
    * Return `true` if any problems were found while processing the log file.
    */
   bool get hasProblems => _problems != null;
 
   /**
    * Return the value of the component used to indicate the kind of the entry.
    * This is the abbreviation recorded in the entry.
@@ skipping 68 matching lines @@
       } else {
         component.writeCharCode(char);
       }
     }
     components.add(component.toString());
     return components;
   }
 }
 
 /**
+ * A representation of a malformed log entry.
+ */
+class MalformedLogEntry extends LogEntry {
+  final String entry;
+
+  MalformedLogEntry(int index, this.entry) : super(index, -1);
+
+  @override
+  String get kind => 'Mal';
+
+  @override
+  void _appendDetails(StringBuffer buffer) {
+    super._appendDetails(buffer);
+    buffer.write(entry);
+    buffer.write('<br>');
+  }
+}
+
+/**
  * A log entry representing a notification that was sent from the server to the
  * client.
  */
 class NotificationEntry extends JsonBasedEntry {
   /**
    * Initialize a newly created response to have the given [timeStamp] and
    * [notificationData].
    */
   NotificationEntry(int index, int timeStamp, Map notificationData)
       : super(index, timeStamp, notificationData);
 
   /**
    * Return the event field of the request.
    */
   String get event => data['event'];
 
   /**
+   * Return `true` if this is a server error notification.
+   */
+  bool get isServerError => event == 'server.error';
+
+  /**
    * Return `true` if this is a server status notification.
    */
   bool get isServerStatus => event == 'server.status';
 
   @override
   String get kind => 'Noti';
 
   /**
    * Return the value of the parameter with the given [parameterName], or `null`
    * if there is no such parameter.
@@ skipping 160 matching lines @@
     int slash = context.lastIndexOf('/');
     if (slash < 0) {
       slash = context.lastIndexOf('\\');
     }
     if (slash >= 0) {
       String prefix = context.substring(0, slash);
       _target = _target.replaceAll(prefix, '...');
     }
   }
 }