tools/android/loading: Add support for multiple redirects.

tools/android/loading/request_track.py

 # Copyright (c) 2016 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """The request data track.
 
 When executed, parses a JSON dump of DevTools messages.
 """
 
 import collections
@@ skipping 28 matching lines @@
   """
   return json.loads(json.dumps(timing))
 
 class Request(object):
   """Represents a single request.
 
   Generally speaking, fields here closely mirror those documented in
   third_party/WebKit/Source/devtools/protocol.json.
 
   Fields:
-    request_id: (str) unique request ID. Postfixed with REDIRECT_SUFFIX for
+    request_id: (str) unique request ID. Postfixed with _REDIRECT_SUFFIX for
                 redirects.
     frame_id: (str) unique frame identifier.
     loader_id: (str) unique frame identifier.
     document_url: (str) URL of the document this request is loaded for.
     url: (str) Request URL.
     protocol: (str) protocol used for the request.
     method: (str) HTTP method, such as POST or GET.
     request_headers: (dict) {'header': 'value'} Request headers.
     response_headers: (dict) {'header': 'value'} Response headers.
     initial_priority: (str) Initial request priority, in REQUEST_PRIORITIES.
@@ skipping 10 matching lines @@
     encoded_data_length: (int) Total encoded data length.
     data_chunks: (list) [(offset, encoded_data_length), ...] List of data
                  chunks received, with their offset in ms relative to
                  Timing.requestTime.
     failed: (bool) Whether the request failed.
   """
   REQUEST_PRIORITIES = ('VeryLow', 'Low', 'Medium', 'High', 'VeryHigh')
   RESOURCE_TYPES = ('Document', 'Stylesheet', 'Image', 'Media', 'Font',
                     'Script', 'TextTrack', 'XHR', 'Fetch', 'EventSource',
                     'WebSocket', 'Manifest', 'Other')
-  INITIATORS = ('parser', 'script', 'other')
+  INITIATORS = ('parser', 'script', 'other', 'redirect')
+  INITIATING_REQUEST = 'initiating_request'
+  ORIGINAL_INITIATOR = 'original_initiator'
   def __init__(self):
     self.request_id = None
     self.frame_id = None
     self.loader_id = None
     self.document_url = None
     self.url = None
     self.protocol = None
     self.method = None
     self.request_headers = None
     self.response_headers = None
@@ skipping 74 matching lines @@
 
   def __hash__(self):
     return hash(self.request_id)
 
   def __str__(self):
     return json.dumps(self.ToJsonDict(), sort_keys=True, indent=2)
 
 
 class RequestTrack(devtools_monitor.Track):
   """Aggregates request data."""
-  REDIRECT_SUFFIX = '.redirect'
+  _REDIRECT_SUFFIX = '.redirect'
   # Request status
   _STATUS_SENT = 0
   _STATUS_RESPONSE = 1
   _STATUS_DATA = 2
   _STATUS_FINISHED = 3
   _STATUS_FAILED = 4
   # Serialization KEYS
   _EVENTS_KEY = 'events'
   _METADATA_KEY = 'metadata'
   _DUPLICATES_KEY = 'duplicates_count'
+  _INCONSISTENT_INITIATORS_KEY = 'inconsistent_initiators'
   def __init__(self, connection):
     super(RequestTrack, self).__init__(connection)
     self._connection = connection
     self._requests = []
     self._requests_in_flight = {}  # requestId -> (request, status)
     self._completed_requests_by_id = {}
+    self._redirects_count_by_id = collections.defaultdict(int)
     if connection:  # Optional for testing.
       for method in RequestTrack._METHOD_TO_HANDLER:
         self._connection.RegisterListener(method, self)
     # responseReceived message are sometimes duplicated. Records the message to
     # detect this.
     self._request_id_to_response_received = {}
     self.duplicates_count = 0
+    self.inconsistent_initiators_count = 0
 
   def Handle(self, method, msg):
     assert method in RequestTrack._METHOD_TO_HANDLER
     params = msg['params']
     request_id = params['requestId']
     RequestTrack._METHOD_TO_HANDLER[method](self, request_id, params)
 
   def GetEvents(self):
     if self._requests_in_flight:
       logging.warning('Number of requests still in flight: %d.'
                       % len(self._requests_in_flight))
     return self._requests
 
   def ToJsonDict(self):
     if self._requests_in_flight:
       logging.warning('Requests in flight, will be ignored in the dump')
     return {self._EVENTS_KEY: [
         request.ToJsonDict() for request in self._requests],
-            self._METADATA_KEY: {self._DUPLICATES_KEY: self.duplicates_count}}
+            self._METADATA_KEY: {
+                self._DUPLICATES_KEY: self.duplicates_count,
+                self._INCONSISTENT_INITIATORS_KEY:
+                self.inconsistent_initiators_count}}
 
   @classmethod
   def FromJsonDict(cls, json_dict):
     assert cls._EVENTS_KEY in json_dict
     assert cls._METADATA_KEY in json_dict
     result = RequestTrack(None)
     requests = [Request.FromJsonDict(request)
                 for request in json_dict[cls._EVENTS_KEY]]
     result._requests = requests
-    result.duplicates_count = json_dict[cls._METADATA_KEY][cls._DUPLICATES_KEY]
+    metadata = json_dict[cls._METADATA_KEY]
+    result.duplicates_count = metadata[cls._DUPLICATES_KEY]
+    result.inconsistent_initiators_count = metadata[
+        cls._INCONSISTENT_INITIATORS_KEY]

[mattcary, 2016/01/26 13:20:26]

Call get() instead so we can load previous versions of a trace? 

Doesn't matter too much now but I suspect we're going to continue to extend the
metadata so might be nice to establish a convention.

[Benoit L, 2016/01/26 13:40:47]

Done.

     return result
 
   def _RequestWillBeSent(self, request_id, params):
     # Several "requestWillBeSent" events can be dispatched in a row in the case
     # of redirects.
+    redirect_initiator = None
     if request_id in self._requests_in_flight:
-      self._HandleRedirect(request_id, params)
+      redirect_initiator = self._HandleRedirect(request_id, params)
     assert (request_id not in self._requests_in_flight
             and request_id not in self._completed_requests_by_id)
     r = Request()
     r.request_id = request_id
     _CopyFromDictToObject(
         params, r, (('frameId', 'frame_id'), ('loaderId', 'loader_id'),
                     ('documentURL', 'document_url'),
                     ('timestamp', 'timestamp'), ('wallTime', 'wall_time'),
                     ('initiator', 'initiator')))
     request = params['request']
     _CopyFromDictToObject(
         request, r, (('url', 'url'), ('method', 'method'),
                      ('headers', 'headers'),
                      ('initialPriority', 'initial_priority')))
     r.resource_type = params.get('type', 'Other')
+    if redirect_initiator:
+      original_initiator = r.initiator
+      r.initiator = redirect_initiator
+      r.initiator[Request.ORIGINAL_INITIATOR] = original_initiator
+      initiating_request = self._completed_requests_by_id[
+          redirect_initiator[Request.INITIATING_REQUEST]]
+      initiating_initiator = initiating_request.initiator.get(
+          Request.ORIGINAL_INITIATOR, initiating_request.initiator)
+      if initiating_initiator != original_initiator:
+        self.inconsistent_initiators_count += 1
     self._requests_in_flight[request_id] = (r, RequestTrack._STATUS_SENT)
 
   def _HandleRedirect(self, request_id, params):
     (r, status) = self._requests_in_flight[request_id]
     assert status == RequestTrack._STATUS_SENT
     # The second request contains timing information pertaining to the first
     # one. Finalize the first request.
     assert 'redirectResponse' in params
     redirect_response = params['redirectResponse']
+
     _CopyFromDictToObject(redirect_response, r,
                           (('headers', 'response_headers'),
                            ('encodedDataLength', 'encoded_data_length'),
                            ('fromDiskCache', 'from_disk_cache')))
     r.timing = TimingFromDict(redirect_response['timing'])
-    r.request_id = request_id + self.REDIRECT_SUFFIX
+
+    redirect_index = self._redirects_count_by_id[request_id]
+    self._redirects_count_by_id[request_id] += 1
+    r.request_id = '%s%s.%d' % (request_id, self._REDIRECT_SUFFIX,
+                                 redirect_index + 1)
+    initiator = {
+        'type': 'redirect', Request.INITIATING_REQUEST: r.request_id}
     self._requests_in_flight[r.request_id] = (r, RequestTrack._STATUS_FINISHED)
     del self._requests_in_flight[request_id]
     self._FinalizeRequest(r.request_id)
+    return initiator
 
   def _RequestServedFromCache(self, request_id, _):
     assert request_id in self._requests_in_flight
     (request, status) = self._requests_in_flight[request_id]
     assert status == RequestTrack._STATUS_SENT
     request.served_from_cache = True
 
   def _ResponseReceived(self, request_id, params):
     assert request_id in self._requests_in_flight
     (r, status) = self._requests_in_flight[request_id]
@@ skipping 96 matching lines @@
 
 
 if __name__ == '__main__':
   import json
   import sys
   events = json.load(open(sys.argv[1], 'r'))
   request_track = RequestTrack(None)
   for event in events:
     event_method = event['method']
     request_track.Handle(event_method, event)