Added support to listen on Network.responseReceived and expose that through tab.py

tools/telemetry/telemetry/core/backends/chrome/inspector_network.py

Index: tools/telemetry/telemetry/core/backends/chrome/inspector_network.py
diff --git a/tools/telemetry/telemetry/core/backends/chrome/inspector_network.py b/tools/telemetry/telemetry/core/backends/chrome/inspector_network.py
index 5ef2edbe991c4452f9c95c5a62d5621bee4802d7..b1c3b96378b7d9c2034834d3a9c5b631d164729d 100644
--- a/tools/telemetry/telemetry/core/backends/chrome/inspector_network.py
+++ b/tools/telemetry/telemetry/core/backends/chrome/inspector_network.py
@@ -1,17 +1,230 @@
 # Copyright 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
+import logging
+
+from cherrypy.lib import httputil as _httputil

[tonyg, 2014/03/04 05:27:31]

I don't think we can add this dependency. Is it on chrome developers'
workstations by default? It isn't on mine.

[bolian, 2014/03/04 20:32:44]

Oh, I probably wrong. It is on my chromium + clank client by default, so I
thought it is available for all chrome devs by default. I will see if there is
other lib I can use. Let me know if you something in mind for this purpose. I
need this httputil here to normalize HTTP response headers (for example,
"content-type: text/html", will become "Content-Type: text/html").

[bolian, 2014/03/05 01:22:40]

Actually, all I need is camel case the header key. So I removed this dependency
and directly use title().

+from telemetry.core import util
+from telemetry.core.backends.chrome import timeline_recorder
+
+
+class RecordedHTTPResponseException(Exception):
+  pass
+
+
+class RecordedHTTPResponse(object):

[tonyg, 2014/03/04 05:27:31]

Seems like this should be an InspectorNetworkData akin to InspectorTimelineData.

[bolian, 2014/03/04 20:32:44]

Hmm, this is only one of a few types of data received from the network domain
(Network.responseReceived). InspectorNetworkData seems too general if we add
other data types later.  Or, maybe I can change it to
InspectorNetworkHTTPReponseData?  what do you think?

On 2014/03/04 05:27:31, tonyg wrote:

[bolian, 2014/03/05 01:22:40]

Now I see that the InspectorTimelineData recently added. I synced the change to
HEAD and merged with that change. In that context, this RecordedHTTPResponse
class corresponds to the "raw event" not the timeline data. I renamed the class
to be InspectorNetworkResponseData.

+  def __init__(self, inspector_network, params):
+    self._inspector_network = inspector_network
+    self._request_id = params['requestId']
+    self._timestamp = params['timestamp']
+
+    self._response = params['response']
+    if not self._response:
+      raise RecordedHTTPResponseException('response must exist')
+
+    # Response headers.
+    headers = self._response['headers']
+    self._header_map = _httputil.HeaderMap()
+    for k, v in headers.iteritems():
+      self._header_map[k] = v
+
+    # Request headers.
+    self._request_header_map = None
+    request_headers = self._response['requestHeaders']
+    if request_headers:
+      # Normalize headers with httputil.HeaderMap.
+      self._request_header_map = _httputil.HeaderMap()
+      for k, v in request_headers.iteritems():
+        self._request_header_map[k] = v
+
+    self._body = None
+    self._base64_encoded = False
+    if self._inspector_network:
+      self._served_from_cache = (
+          self._inspector_network.HTTPResponseServedFromCache(self._request_id))
+    else:
+      self._served_from_cache = False
+
+  @property
+  def status(self):
+    return self._response['status']
+
+  def status_text(self):
+    return self._response['status_text']
+
+  @property
+  def headers(self):
+    return self._header_map
+
+  @property
+  def request_headers(self):
+    return self._request_header_map
+
+  @property
+  def timestamp(self):
+    return self._timestamp
+
+  @property
+  def timing(self):
+    if 'timing' in self._response:
+      return self._response['timing']
+    return None
+
+  @property
+  def url(self):
+    return self._response['url']
+
+  @property
+  def request_id(self):
+    return self._request_id
+
+  @property
+  def served_from_cache(self):
+    self._served_from_cache = False
+
+  def GetHeader(self, name):
+    if name in self.headers:
+      return self.headers[name]
+    return None
+
+  def GetBody(self, timeout=60):
+    if not self._body:
+      self._body, self._base64_encoded = (
+        self._inspector_network.GetHTTPResponseBody(self._request_id, timeout))
+    return self._body, self._base64_encoded
+
+  def AsTimelineEvent(self):
+    event = {}
+    event['type'] = 'HTTPResponse'
+    event['startTime'] = self.timestamp
+    # There is no end time. Just return the timestamp instead.
+    event['endTime'] = self.timestamp
+    event['requestId'] = self.request_id
+    event['response'] = self._response
+    event['body'], event['base64_encoded_body'] = self.GetBody()
+    event['served_from_cache'] = self.served_from_cache
+    return event
+
+  @staticmethod
+  def FromTimelineEvent(event):
+    assert event.name == 'HTTPResponse'
+    params = {}
+    params['timestamp'] = event.start
+    params['requestId'] = event.args['requestId']
+    params['response'] = event.args['response']
+    recorded = RecordedHTTPResponse(None, params)
+    recorded._body = event.args['body']
+    recorded._base64_encoded = event.args['base64_encoded_body']
+    recorded._served_from_cache = event.args['served_from_cache']
+    return recorded
+
 
 class InspectorNetwork(object):
   def __init__(self, inspector_backend):
     self._inspector_backend = inspector_backend
+    # Map a url to a list of responses.
+    self._http_responses = []
+    self._served_from_cache = set()
+    self._timeline_recorder = None
 
   def ClearCache(self, timeout=60):
     """Clears the browser's disk and memory cache."""
     res = self._inspector_backend.SyncRequest({
         'method': 'Network.canClearBrowserCache'
         }, timeout)
-    assert res['result'], 'Cache clearing is not supported by this browser'
+    assert res['result'], 'Cache clearing is not supported by this browser.'
     self._inspector_backend.SyncRequest({
         'method': 'Network.clearBrowserCache'
         }, timeout)
+
+  def StartMonitoringNetwork(self):
+    """Starts monitoring network notifications and recording HTTP responses."""
+    self.ClearRecordedHTTPResponses()
+    self._inspector_backend.RegisterDomain(
+        'Network',
+        self._OnNetworkNotification,
+        self._OnClose)
+    request = {
+        'method': 'Network.enable'
+        }
+    self._inspector_backend.SyncRequest(request)
+
+  def StopMonitoringNetwork(self):
+    """Stops monitoring network notifications and recording HTTP responses."""
+    self._inspector_backend.UnregisterDomain('Network')
+    request = {
+        'method': 'Network.disable'
+        }
+    self._inspector_backend.SyncRequest(request)
+
+  def GetRecordedHTTPResponses(self):
+    """Returns all recorded HTTP responses."""
+    return self._http_responses
+
+  def ClearRecordedHTTPResponses(self):
+    """Clears recorded HTTP responses."""
+    self._http_responses = []
+    self._served_from_cache.clear()
+
+  def _OnNetworkNotification(self, msg):
+    if msg['method'] == 'Network.responseReceived':
+      self._RecordHTTPResponse(msg['params'])
+    elif msg['method'] == 'Network.requestServedFromCache':
+      self._served_from_cache.add(msg['params']['requestId'])
+
+  def _RecordHTTPResponse(self, params):
+    required_fields = ['requestId', 'timestamp', 'response']
+    for field in required_fields:
+      if field not in params:
+        logging.waring('HTTP Response missing required field: %s', field)
+        return
+    self._http_responses.append(RecordedHTTPResponse(self, params))
+
+  def GetHTTPResponseBody(self, request_id, timeout=60):
+    try:
+      res =  self._inspector_backend.SyncRequest({
+          'method': 'Network.getResponseBody',
+          'params': {
+              'requestId': request_id,
+              }
+          }, timeout)
+    except util.TimeoutException:
+      logging.warning('Timeout during fetching body for %s' % request_id)
+      return None, False
+    if 'error' in res:
+      return None, False
+    return res['result']['body'], res['result']['base64Encoded']
+
+  def HTTPResponseServedFromCache(self, request_id):
+    return request_id and request_id in self._served_from_cache
+
+  def _OnClose(self):
+    pass
+
+  @property
+  def timeline_recorder (self):
+    if not self._timeline_recorder:
+      self._timeline_recorder = TimelineRecorder(self)
+    return self._timeline_recorder
+
+
+class TimelineRecorder(timeline_recorder.TimelineRecorder):
+  def __init__(self, inspector_network):
+    super(TimelineRecorder, self).__init__()
+    self._inspector_network = inspector_network
+    self._is_recording = False
+
+  def Start(self):
+    assert not self._is_recording, 'Start should only be called once.'
+    self._is_recording = True
+    self._inspector_network.StartMonitoringNetwork()
+
+  def Stop(self, timeline_model):
+    assert self._is_recording, 'Stop should be called after Start.'
+    responses = self._inspector_network.GetRecordedHTTPResponses()
+    events = [r.AsTimelineEvent() for r in responses]
+    self._inspector_network.StopMonitoringNetwork()
+    if len(events) > 0:
+      timeline_model.ImportTraces(
+          [events], shift_world_to_zero=False, freeze=False)
+    self._is_recording = False