[Telemetry] Refactoring in preparation for video-based Speed Index support.

tools/perf/metrics/speedindex.py

 # Copyright 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import collections
 import os
 
 from metrics import Metric
 
+
 class SpeedIndexMetric(Metric):
   """The speed index metric is one way of measuring page load speed.
 
   It is meant to approximate user perception of page load speed, and it
   is based on the amount of time that it takes to paint to the visual
   portion of the screen. It includes paint events that occur after the
   onload event, and it doesn't include time loading things off-screen.
 
-  This speed index metric is based on the devtools speed index at
-  WebPageTest.org (WPT). For more info see: http://goo.gl/e7AH5l
+  This speed index metric is based on WebPageTest.org (WPT).
+  For more info see: http://goo.gl/e7AH5l
   """
   def __init__(self):
     super(SpeedIndexMetric, self).__init__()
+    self._impl = None
     self._script_is_loaded = False
     self._is_finished = False
     with open(os.path.join(os.path.dirname(__file__), 'speedindex.js')) as f:
       self._js = f.read()
 
   def Start(self, _, tab):
     """Start recording events.
 
     This method should be called in the WillNavigateToPage method of
     a PageMeasurement, so that all the events can be captured. If it's called
     in DidNavigateToPage, that will be too late.
     """
-    tab.StartTimelineRecording()
+    self._impl = (VideoSpeedIndexImpl(tab) if tab.video_capture_supported else
+                  PaintRectSpeedIndexImpl(tab))

[slamm, 2013/12/02 22:51:15]

Given that the two speed indexes have such different implementations, it seems
like the type of speed index should be indicated where the results are
presented.

This line makes it look like they are treated as interchangeable.

[tonyg, 2013/12/02 23:18:36]

I see your point. But I think the Measurements themselves just want SpeedIndex
captured in the most accurate way possible. I'd like to set the page cyclers on
the bots to always capture it.

Then the implementation just chooses the best method available. We know that
video always beats paint rect, and I'd like to implement video in more places.

[slamm, 2013/12/03 01:17:32]

Thanks for the context. 

+    self._impl.Start()
     self._script_is_loaded = False
     self._is_finished = False
 
   def Stop(self, _, tab):
     """Stop timeline recording."""
-    assert self.IsFinished(tab)
-    tab.StopTimelineRecording()
+    assert self._impl, 'Must call Start() before Stop()'
+    assert self.IsFinished(tab), 'Must wait for IsFinished() before Stop()'
+    self._impl.Stop()
 
   # Optional argument chart_name is not in base class Metric.
   # pylint: disable=W0221
   def AddResults(self, tab, results, chart_name=None):
     """Calculate the speed index and add it to the results."""
-    events = tab.timeline_model.GetAllEvents()
-    index = _SpeedIndex(events, _GetViewportSize(tab))
+    index = self._impl.CalculateSpeedIndex()
     results.Add('speed_index', 'ms', index, chart_name=chart_name)
 
   def IsFinished(self, tab):
     """Decide whether the timeline recording should be stopped.
 
     When the timeline recording is stopped determines which paint events
     are used in the speed index metric calculation. In general, the recording
     should continue if there has just been some data received, because
     this suggests that painting may continue.
 
@@ skipping 19 matching lines @@
     if not self._script_is_loaded:
       tab.ExecuteJavaScript(self._js)
       self._script_is_loaded = True
 
     time_since_last_response_ms = tab.EvaluateJavaScript(
         "window.timeSinceLastResponseAfterLoadMs()")
     self._is_finished = time_since_last_response_ms > 2000
     return self._is_finished
 
 
-def _GetViewportSize(tab):
-  """Returns dimensions of the viewport."""
-  return tab.EvaluateJavaScript('[ window.innerWidth, window.innerHeight ]')
-
-
-def _SpeedIndex(events, viewport):
-  """Calculate the speed index of a page load from a list of events.
-
-  The speed index number conceptually represents the number of milliseconds
-  that the page was "visually incomplete". If the page were 0% complete for
-  1000 ms, then the score would be 1000; if it were 0% complete for 100 ms
-  then 90% complete (ie 10% incomplete) for 900 ms, then the score would be
-  1.0*100 + 0.1*900 = 190.
-
-  Args:
-    events: A list of telemetry.core.timeline.slice.Slice objects
-    viewport: A tuple (width, height) of the window.
-
-  Returns:
-    A single number, milliseconds of visual incompleteness.
-  """
-  paint_events = _IncludedPaintEvents(events)
-  time_area_dict = _TimeAreaDict(paint_events, viewport)
-  time_completeness_dict = _TimeCompletenessDict(time_area_dict)
-  # The first time interval starts from the start of the first event.
-  prev_time = events[0].start
-  prev_completeness = 0.0
-  speed_index = 0.0
-  for time, completeness in sorted(time_completeness_dict.items()):
-    # Add the incemental value for the interval just before this event.
-    elapsed_time = time - prev_time
-    incompleteness = (1.0 - prev_completeness)
-    speed_index += elapsed_time * incompleteness
-
-    # Update variables for next iteration.
-    prev_completeness = completeness
-    prev_time = time
-
-  return speed_index
-
-
-def _TimeCompletenessDict(time_area_dict):
-  """Make a dictionary of time to visual completeness.
-
-  In the WPT PHP implementation, this is also called 'visual progress'.
-  """
-  total_area = sum(time_area_dict.values())
-  assert total_area > 0.0, 'Total paint event area must be greater than 0.'
-  completeness = 0.0
-  time_completeness_dict = {}
-  for time, area in sorted(time_area_dict.items()):
-    completeness += float(area) / total_area
-    # Visual progress is rounded to the nearest percentage point as in WPT.
-    time_completeness_dict[time] = round(completeness, 2)
-  return time_completeness_dict
-
-
-def _IncludedPaintEvents(events):
-  """Get all events that are counted in the calculation of the speed index.
-
-  There's one category of paint event that's filtered out: paint events
-  that occur before the first 'ResourceReceiveResponse' and 'Layout' events.
-
-  Previously in the WPT speed index, paint events that contain children paint
-  events were also filtered out.
-  """
-  def FirstLayoutTime(events):
-    """Get the start time of the first layout after a resource received."""
-    has_received_response = False
-    for event in events:
-      if event.name == 'ResourceReceiveResponse':
-        has_received_response = True
-      elif has_received_response and event.name == 'Layout':
-        return event.start
-    assert False, 'There were no layout events after resource receive events.'
-
-  first_layout_time = FirstLayoutTime(events)
-  paint_events = [e for e in events
-                  if e.start >= first_layout_time and e.name == 'Paint']
-  return paint_events
-
-
-def _TimeAreaDict(paint_events, viewport):
-  """Make a dict from time to adjusted area value for events at that time.
-
-  The adjusted area value of each paint event is determined by how many paint
-  events cover the same rectangle, and whether it's a full-window paint event.
-  "Adjusted area" can also be thought of as "points" of visual completeness --
-  each rectangle has a certain number of points and these points are
-  distributed amongst the paint events that paint that rectangle.
-
-  Args:
-    paint_events: A list of paint events
-    viewport: A tuple (width, height) of the window.
-
-  Returns:
-    A dictionary of times of each paint event (in milliseconds) to the
-    adjusted area that the paint event is worth.
-  """
-  width, height = viewport
-  fullscreen_area = width * height
-
-  def ClippedArea(rectangle):
-    """Returns rectangle area clipped to viewport size."""
-    _, x0, y0, x1, y1 = rectangle
-    x0 = max(0, x0)
-    y0 = max(0, y0)
-    x1 = min(width, x1)
-    y1 = min(height, y1)
-    return max(0, x1 - x0) * max(0, y1 - y0)
-
-  grouped = _GroupEventByRectangle(paint_events)
-  event_area_dict = collections.defaultdict(int)
-
-  for rectangle, events in grouped.items():
-    # The area points for each rectangle are divided up among the paint
-    # events in that rectangle.
-    area = ClippedArea(rectangle)
-    update_count = len(events)
-    adjusted_area = float(area) / update_count
-
-    # Paint events for the largest-area rectangle are counted as 50%.
-    if area == fullscreen_area:
-      adjusted_area /= 2
-
-    for event in events:
-      # The end time for an event is used for that event's time.
-      event_time = event.end
-      event_area_dict[event_time] += adjusted_area
-
-  return event_area_dict
-
-
-def _GetRectangle(paint_event):
-  """Get the specific rectangle on the screen for a paint event.
-
-  Each paint event belongs to a frame (as in html <frame> or <iframe>).
-  This, together with location and dimensions, comprises a rectangle.
-  In the WPT source, this 'rectangle' is also called a 'region'.
-  """
-  def GetBox(quad):
-    """Gets top-left and bottom-right coordinates from paint event.
-
-    In the timeline data from devtools, paint rectangle dimensions are
-    represented x-y coordinates of four corners, clockwise from the top-left.
-    See: function WebInspector.TimelinePresentationModel.quadFromRectData
-    in file src/out/Debug/obj/gen/devtools/TimelinePanel.js.
-    """
-    x0, y0, _, _, x1, y1, _, _ = quad
-    return (x0, y0, x1, y1)
-
-  assert paint_event.name == 'Paint'
-  frame = paint_event.args['frameId']
-  return (frame,) + GetBox(paint_event.args['data']['clip'])
-
-
-def _GroupEventByRectangle(paint_events):
-  """Group all paint events according to the rectangle that they update."""
-  result = collections.defaultdict(list)
-  for event in paint_events:
-    assert event.name == 'Paint'
-    result[_GetRectangle(event)].append(event)
-  return result
+class SpeedIndexImpl(object):
+
+  def __init__(self, tab):
+    self.tab = tab
+
+  def Start(self):
+    raise NotImplementedError()
+
+  def Stop(self):
+    raise NotImplementedError()
+
+  def GetTimeCompletenessList(self):
+    """Returns a list of time to visual completeness tuples.
+
+    In the WPT PHP implementation, this is also called 'visual progress'.
+    """
+    raise NotImplementedError()
+
+  def CalculateSpeedIndex(self):
+    """Calculate the speed index.
+
+    The speed index number conceptually represents the number of milliseconds
+    that the page was "visually incomplete". If the page were 0% complete for
+    1000 ms, then the score would be 1000; if it were 0% complete for 100 ms
+    then 90% complete (ie 10% incomplete) for 900 ms, then the score would be
+    1.0*100 + 0.1*900 = 190.
+
+    Returns:
+      A single number, milliseconds of visual incompleteness.
+    """
+    time_completeness_list = self.GetTimeCompletenessList()
+    prev_completeness = 0.0
+    speed_index = 0.0
+    prev_time = time_completeness_list[0][0]
+    for time, completeness in time_completeness_list:
+      # Add the incemental value for the interval just before this event.
+      elapsed_time = time - prev_time
+      incompleteness = (1.0 - prev_completeness)
+      speed_index += elapsed_time * incompleteness
+
+      # Update variables for next iteration.
+      prev_completeness = completeness
+      prev_time = time
+    return speed_index
+
+
+class VideoSpeedIndexImpl(SpeedIndexImpl):
+
+  def __init__(self, tab):

[slamm, 2013/12/02 22:51:15]

Document "tab"?

[tonyg, 2013/12/02 23:18:36]

Done (in the base class).

+    super(VideoSpeedIndexImpl, self).__init__(tab)
+    assert self.tab.video_capture_supported
+    self._time_completeness_list = None
+
+  def Start(self):
+    # TODO(tonyg): Bitrate is arbitrary here. Experiment with screen capture
+    # overhead vs. speed index accuracy and set the bitrate appropriately.
+    self.tab.StartVideoCapture(min_bitrate_mbps=4)
+
+  def Stop(self):
+    self._time_completeness_list = []
+    for time_ms, bitmap in self.tab.StopVideoCapture():
+      # TODO(tonyg/szym): Implement this.
+      raise NotImplementedError('SpeedIndex video calculation not implemented.')
+
+  def GetTimeCompletenessList(self):
+    assert self._time_completeness_list, 'Must call Stop() first.'
+    return self._time_completeness_list
+
+
+class PaintRectSpeedIndexImpl(SpeedIndexImpl):
+
+  def __init__(self, tab):
+    super(PaintRectSpeedIndexImpl, self).__init__(tab)
+
+  def Start(self):
+    self.tab.StartTimelineRecording()
+
+  def Stop(self):
+    self.tab.StopTimelineRecording()
+
+  def GetTimeCompletenessList(self):
+    events = self.tab.timeline_model.GetAllEvents()
+    viewport = self._GetViewportSize()
+    paint_events = self._IncludedPaintEvents(events)
+    time_area_dict = self._TimeAreaDict(paint_events, viewport)
+    total_area = sum(time_area_dict.values())
+    assert total_area > 0.0, 'Total paint event area must be greater than 0.'
+    completeness = 0.0
+    time_completeness_list = []
+
+    # TODO(tonyg): This sets the start time to the start of the first paint
+    # event. That can't be correct. The start time should be navigationStart.
+    # Since the previous screen is not cleared at navigationStart, we should
+    # probably assume the completeness is 0 until the first paint and add the
+    # time of navigationStart as the start. We need to confirm what WPT does.
+    time_completeness_list.append(
+        (self.tab.timeline_model.GetAllEvents()[0].start, completeness))
+
+    for time, area in sorted(time_area_dict.items()):
+      completeness += float(area) / total_area
+      # Visual progress is rounded to the nearest percentage point as in WPT.
+      time_completeness_list.append((time, round(completeness, 2)))
+    return time_completeness_list
+
+  def _GetViewportSize(self):
+    """Returns dimensions of the viewport."""
+    return self.tab.EvaluateJavaScript(
+        '[ window.innerWidth, window.innerHeight ]')
+
+  def _IncludedPaintEvents(self, events):
+    """Get all events that are counted in the calculation of the speed index.
+
+    There's one category of paint event that's filtered out: paint events
+    that occur before the first 'ResourceReceiveResponse' and 'Layout' events.
+
+    Previously in the WPT speed index, paint events that contain children paint
+    events were also filtered out.
+    """
+    def FirstLayoutTime(events):
+      """Get the start time of the first layout after a resource received."""
+      has_received_response = False
+      for event in events:
+        if event.name == 'ResourceReceiveResponse':
+          has_received_response = True
+        elif has_received_response and event.name == 'Layout':
+          return event.start
+      assert False, 'There were no layout events after resource receive events.'
+
+    first_layout_time = FirstLayoutTime(events)
+    paint_events = [e for e in events
+                    if e.start >= first_layout_time and e.name == 'Paint']
+    return paint_events
+
+  def _TimeAreaDict(self, paint_events, viewport):
+    """Make a dict from time to adjusted area value for events at that time.
+
+    The adjusted area value of each paint event is determined by how many paint
+    events cover the same rectangle, and whether it's a full-window paint event.
+    "Adjusted area" can also be thought of as "points" of visual completeness --
+    each rectangle has a certain number of points and these points are
+    distributed amongst the paint events that paint that rectangle.
+
+    Args:
+      paint_events: A list of paint events
+      viewport: A tuple (width, height) of the window.
+
+    Returns:
+      A dictionary of times of each paint event (in milliseconds) to the
+      adjusted area that the paint event is worth.
+    """
+    width, height = viewport
+    fullscreen_area = width * height
+
+    def ClippedArea(rectangle):
+      """Returns rectangle area clipped to viewport size."""
+      _, x0, y0, x1, y1 = rectangle
+      x0 = max(0, x0)
+      y0 = max(0, y0)
+      x1 = min(width, x1)
+      y1 = min(height, y1)

[slamm, 2013/12/02 22:51:15]

Better or worse?

clipped_width = max(0, min(width, x1) - max(0, x0))
clipped_height = max(0, min(height, y1) - max(0, y0))
return clipped_width * clipped_height

It avoids x0, x1, y0, y1 having two different meanings (original rectangle coord
and partially clipped rectangle coord).

[tonyg, 2013/12/02 23:18:36]

I like your implementation. Changed here.

+      return max(0, x1 - x0) * max(0, y1 - y0)
+
+    grouped = self._GroupEventByRectangle(paint_events)
+    event_area_dict = collections.defaultdict(int)
+
+    for rectangle, events in grouped.items():
+      # The area points for each rectangle are divided up among the paint
+      # events in that rectangle.
+      area = ClippedArea(rectangle)
+      update_count = len(events)
+      adjusted_area = float(area) / update_count
+
+      # Paint events for the largest-area rectangle are counted as 50%.
+      if area == fullscreen_area:
+        adjusted_area /= 2
+
+      for event in events:
+        # The end time for an event is used for that event's time.
+        event_time = event.end
+        event_area_dict[event_time] += adjusted_area
+
+    return event_area_dict
+
+  def _GetRectangle(self, paint_event):
+    """Get the specific rectangle on the screen for a paint event.
+
+    Each paint event belongs to a frame (as in html <frame> or <iframe>).
+    This, together with location and dimensions, comprises a rectangle.
+    In the WPT source, this 'rectangle' is also called a 'region'.
+    """
+    def GetBox(quad):
+      """Gets top-left and bottom-right coordinates from paint event.
+
+      In the timeline data from devtools, paint rectangle dimensions are
+      represented x-y coordinates of four corners, clockwise from the top-left.
+      See: function WebInspector.TimelinePresentationModel.quadFromRectData
+      in file src/out/Debug/obj/gen/devtools/TimelinePanel.js.
+      """
+      x0, y0, _, _, x1, y1, _, _ = quad
+      return (x0, y0, x1, y1)
+
+    assert paint_event.name == 'Paint'
+    frame = paint_event.args['frameId']
+    return (frame,) + GetBox(paint_event.args['data']['clip'])

[slamm, 2013/12/02 22:51:15]

Perhaps one of the following:

 (frame, GetBox(...))

frame = ...
box = ...
return (frame, box)

[tonyg, 2013/12/02 23:18:36]

GetBox returns a tuple, so I don't think those implementations are equivalent.

[slamm, 2013/12/03 01:17:32]

Ah, of course. It's right there.

+
+  def _GroupEventByRectangle(self, paint_events):
+    """Group all paint events according to the rectangle that they update."""
+    result = collections.defaultdict(list)
+    for event in paint_events:
+      assert event.name == 'Paint'
+      result[self._GetRectangle(event)].append(event)
+    return result