Upgrade analyze.py and related scripts to new world order.

tools/android/loading/loading_model.py

 # Copyright 2015 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Models for loading in chrome.
 
 (Redirect the following to the general model module once we have one)
 A model is an object with the following methods.
   CostMs(): return the cost of the model in milliseconds.
   Set(): set model-specific parameters.
 
 ResourceGraph
   This creates a DAG of resource dependencies from loading.log_requests to model
   loading time. The model may be parameterized by changing the loading time of
   a particular or all resources.
 """
 
 import logging
 import os
 import urlparse
 import sys
 
 import dag
+import loading_trace
 import request_dependencies_lens
 
 class ResourceGraph(object):
   """A model of loading by a DAG (tree?) of resource dependancies.
 
   Set parameters:
     cache_all: if true, assume zero loading time for all resources.
   """
   def __init__(self, trace):
-    """Create from a LoadingTrace.
+    """Create from a LoadingTrace (or json of a trace).
 
     Args:
-      trace: (LoadingTrace) Loading trace.
+      trace: (LoadingTrace/JSON) Loading trace or JSON of a trace.
     """
+    if type(trace) == dict:
+      trace = loading_trace.LoadingTrace.FromJsonDict(trace)
     self._BuildDag(trace)
     self._global_start = min([n.StartTime() for n in self._node_info])
     # Sort before splitting children so that we can correctly dectect if a
     # reparented child is actually a dependency for a child of its new parent.
     try:
       for n in dag.TopologicalSort(self._nodes):
         self._SplitChildrenByTime(self._node_info[n.Index()])
     except AssertionError as exc:
       sys.stderr.write('Bad topological sort: %s\n'
                        'Skipping child split\n' % str(exc))
@@ skipping 235 matching lines @@
       for c in children:
         assert n in c.Predecessors()  # Integrity checking
         queue.append(c)
     assert len(visited) == len(self._nodes)
     return '\n'.join(output)
 
   ##
   ## Internal items
   ##
 
-  _CONTENT_TYPE_TO_COLOR = {'html': 'red', 'css': 'green', 'script': 'blue',
-                            'json': 'purple', 'gif_image': 'grey',
-                            'image': 'orange', 'other': 'white'}
+  _CONTENT_KIND_TO_COLOR = {
+      'application':     'blue',      # Scripts.
+      'font':            'grey70',
+      'image':           'orange',    # This probably catches gifs?
+      'video':           'hotpink1',
+      }
+
+  _CONTENT_TYPE_TO_COLOR = {
+      'html':            'red',
+      'css':             'green',
+      'script':          'blue',
+      'javascript':      'blue',
+      'json':            'purple',
+      'gif':             'grey',
+      'image':           'orange',
+      'jpeg':            'orange',
+      'png':             'orange',
+      'plain':           'brown3',
+      'octet-stream':    'brown3',
+      'other':           'white',
+      }
 
   # This resource type may induce a timing dependency. See _SplitChildrenByTime
   # for details.
   # TODO(mattcary): are these right?
   _CAN_BE_TIMING_PARENT = set(['script', 'magic-debug-content'])
   _CAN_MAKE_TIMING_DEPENDENCE = set(['json', 'other', 'magic-debug-content'])
 
   class _NodeInfo(object):
     """Our internal class that adds cost and other information to nodes.
 
@@ skipping 10 matching lines @@
       Args:
         node: The node to augment.
         request: The request associated with this node.
       """
       self._request = request
       self._node = node
       self._edge_costs = {}
       self._edge_annotations = {}
       # All fields in timing are millis relative to request_time, which is epoch
       # seconds.
-      self._node_cost = max([t for f, t in request.timing._asdict().iteritems()
-                             if f != 'request_time'])
+      self._node_cost = max(
+          [0] + [t for f, t in request.timing._asdict().iteritems()
+                 if f != 'request_time'])
 
     def __str__(self):
       return self.ShortName()
 
     def Node(self):
       return self._node
 
     def Index(self):
       return self._node.Index()
 
@@ skipping 18 matching lines @@
 
     def ContentType(self):
       return self._request.GetContentType()
 
     def ShortName(self):
       """Returns either the hostname of the resource, or the filename,
       or the end of the path. Tries to include the domain as much as possible.
       """
       parsed = urlparse.urlparse(self._request.url)
       path = parsed.path
+      hostname = parsed.hostname if parsed.hostname else '?.?.?'
       if path != '' and path != '/':
         last_path = parsed.path.split('/')[-1]
         if len(last_path) < 10:
           if len(path) < 10:
-            return parsed.hostname + '/' + path
+            return hostname + '/' + path
           else:
-            return parsed.hostname + '/..' + parsed.path[-10:]
+            return hostname + '/..' + parsed.path[-10:]
         elif len(last_path) > 10:
-          return parsed.hostname + '/..' + last_path[:5]
+          return hostname + '/..' + last_path[:5]
         else:
-          return parsed.hostname + '/..' + last_path
+          return hostname + '/..' + last_path
       else:
-        return parsed.hostname
+        return hostname
 
     def Url(self):
       return self._request.url
 
     def SetEdgeCost(self, child, cost):
       assert child.Node() in self._node.Successors()
       self._edge_costs[child] = cost
 
     def AddEdgeAnnotation(self, s, annotation):
       assert s.Node() in self._node.Successors()
@@ skipping 67 matching lines @@
       next_index = len(self._nodes)
       assert request not in index_by_request
       index_by_request[request] = next_index
       node = dag.Node(next_index)
       node_info = self._NodeInfo(node, request)
       self._nodes.append(node)
       self._node_info.append(node_info)
 
     dependencies = request_dependencies_lens.RequestDependencyLens(
         trace).GetRequestDependencies()
-    for child_rq, parent_rq, reason in dependencies:
+    for parent_rq, child_rq, reason in dependencies:
       parent = self._node_info[index_by_request[parent_rq]]
       child = self._node_info[index_by_request[child_rq]]
       edge_cost = child.StartTime() - parent.EndTime()
       if edge_cost < 0:
         edge_cost = 0
         if child.StartTime() < parent.StartTime():
           logging.error('Inverted dependency: %s->%s',
                         parent.ShortName(), child.ShortName())
           # Note that child.StartTime() < parent.EndTime() appears to happen a
           # fair amount in practice.
@@ skipping 65 matching lines @@
           break
       if end_mark >= len(children_by_end_time):
         break  # It's not possible to rearrange any more children.
       if go_to_next_child:
         continue  # We can't rearrange this child, but the next child may be
                   # eligible.
       if children_by_end_time[end_mark].EndTime() <= current.StartTime():
         current.ReparentTo(parent, children_by_end_time[end_mark])
         children_by_end_time[end_mark].AddEdgeAnnotation(current, 'timing')
 
+  def _ContentTypeToColor(self, content_type):
+    if not content_type:
+      type_str = 'other'
+    elif '/' in content_type:
+      kind, type_str = content_type.split('/')
+      if kind in self._CONTENT_KIND_TO_COLOR:
+        return self._CONTENT_KIND_TO_COLOR[kind]
+    else:
+      type_str = content_type
+    return self._CONTENT_TYPE_TO_COLOR[type_str]
+
   def _GraphvizNode(self, index, highlight):
     """Returns a graphviz node description for a given node.
 
     Args:
       index: index of the node.
       highlight: a list of node items to emphasize. Any resource url which
         contains any highlight text will be distinguished in the output.
 
     Returns:
       A string describing the resource in graphviz format.
       The resource is color-coded according to its content type, and its shape
       is oval if its max-age is less than 300s (or if it's not cacheable).
     """
     node_info = self._node_info[index]
-    color = self._CONTENT_TYPE_TO_COLOR[node_info.ContentType()]
+    color = self._ContentTypeToColor(node_info.ContentType())
     max_age = node_info.Request().MaxAge()
     shape = 'polygon' if max_age > 300 else 'oval'
     styles = ['filled']
     if highlight:
       for fragment in highlight:
         if fragment in node_info.Url():
           styles.append('dotted')
           break
     return ('%d [label = "%s\\n%.2f->%.2f (%.2f)"; style = "%s"; '
             'fillcolor = %s; shape = %s];\n'
@@ skipping 89 matching lines @@
       Dict of image url + short name to NodeInfo.
     """
     image_to_info = {}
     for n in self._node_info:
       if (n.ContentType().startswith('image') and
           not self._IsAdUrl(n.Url())):
         key = str((n.Url(), n.ShortName(), n.StartTime()))
         assert key not in image_to_info, n.Url()
         image_to_info[key] = n
     return image_to_info