tools/android/loading: ContentClassificationLens, ads and tracking requests.

tools/android/loading/loading_model.py

 # Copyright 2015 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Models for loading in chrome.
 
 (Redirect the following to the general model module once we have one)
 A model is an object with the following methods.
   CostMs(): return the cost of the model in milliseconds.
   Set(): set model-specific parameters.
@@ skipping 12 matching lines @@
 import dag
 import loading_trace
 import request_dependencies_lens
 
 class ResourceGraph(object):
   """A model of loading by a DAG (tree?) of resource dependancies.
 
   Set parameters:
     cache_all: if true, assume zero loading time for all resources.
   """
-  def __init__(self, trace):
+  def __init__(self, trace, content_lens=None):
     """Create from a LoadingTrace (or json of a trace).
 
     Args:
       trace: (LoadingTrace/JSON) Loading trace or JSON of a trace.
+      content_lens: (ContentClassificationLens) Lens used to annotate the
+                    nodes, or None.
     """
     if type(trace) == dict:
       trace = loading_trace.LoadingTrace.FromJsonDict(trace)
+    self._content_lens = content_lens
     self._BuildDag(trace)
     self._global_start = min([n.StartTime() for n in self._node_info])
     # Sort before splitting children so that we can correctly dectect if a
     # reparented child is actually a dependency for a child of its new parent.
     try:
       for n in dag.TopologicalSort(self._nodes):
         self._SplitChildrenByTime(self._node_info[n.Index()])
     except AssertionError as exc:
       sys.stderr.write('Bad topological sort: %s\n'
                        'Skipping child split\n' % str(exc))
@@ skipping 273 matching lines @@
   class _NodeInfo(object):
     """Our internal class that adds cost and other information to nodes.
 
     Costs are stored on the node as well as edges. Edge information is only
     stored on successor edges and not predecessor, that is, you get them from
     the parent and not the child.
 
     We also store the request on the node, and expose request-derived
     information like content type.
     """
-    def __init__(self, node, request):
+    def __init__(self, node, request, is_ad, is_tracking):
       """Create a new node info.
 
       Args:
         node: The node to augment.
         request: The request associated with this node.
+        is_ad: (bool) Whether the request is an Ad.
+        is_tracking: (bool) Whether the request is related to tracking.
       """
       self._request = request
+      self._is_ad = is_ad
+      self._is_tracking = is_tracking
       self._node = node
       self._edge_costs = {}
       self._edge_annotations = {}
       # All fields in timing are millis relative to request_time, which is epoch
       # seconds.
       self._node_cost = max(
           [0] + [t for f, t in request.timing._asdict().iteritems()
                  if f != 'request_time'])
 
     def __str__(self):
       return self.ShortName()
 
     def Node(self):
       return self._node
 
     def Index(self):
       return self._node.Index()
 
+    def IsAd(self):
+      return self._is_ad
+
+    def IsTracking(self):
+      return self._is_tracking
+
     def Request(self):
       return self._request
 
     def NodeCost(self):
       return self._node_cost
 
     def EdgeCost(self, s):
       return self._edge_costs[s]
 
     def StartTime(self):
@@ skipping 103 matching lines @@
       trace: A LoadingTrace.
     """
     self._nodes = []
     self._node_info = []
     index_by_request = {}
     for request in trace.request_track.GetEvents():
       next_index = len(self._nodes)
       assert request not in index_by_request
       index_by_request[request] = next_index
       node = dag.Node(next_index)
-      node_info = self._NodeInfo(node, request)
+      is_ad = (self._content_lens.IsAdRequest(request)
+               if self._content_lens else False)
+      is_tracking = (self._content_lens.IsTrackingRequest(request)
+               if self._content_lens else False)
+      node_info = self._NodeInfo(node, request, is_ad, is_tracking)

[mattcary, 2016/01/25 16:08:58]

I think it would be nicer to add setters to NodeInfo for tracking (or maybe even
a single SetRequestContent(is_ad, is_tracking) method, have NodeInfo default
those members to false, and then conditionally set them if we have a
content_lens.

IMHO the pattern of infinitely extending constructor parameters gets
unmaintainable, particularly in tests (true, that doesn't matter here, but
what's a code review if not a chance to preach general principles that don't
apply to the code in question?).

[Benoit L, 2016/01/26 13:31:13]

Absolutely, thanks for the suggestion.
Done.

       self._nodes.append(node)
       self._node_info.append(node_info)
 
     dependencies = request_dependencies_lens.RequestDependencyLens(
         trace).GetRequestDependencies()
     for parent_rq, child_rq, reason in dependencies:
       parent = self._node_info[index_by_request[parent_rq]]
       child = self._node_info[index_by_request[child_rq]]
       edge_cost = child.StartTime() - parent.EndTime()
       if edge_cost < 0:
@@ skipping 105 matching lines @@
     node_info = self._node_info[index]
     color = self._ContentTypeToColor(node_info.ContentType())
     max_age = node_info.Request().MaxAge()
     shape = 'polygon' if max_age > 300 else 'oval'
     styles = ['filled']
     if highlight:
       for fragment in highlight:
         if fragment in node_info.Url():
           styles.append('dotted')
           break
+    if node_info.IsAd() or node_info.IsTracking():
+      styles += ['bold', 'diagonals']
     return ('%d [label = "%s\\n%.2f->%.2f (%.2f)"; style = "%s"; '
             'fillcolor = %s; shape = %s];\n'
             % (index, node_info.ShortName(),
                node_info.StartTime() - self._global_start,
                node_info.EndTime() - self._global_start,
                node_info.EndTime() - node_info.StartTime(),
                ','.join(styles), color, shape))
 
   @classmethod
   def _IsAdUrl(cls, url):
@@ skipping 81 matching lines @@
       Dict of image url + short name to NodeInfo.
     """
     image_to_info = {}
     for n in self._node_info:
       if (n.ContentType().startswith('image') and
           not self._IsAdUrl(n.Url())):
         key = str((n.Url(), n.ShortName(), n.StartTime()))
         assert key not in image_to_info, n.Url()
         image_to_info[key] = n
     return image_to_info