Upgrade analyze.py and related scripts to new world order.

tools/android/loading/process_request_log.py

-#! /usr/bin/python
-# Copyright 2015 The Chromium Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""Creates a Graphviz file visualizing the resource dependencies from a JSON
-file dumped by log_requests.py.
-"""
-
-import collections
-import sys
-import urlparse
-
-import log_parser
-from log_parser import Resource
-
-
-def _BuildResourceDependencyGraph(requests):
-  """Builds the graph of resource dependencies.
-
-  Args:
-    requests: [RequestData, ...]
-
-  Returns:
-    A tuple ([Resource], [(resource1, resource2, reason), ...])
-  """
-  resources = log_parser.GetResources(requests)
-  resources_from_url = {resource.url: resource for resource in resources}
-  requests_by_completion = log_parser.SortedByCompletion(requests)
-  deps = []
-  for r in requests:
-    resource = Resource.FromRequest(r)
-    initiator = r.initiator
-    initiator_type = initiator['type']
-    dep = None
-    if initiator_type == 'parser':
-      url = initiator['url']
-      blocking_resource = resources_from_url.get(url, None)
-      if blocking_resource is None:
-        continue
-      dep = (blocking_resource, resource, 'parser')
-    elif initiator_type == 'script' and 'stackTrace' in initiator:
-      for frame in initiator['stackTrace']:
-        url = frame['url']
-        blocking_resource = resources_from_url.get(url, None)
-        if blocking_resource is None:
-          continue
-        dep = (blocking_resource, resource, 'stack')
-        break
-    else:
-      # When the initiator is a script without a stackTrace, infer that it comes
-      # from the most recent script from the same hostname.
-      # TLD+1 might be better, but finding what is a TLD requires a database.
-      request_hostname = urlparse.urlparse(r.url).hostname
-      sorted_script_requests_from_hostname = [
-          r for r in requests_by_completion
-          if (resource.GetContentType() in ('script', 'html', 'json')
-              and urlparse.urlparse(r.url).hostname == request_hostname)]
-      most_recent = None
-      # Linear search is bad, but this shouldn't matter here.
-      for request in sorted_script_requests_from_hostname:
-        if request.timestamp < r.timing.requestTime:
-          most_recent = request
-        else:
-          break
-      if most_recent is not None:
-        blocking = resources_from_url.get(most_recent.url, None)
-        if blocking is not None:
-          dep = (blocking, resource, 'script_inferred')
-    if dep is not None:
-      deps.append(dep)
-  return (resources, deps)
-
-
-def PrefetchableResources(requests):
-  """Returns a list of resources that are discoverable without JS.
-
-  Args:
-    requests: List of requests.
-
-  Returns:
-    List of discoverable resources, with their initial request.
-  """
-  resource_to_request = log_parser.ResourceToRequestMap(requests)
-  (_, all_deps) = _BuildResourceDependencyGraph(requests)
-  # Only keep "parser" arcs
-  deps = [(first, second) for (first, second, reason) in all_deps
-          if reason == 'parser']
-  deps_per_resource = collections.defaultdict(list)
-  for (first, second) in deps:
-    deps_per_resource[first].append(second)
-  result = []
-  visited = set()
-  to_visit = [deps[0][0]]
-  while len(to_visit) != 0:
-    r = to_visit.pop()
-    visited.add(r)
-    to_visit += deps_per_resource[r]
-    result.append(resource_to_request[r])
-  return result
-
-
-_CONTENT_TYPE_TO_COLOR = {'html': 'red', 'css': 'green', 'script': 'blue',
-                          'json': 'purple', 'gif_image': 'grey',
-                          'image': 'orange', 'other': 'white'}
-
-
-def _ResourceGraphvizNode(resource, request, resource_to_index):
-  """Returns the node description for a given resource.
-
-  Args:
-    resource: Resource.
-    request: RequestData associated with the resource.
-    resource_to_index: {Resource: int}.
-
-  Returns:
-    A string describing the resource in graphviz format.
-    The resource is color-coded according to its content type, and its shape is
-    oval if its max-age is less than 300s (or if it's not cacheable).
-  """
-  color = _CONTENT_TYPE_TO_COLOR[resource.GetContentType()]
-  max_age = log_parser.MaxAge(request)
-  shape = 'polygon' if max_age > 300 else 'oval'
-  return ('%d [label = "%s"; style = "filled"; fillcolor = %s; shape = %s];\n'
-          % (resource_to_index[resource], resource.GetShortName(), color,
-             shape))
-
-
-def _GraphvizFileFromDeps(resources, requests, deps, output_filename):
-  """Writes a graphviz file from a set of resource dependencies.
-
-  Args:
-    resources: [Resource, ...]
-    requests: list of requests
-    deps: [(resource1, resource2, reason), ...]
-    output_filename: file to write the graph to.
-  """
-  with open(output_filename, 'w') as f:
-    f.write("""digraph dependencies {
-    rankdir = LR;
-    """)
-    resource_to_request = log_parser.ResourceToRequestMap(requests)
-    resource_to_index = {r: i for (i, r) in enumerate(resources)}
-    resources_with_edges = set()
-    for (first, second, reason) in deps:
-      resources_with_edges.add(first)
-      resources_with_edges.add(second)
-    if len(resources_with_edges) != len(resources):
-      f.write("""subgraph cluster_orphans {
-  color=black;
-  label="Orphans";
-""")
-      for resource in resources:
-        if resource not in resources_with_edges:
-          request = resource_to_request[resource]
-          f.write(_ResourceGraphvizNode(resource, request, resource_to_index))
-      f.write('}\n')
-
-    f.write("""subgraph cluster_nodes {
-  color=invis;
-""")
-    for resource in resources:
-      request = resource_to_request[resource]
-      print resource.url
-      if resource in resources_with_edges:
-        f.write(_ResourceGraphvizNode(resource, request, resource_to_index))
-    for (first, second, reason) in deps:
-      arrow = ''
-      if reason == 'parser':
-        arrow = '[color = red]'
-      elif reason == 'stack':
-        arrow = '[color = blue]'
-      elif reason == 'script_inferred':
-        arrow = '[color = blue; style=dotted]'
-      f.write('%d -> %d %s;\n' % (
-          resource_to_index[first], resource_to_index[second], arrow))
-    f.write('}\n}\n')
-
-
-def main():
-  filename = sys.argv[1]
-  requests = log_parser.ParseJsonFile(filename)
-  requests = log_parser.FilterRequests(requests)
-  (resources, deps) = _BuildResourceDependencyGraph(requests)
-  _GraphvizFileFromDeps(resources, requests, deps, filename + '.dot')
-
-
-if __name__ == '__main__':
-  main()