Enable the ninja parsing code all the time.

tools/ninja_parser.py

+#!/usr/bin/env python
+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Primitive ninja parser.
+
+It's primary use case is to be used by isolate_driver.py. It is a standalone
+tool so it can be used to verify its behavior more efficiently. It's a quirky
+tool since it is designed only to gather binary dependencies.
+
+<directory> is assumed to be based on src/, so usually it will have the form
+out/Release. Any target outside this directory is ignored.
+"""
+
+import logging
+import os
+import optparse
+import sys
+import time
+
+TOOLS_DIR = os.path.dirname(os.path.abspath(__file__))
+SRC_DIR = os.path.dirname(TOOLS_DIR)
+
+
+### Private stuff.
+
+
+_HEX = frozenset('0123456789abcdef')
+
+
+# Ignore any dependency with one of these extensions. In particular, we do not
+# want to mark tool that would have generated these as binary dependencies,
+# these are likely tools only used during the build process that are not
+# necessary at runtime. The worst thing that would happen is either:
+# - Mapping tools generating a source file but not necessary at run time.
+# - Slower processing.
+# On the other hand, being too aggressive will hide legitimate runtime
+# dependencies. In particular, .a may cause an explicit dependency on a .so.
+_IGNORED_EXTENSIONS = frozenset([
+  '.asm', '.c', '.cc', '.cpp', '.css', '.def', '.grd', '.gypcmd', '.h',
+  '.html', '.idl', '.in', '.jinja2', '.js', '.json', '.manifest', '.mm', '.o',
+  '.obj', '.pak', '.pickle', '.png', '.pdb', '.prep', '.proto', '.py', '.rc',
+  '.strings', '.svg', '.tmp', '.ttf', '.txt', '.xtb', '.wav',
+  ])
+
+
+def _load_ninja_recursively(build_dir, ninja_path, build_steps):
+  """Crudely extracts all the subninja and build referenced in ninja_path.
+
+  In particular, it ignores rule and variable declarations. The goal is to be
+  performant (well, as much as python can be performant) which is currently in
+  the <200ms range for a complete chromium tree. As such the code is laid out
+  for performance instead of readability.
+  """
+  logging.debug('Loading %s', ninja_path)
+  try:
+    with open(os.path.join(build_dir, ninja_path), 'rb') as f:
+      line = None
+      merge_line = ''
+      subninja = []
+      for line in f:
+        line = line.rstrip()
+        if not line:
+          continue
+
+        if line[-1] == '$':
+          # The next line needs to be merged in.
+          if merge_line:
+            merge_line += ' ' + line[:-1].strip(' ')
+          else:
+            merge_line = line[:-1].strip(' ')
+          continue
+
+        if merge_line:
+          line = merge_line + ' ' + line.strip(' ')
+          merge_line = ''
+
+        statement = line[:line.find(' ')]
+        if statement == 'build':
+          # Save the dependency list as a raw string. Only the lines needed will
+          # be processed within _recurse(). This saves a good 70ms of processing
+          # time.
+          build_target, dependencies = line[6:].split(': ', 1)
+          # Interestingly, trying to be smart and only saving the build steps
+          # with the intended extensions ('', '.stamp', '.so') slows down
+          # parsing even if 90% of the build rules can be skipped.
+          # On Windows, a single step may generate two target, so split items
+          # accordingly. It has only been seen for .exe/.exe.pdb combos.
+          for i in build_target.strip().split():
+            build_steps[i] = dependencies
+        elif statement == 'subninja':
+          subninja.append(line[9:])
+  except IOError:
+    print >> sys.stderr, 'Failed to open %s' % ninja_path
+    raise
+
+  total = 1
+  for rel_path in subninja:
+    try:
+      # Load each of the files referenced.
+      # TODO(maruel): Skip the files known to not be needed. It saves an aweful
+      # lot of processing time.
+      total += _load_ninja_recursively(build_dir, rel_path, build_steps)
+    except IOError:
+      print >> sys.stderr, '... as referenced by %s' % ninja_path
+      raise
+  return total
+
+
+def _simple_blacklist(item):
+  """Returns True if an item should be analyzed."""
+  return item not in ('', '|', '||')
+
+
+def _using_blacklist(item):
+  """Returns True if an item should be analyzed.
+
+  Ignores many rules that are assumed to not depend on a dynamic library. If
+  the assumption doesn't hold true anymore for a file format, remove it from
+  this list. This is simply an optimization.
+  """
+  # ninja files use native path format.
+  ext = os.path.splitext(item)[1]
+  if ext in _IGNORED_EXTENSIONS:
+    return False
+  # Special case Windows, keep .dll.lib but discard .lib.
+  if item.endswith('.dll.lib'):
+    return True
+  if ext == '.lib':
+    return False
+  return _simple_blacklist(item)
+
+
+def _should_process(build_dir, target, build_steps, rules_seen):
+  """Returns the raw dependencies if the target should be processed."""
+  if target in rules_seen:
+    # The rule was already seen. Since rules_seen is not scoped at the target
+    # visibility, it usually simply means that multiple targets depends on the
+    # same dependencies. It's fine.
+    return None
+
+  raw_dependencies = build_steps.get(target, None)
+  if raw_dependencies is None:
+    # There is not build step defined to generate 'target'.
+    parts = target.rsplit('_', 1)
+    if len(parts) == 2 and len(parts[1]) == 32 and _HEX.issuperset(parts[1]):
+      # It's because it is a phony rule.
+      return None
+
+    # Kind of a hack, assume source files are always outside build_bir.
+    if (target.startswith('..') and
+        os.path.exists(os.path.join(build_dir, target))):
+      # It's because it is a source file.
+      return None
+
+    logging.debug('Failed to find a build step to generate: %s', target)
+    return None
+  return raw_dependencies
+
+
+def _recurse(build_dir, target, build_steps, rules_seen, blacklist):
+  raw_dependencies = _should_process(build_dir, target, build_steps, rules_seen)
+  rules_seen.add(target)
+  if raw_dependencies is None:
+    return []
+
+  out = [target]
+  # Filter out what we don't want to speed things up. This cuts off large parts
+  # of the dependency tree to analyze.
+  # The first item is the build rule, e.g. 'link', 'cxx', 'phony', 'stamp', etc.
+  dependencies = filter(blacklist, raw_dependencies.split(' ')[1:])
+  logging.debug('recurse(%s) -> %s', target, dependencies)
+  for dependency in dependencies:
+    out.extend(_recurse(
+        build_dir, dependency, build_steps, rules_seen, blacklist))
+  return out
+
+
+def _find_link(build_dir, target, build_steps, rules_seen, search_for):
+  raw_dependencies = _should_process(build_dir, target, build_steps, rules_seen)
+  rules_seen.add(target)
+  if raw_dependencies is None:
+    return
+
+  # Filter out what we don't want to speed things up. This cuts off large parts
+  # of the dependency tree to analyze.
+  # The first item is the build rule, e.g. 'link', 'cxx', 'phony', 'stamp', etc.
+  dependencies = filter(_simple_blacklist, raw_dependencies.split(' ')[1:])
+  for dependency in dependencies:
+    if dependency == search_for:
+      yield [dependency]
+    else:
+      for out in _find_link(
+          build_dir, dependency, build_steps, rules_seen, search_for):
+        yield [dependency] + out
+
+
+### Public API.
+
+
+def load_ninja(build_dir):
+  """Loads build.ninja and the tree of .ninja files in build_dir.

[Vadim Sh., 2014/05/07 18:08:34]

Doc string: What does it return?

[M-A Ruel, 2014/05/07 19:39:08]

Done.

+
+  TODO(maruel): This should really just be done by ninja itself, then simply
+  process its output.
+  """
+  build_steps = {}
+  total = _load_ninja_recursively(build_dir, 'build.ninja', build_steps)
+  logging.info('Loaded %d ninja files, %d build steps', total, len(build_steps))
+  return build_steps
+
+
+def recurse(build_dir, target, build_steps, blacklist=_using_blacklist):
+  """Recursively returns all the interesting dependencies for the target
+  specified.
+  """
+  return _recurse(build_dir, target, build_steps, set(), blacklist)
+
+
+def find_link(build_dir, target, build_steps, search_for):
+  """Finds all the links from 'target' to 'search_for'."""

[Vadim Sh., 2014/05/07 18:08:34]

Doc string: What is 'link' exactly? List of targets?

[M-A Ruel, 2014/05/07 19:39:08]

Done.

+  for link in _find_link(build_dir, target, build_steps, set(), search_for):
+    yield link
+
+
+def post_process_deps(build_dir, dependencies):
+  """Processes the dependency list with OS specific rules."""

[Vadim Sh., 2014/05/07 18:08:34]

Doc string: what does it do exactly? Nor name, nor "processes the dependency
list" doesn't hint on why this function exists...

Also mention somewhere that it works only after ninja built all the targets
(since it checks X file flag). I think it is worth to mention this assumption in
module-level doc string. i.e. when I see 'ninja_parser' my first assumption
would be that it works purely on top of ninja files, not the build output
directory.

[M-A Ruel, 2014/05/07 19:39:08]

Done.

+  def filter_item(i):
+    if i.endswith('.so.TOC'):
+      # Remove only the suffix .TOC, not the .so!
+      return i[:-4]
+    if i.endswith('.dylib.TOC'):
+      # Remove only the suffix .TOC, not the .dylib!
+      return i[:-4]
+    if i.endswith('.dll.lib'):
+      # Remove only the suffix .lib, not the .dll!
+      return i[:-4]
+    return i
+
+  # Check for execute access. This gets rid of all the phony rules.
+  return [
+    i for i in map(filter_item, dependencies)
+    if os.access(os.path.join(build_dir, i), os.X_OK)
+  ]
+
+
+def main():
+  parser = optparse.OptionParser(
+      usage='%prog [options] <directory> <target> <search_for>',
+      description=sys.modules[__name__].__doc__)
+  parser.add_option(
+      '-v', '--verbose', action='count', default=0,
+      help='Use twice for more info')
+  options, args = parser.parse_args()
+
+  levels = (logging.ERROR, logging.INFO, logging.DEBUG)
+  logging.basicConfig(
+      level=levels[min(len(levels)-1, options.verbose)],
+      format='%(levelname)7s %(message)s')
+  if len(args) == 2:
+    build_dir, target = args
+    search_for = None
+  elif len(args) == 3:
+    build_dir, target, search_for = args
+  else:
+    parser.error('Please provide a directory, a target, optionally a link')
+
+  start = time.time()
+  build_dir = os.path.abspath(build_dir)
+  if not os.path.isdir(build_dir):
+    parser.error('build dir must exist')
+  build_steps = load_ninja(build_dir)
+
+  if search_for:
+    found_one = False
+    # Find how to get to link from target.
+    for rules in find_link(build_dir, target, build_steps, search_for):
+      found_one = True
+      print('%s -> %s' % (target, ' -> '.join(rules)))
+    if not found_one:
+      print('Find to find a link between %s and %s' % (target, search_for))
+    end = time.time()
+    logging.info('Processing took %.3fs', end-start)
+  else:
+    binary_deps = post_process_deps(
+        build_dir, recurse(build_dir, target, build_steps))
+    end = time.time()
+    logging.info('Processing took %.3fs', end-start)
+    print('Binary dependencies:%s' % ''.join('\n  ' + i for i in binary_deps))
+  return 0
+
+
+if __name__ == '__main__':
+  sys.exit(main())