Revert of Enable the ninja parsing code all the time.

tools/ninja_parser.py

-#!/usr/bin/env python
-# Copyright 2014 The Chromium Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""Primitive ninja parser.
-
-It's primary use case is to be used by isolate_driver.py. It is a standalone
-tool so it can be used to verify its behavior more efficiently. It's a quirky
-tool since it is designed only to gather binary dependencies.
-
-<directory> is assumed to be based on src/, so usually it will have the form
-out/Release. Any target outside this directory is ignored.
-
-This script is assumed to be run *after* the target was built, since it looks
-for the +x bit on the files and only returns the ones that exist.
-"""
-
-import logging
-import os
-import optparse
-import sys
-import time
-
-TOOLS_DIR = os.path.dirname(os.path.abspath(__file__))
-SRC_DIR = os.path.dirname(TOOLS_DIR)
-
-
-### Private stuff.
-
-
-_HEX = frozenset('0123456789abcdef')
-
-
-# Ignore any dependency with one of these extensions. In particular, we do not
-# want to mark tool that would have generated these as binary dependencies,
-# these are likely tools only used during the build process that are not
-# necessary at runtime. The worst thing that would happen is either:
-# - Mapping tools generating a source file but not necessary at run time.
-# - Slower processing.
-# On the other hand, being too aggressive will hide legitimate runtime
-# dependencies. In particular, .a may cause an explicit dependency on a .so.
-_IGNORED_EXTENSIONS = frozenset([
-  '.asm', '.c', '.cc', '.cpp', '.css', '.def', '.grd', '.gypcmd', '.h',
-  '.html', '.idl', '.in', '.jinja2', '.js', '.json', '.manifest', '.mm', '.o',
-  '.obj', '.pak', '.pickle', '.png', '.pdb', '.prep', '.proto', '.py', '.rc',
-  '.strings', '.svg', '.tmp', '.ttf', '.txt', '.xtb', '.wav',
-  ])
-
-
-def _load_ninja_recursively(build_dir, ninja_path, build_steps):
-  """Crudely extracts all the subninja and build referenced in ninja_path.
-
-  In particular, it ignores rule and variable declarations. The goal is to be
-  performant (well, as much as python can be performant) which is currently in
-  the <200ms range for a complete chromium tree. As such the code is laid out
-  for performance instead of readability.
-  """
-  logging.debug('Loading %s', ninja_path)
-  try:
-    with open(os.path.join(build_dir, ninja_path), 'rb') as f:
-      line = None
-      merge_line = ''
-      subninja = []
-      for line in f:
-        line = line.rstrip()
-        if not line:
-          continue
-
-        if line[-1] == '$':
-          # The next line needs to be merged in.
-          if merge_line:
-            merge_line += ' ' + line[:-1].strip(' ')
-          else:
-            merge_line = line[:-1].strip(' ')
-          continue
-
-        if merge_line:
-          line = merge_line + ' ' + line.strip(' ')
-          merge_line = ''
-
-        statement = line[:line.find(' ')]
-        if statement == 'build':
-          # Save the dependency list as a raw string. Only the lines needed will
-          # be processed within _recurse(). This saves a good 70ms of processing
-          # time.
-          build_target, dependencies = line[6:].split(': ', 1)
-          # Interestingly, trying to be smart and only saving the build steps
-          # with the intended extensions ('', '.stamp', '.so') slows down
-          # parsing even if 90% of the build rules can be skipped.
-          # On Windows, a single step may generate two target, so split items
-          # accordingly. It has only been seen for .exe/.exe.pdb combos.
-          for i in build_target.strip().split():
-            build_steps[i] = dependencies
-        elif statement == 'subninja':
-          subninja.append(line[9:])
-  except IOError:
-    print >> sys.stderr, 'Failed to open %s' % ninja_path
-    raise
-
-  total = 1
-  for rel_path in subninja:
-    try:
-      # Load each of the files referenced.
-      # TODO(maruel): Skip the files known to not be needed. It saves an aweful
-      # lot of processing time.
-      total += _load_ninja_recursively(build_dir, rel_path, build_steps)
-    except IOError:
-      print >> sys.stderr, '... as referenced by %s' % ninja_path
-      raise
-  return total
-
-
-def _simple_blacklist(item):
-  """Returns True if an item should be analyzed."""
-  return item not in ('', '|', '||')
-
-
-def _using_blacklist(item):
-  """Returns True if an item should be analyzed.
-
-  Ignores many rules that are assumed to not depend on a dynamic library. If
-  the assumption doesn't hold true anymore for a file format, remove it from
-  this list. This is simply an optimization.
-  """
-  # ninja files use native path format.
-  ext = os.path.splitext(item)[1]
-  if ext in _IGNORED_EXTENSIONS:
-    return False
-  # Special case Windows, keep .dll.lib but discard .lib.
-  if item.endswith('.dll.lib'):
-    return True
-  if ext == '.lib':
-    return False
-  return _simple_blacklist(item)
-
-
-def _should_process(build_dir, target, build_steps, rules_seen):
-  """Returns the raw dependencies if the target should be processed."""
-  if target in rules_seen:
-    # The rule was already seen. Since rules_seen is not scoped at the target
-    # visibility, it usually simply means that multiple targets depends on the
-    # same dependencies. It's fine.
-    return None
-
-  raw_dependencies = build_steps.get(target, None)
-  if raw_dependencies is None:
-    # There is not build step defined to generate 'target'.
-    parts = target.rsplit('_', 1)
-    if len(parts) == 2 and len(parts[1]) == 32 and _HEX.issuperset(parts[1]):
-      # It's because it is a phony rule.
-      return None
-
-    # Kind of a hack, assume source files are always outside build_bir.
-    if (target.startswith('..') and
-        os.path.exists(os.path.join(build_dir, target))):
-      # It's because it is a source file.
-      return None
-
-    logging.debug('Failed to find a build step to generate: %s', target)
-    return None
-  return raw_dependencies
-
-
-def _recurse(build_dir, target, build_steps, rules_seen, blacklist):
-  raw_dependencies = _should_process(build_dir, target, build_steps, rules_seen)
-  rules_seen.add(target)
-  if raw_dependencies is None:
-    return []
-
-  out = [target]
-  # Filter out what we don't want to speed things up. This cuts off large parts
-  # of the dependency tree to analyze.
-  # The first item is the build rule, e.g. 'link', 'cxx', 'phony', 'stamp', etc.
-  dependencies = filter(blacklist, raw_dependencies.split(' ')[1:])
-  logging.debug('recurse(%s) -> %s', target, dependencies)
-  for dependency in dependencies:
-    out.extend(_recurse(
-        build_dir, dependency, build_steps, rules_seen, blacklist))
-  return out
-
-
-def _find_link(build_dir, target, build_steps, rules_seen, search_for):
-  raw_dependencies = _should_process(build_dir, target, build_steps, rules_seen)
-  rules_seen.add(target)
-  if raw_dependencies is None:
-    return
-
-  # Filter out what we don't want to speed things up. This cuts off large parts
-  # of the dependency tree to analyze.
-  # The first item is the build rule, e.g. 'link', 'cxx', 'phony', 'stamp', etc.
-  dependencies = filter(_simple_blacklist, raw_dependencies.split(' ')[1:])
-  for dependency in dependencies:
-    if dependency == search_for:
-      yield [dependency]
-    else:
-      for out in _find_link(
-          build_dir, dependency, build_steps, rules_seen, search_for):
-        yield [dependency] + out
-
-
-### Public API.
-
-
-def load_ninja(build_dir):
-  """Loads build.ninja and the tree of .ninja files in build_dir.
-
-  Returns:
-    dict of the target->build_step where build_step is an unprocessed str.
-
-  TODO(maruel): This should really just be done by ninja itself, then simply
-  process its output.
-  """
-  build_steps = {}
-  total = _load_ninja_recursively(build_dir, 'build.ninja', build_steps)
-  logging.info('Loaded %d ninja files, %d build steps', total, len(build_steps))
-  return build_steps
-
-
-def recurse(build_dir, target, build_steps, blacklist=_using_blacklist):
-  """Recursively returns all the interesting dependencies for the target
-  specified.
-  """
-  return _recurse(build_dir, target, build_steps, set(), blacklist)
-
-
-def find_link(build_dir, target, build_steps, search_for):
-  """Finds all the links from 'target' to 'search_for'.
-
-  Example:
-    With target='foo' and search_for='baz', if foo depends on bar which depends
-    on baz, it will yield only one list, ['foo', 'bar', 'baz']. If there were
-    multiple dependency links in the DAG, each one would be yield as a list.
-  """
-  for link in _find_link(build_dir, target, build_steps, set(), search_for):
-    yield link
-
-
-def post_process_deps(build_dir, dependencies):
-  """Processes the dependency list with OS specific rules.
-
-  It maps the dynamic libraries import library into their actual dynamic
-  library.
-
-  Must be run after the build is completed.
-  """
-  def filter_item(i):
-    if i.endswith('.so.TOC'):
-      # Remove only the suffix .TOC, not the .so!
-      return i[:-4]
-    if i.endswith('.dylib.TOC'):
-      # Remove only the suffix .TOC, not the .dylib!
-      return i[:-4]
-    if i.endswith('.dll.lib'):
-      # Remove only the suffix .lib, not the .dll!
-      return i[:-4]
-    return i
-
-  # Check for execute access. This gets rid of all the phony rules.
-  return [
-    i for i in map(filter_item, dependencies)
-    if os.access(os.path.join(build_dir, i), os.X_OK)
-  ]
-
-
-def main():
-  parser = optparse.OptionParser(
-      usage='%prog [options] <directory> <target> <search_for>',
-      description=sys.modules[__name__].__doc__)
-  parser.add_option(
-      '-v', '--verbose', action='count', default=0,
-      help='Use twice for more info')
-  options, args = parser.parse_args()
-
-  levels = (logging.ERROR, logging.INFO, logging.DEBUG)
-  logging.basicConfig(
-      level=levels[min(len(levels)-1, options.verbose)],
-      format='%(levelname)7s %(message)s')
-  if len(args) == 2:
-    build_dir, target = args
-    search_for = None
-  elif len(args) == 3:
-    build_dir, target, search_for = args
-  else:
-    parser.error('Please provide a directory, a target, optionally a link')
-
-  start = time.time()
-  build_dir = os.path.abspath(build_dir)
-  if not os.path.isdir(build_dir):
-    parser.error('build dir must exist')
-  build_steps = load_ninja(build_dir)
-
-  if search_for:
-    found_one = False
-    # Find how to get to link from target.
-    for rules in find_link(build_dir, target, build_steps, search_for):
-      found_one = True
-      print('%s -> %s' % (target, ' -> '.join(rules)))
-    if not found_one:
-      print('Find to find a link between %s and %s' % (target, search_for))
-    end = time.time()
-    logging.info('Processing took %.3fs', end-start)
-  else:
-    binary_deps = post_process_deps(
-        build_dir, recurse(build_dir, target, build_steps))
-    end = time.time()
-    logging.info('Processing took %.3fs', end-start)
-    print('Binary dependencies:%s' % ''.join('\n  ' + i for i in binary_deps))
-  return 0
-
-
-if __name__ == '__main__':
-  sys.exit(main())