Add debug symbols to all generated .isolate.

tools/isolate_driver.py

 #!/usr/bin/env python
 # Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Adaptor script called through build/isolate.gypi.
 
 Creates a wrapping .isolate which 'includes' the original one, that can be
 consumed by tools/swarming_client/isolate.py. Path variables are determined
 based on the current working directory. The relative_cwd in the .isolated file
 is determined based on the .isolate file that declare the 'command' variable to
 be used so the wrapping .isolate doesn't affect this value.
 
 This script loads build.ninja and processes it to determine all the executables
 referenced by the isolated target. It adds them in the wrapping .isolate file.
 
 WARNING: The target to use for build.ninja analysis is the base name of the
 .isolate file plus '_run'. For example, 'foo_test.isolate' would have the target
 'foo_test_run' analysed.
 """
 
 import glob
 import json
 import logging
 import os
 import posixpath
+import re
 import StringIO
 import subprocess
 import sys
 import time
 
 TOOLS_DIR = os.path.dirname(os.path.abspath(__file__))
 SWARMING_CLIENT_DIR = os.path.join(TOOLS_DIR, 'swarming_client')
 SRC_DIR = os.path.dirname(TOOLS_DIR)
 
 sys.path.insert(0, SWARMING_CLIENT_DIR)
@@ skipping 55 matching lines @@
       # TODO(maruel): Skip the files known to not be needed. It saves an aweful
       # lot of processing time.
       total += load_ninja_recursively(build_dir, rel_path, build_steps)
     except IOError:
       print >> sys.stderr, '... as referenced by %s' % ninja_path
       raise
   return total
 
 
 def load_ninja(build_dir):
-  """Loads the tree of .ninja files in build_dir."""
+  """Loads the tree of .ninja files in build_dir.
+
+  Returns:
+    dict(target: list of dependencies).
+  """
   build_steps = {}
   total = load_ninja_recursively(build_dir, 'build.ninja', build_steps)
   logging.info('Loaded %d ninja files, %d build steps', total, len(build_steps))
   return build_steps
 
 
 def using_blacklist(item):
   """Returns True if an item should be analyzed.
 
   Ignores many rules that are assumed to not depend on a dynamic library. If
   the assumption doesn't hold true anymore for a file format, remove it from
   this list. This is simply an optimization.
   """
   # *.json is ignored below, *.isolated.gen.json is an exception, it is produced
   # by isolate_driver.py in 'test_isolation_mode==prepare'.
   if item.endswith('.isolated.gen.json'):
     return True
   IGNORED = (
     '.a', '.cc', '.css', '.dat', '.def', '.frag', '.h', '.html', '.isolate',
     '.js', '.json', '.manifest', '.o', '.obj', '.pak', '.png', '.pdb', '.py',
     '.strings', '.test', '.txt', '.vert',
   )
   # ninja files use native path format.
   ext = os.path.splitext(item)[1]
   if ext in IGNORED:
     return False
   # Special case Windows, keep .dll.lib but discard .lib.
-  if item.endswith('.dll.lib'):
-    return True
-  if ext == '.lib':
-    return False
+  if sys.platform == 'win32':
+    if item.endswith('.dll.lib'):
+      return True
+    if ext == '.lib':
+      return False
   return item not in ('', '|', '||')
 
+# This is a whitelist of known ninja native rules.
+KNOWN_TOOLS = frozenset(
+    (
+      'copy',
+      'copy_infoplist',
+      'cxx',
+      'idl',
+      'link',
+      'link_embed',
+      'mac_tool',
+      'package_framework',
+      'phony',
+      'rc',
+      'solink',
+      'solink_embed',
+      'solink_module',
+      'solink_module_embed',
+      'solink_module_notoc',
+      'solink_notoc',
+      'stamp',
+    ))
+
 
 def raw_build_to_deps(item):
   """Converts a raw ninja build statement into the list of interesting
   dependencies.
   """
-  # TODO(maruel): Use a whitelist instead? .stamp, .so.TOC, .dylib.TOC,
-  # .dll.lib, .exe and empty.
-  # The first item is the build rule, e.g. 'link', 'cxx', 'phony', etc.
-  return filter(using_blacklist, item.split(' ')[1:])
+  items = filter(None, item.split(' '))
+  for i in xrange(len(items) - 2, 0, -1):
+    # Merge back '$ ' escaping.
+    # OMG please delete this code as soon as possible.
+    if items[i].endswith('$'):
+      items[i] = items[i][:-1] + ' ' + items[i+1]
+      items.pop(i+1)
+
+  # Always skip the first item; it is the build rule type, e.g. , etc.
+  if items[0] not in KNOWN_TOOLS:
+    # Check for phony ninja rules.
+    assert re.match(r'^[^.]+_[0-9a-f]{32}$', items[0]), items
+
+  return filter(using_blacklist, items[1:])
 
 
 def collect_deps(target, build_steps, dependencies_added, rules_seen):
   """Recursively adds all the interesting dependencies for |target|
   into |dependencies_added|.
   """
   if rules_seen is None:
     rules_seen = set()
   if target in rules_seen:
     # TODO(maruel): Figure out how it happens.
     logging.warning('Circular dependency for %s!', target)
     return
   rules_seen.add(target)
   try:
     dependencies = raw_build_to_deps(build_steps[target])
   except KeyError:
     logging.info('Failed to find a build step to generate: %s', target)
     return
   logging.debug('collect_deps(%s) -> %s', target, dependencies)
   for dependency in dependencies:
     dependencies_added.add(dependency)
     collect_deps(dependency, build_steps, dependencies_added, rules_seen)
 
 
 def post_process_deps(build_dir, dependencies):
-  """Processes the dependency list with OS specific rules."""
-  def filter_item(i):
-    if i.endswith('.so.TOC'):
-      # Remove only the suffix .TOC, not the .so!
-      return i[:-4]
-    if i.endswith('.dylib.TOC'):
-      # Remove only the suffix .TOC, not the .dylib!
-      return i[:-4]
-    if i.endswith('.dll.lib'):
-      # Remove only the suffix .lib, not the .dll!
-      return i[:-4]
-    return i
+  """Processes the dependency list with OS specific rules.
 
-  def is_exe(i):
-    # This script is only for adding new binaries that are created as part of
-    # the component build.
-    ext = os.path.splitext(i)[1]
-    # On POSIX, executables have no extension.
-    if ext not in ('', '.dll', '.dylib', '.exe', '.nexe', '.so'):
-      return False
+  Returns:
+    list of dependencies to add.
+  """
+  out = []
+  for i in dependencies:
     if os.path.isabs(i):
       # In some rare case, there's dependency set explicitly on files outside
-      # the checkout.
-      return False
-
-    # Check for execute access and strip directories. This gets rid of all the
-    # phony rules.
-    p = os.path.join(build_dir, i)
-    return os.access(p, os.X_OK) and not os.path.isdir(p)
-
-  return filter(is_exe, map(filter_item, dependencies))
+      # the checkout. In practice, it was observed on /usr/bin/eu-strip only on
+      # official Chrome build.
+      continue
+    if os.path.isdir(os.path.join(build_dir, i)):
+      if sys.platform == 'darwin':
+        # This is an application.
+        out.append(i + '/')
+    elif i.endswith('.so.TOC'):
+      out.append(i[:-4])
+    elif i.endswith('.dylib.TOC'):
+      i = i[:-4]
+      out.append(i)
+      # Debug symbols may not be present.
+      i += '.dSym'
+      if os.path.isdir(os.path.join(build_dir, i)):
+        out.append(i + '/')
+    elif i.endswith('.dll.lib'):
+      i = i[:-4]
+      out.append(i)
+      # Naming is inconsistent.
+      if os.path.isfile(os.path.join(build_dir, i + '.pdb')):
+        out.append(i + '.pdb')
+      if os.path.isfile(os.path.join(build_dir, i[:-4] + '.pdb')):
+        out.append(i[:-4] + '.pdb')
+    elif i.endswith('.exe'):
+      out.append(i)
+      # Naming is inconsistent.
+      if os.path.isfile(os.path.join(build_dir, i + '.pdb')):
+        out.append(i + '.pdb')
+      if os.path.isfile(os.path.join(build_dir, i[:-4] + '.pdb')):
+        out.append(i[:-4] + '.pdb')
+    elif i.endswith('.nexe'):
+      out.append(i)
+      i += '.debug'
+      if os.path.isfile(os.path.join(build_dir, i)):
+        out.append(i)
+    elif sys.platform != 'win32':
+      # On POSIX, executables have no extension.
+      if not os.path.splitext(i)[1]:
+        out.append(i)
+  return out
 
 
 def create_wrapper(args, isolate_index, isolated_index):
   """Creates a wrapper .isolate that add dynamic libs.
 
   The original .isolate is not modified.
   """
   cwd = os.getcwd()
   isolate = args[isolate_index]
   # The code assumes the .isolate file is always specified path-less in cwd. Fix
@@ skipping 93 matching lines @@
 
   swarming_client = os.path.join(SRC_DIR, 'tools', 'swarming_client')
   sys.stdout.flush()
   result = subprocess.call(
       [sys.executable, os.path.join(swarming_client, 'isolate.py')] + args)
   return result
 
 
 if __name__ == '__main__':
   sys.exit(main())