Share logic for generating compile DB with run_tool.py

tools/clang/pylib/clang/compile_db.py

 #!/usr/bin/env python
 # Copyright 2016 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import json
 import os
+import re
+import shlex
+import sys
 import subprocess
 
 
+_RSP_RE = re.compile(r' (@(.+?\.rsp)) ')
+_debugging = False
+
+
+def _ProcessEntry(entry):
+  """Transforms one entry in the compile database to be clang-tool friendly."""
+  # Escape backslashes to prevent shlex from interpreting them.
+  escaped_command = entry['command'].replace('\\', '\\\\')
+  split_command = shlex.split(escaped_command)
+  # Drop gomacc.exe from the front, if present.
+  if split_command[0].endswith('gomacc.exe'):
+    split_command = split_command[1:]
+  # Insert --driver-mode=cl as the first argument.
+  split_command = split_command[:1] + ['--driver-mode=cl'] + split_command[1:]
+  entry['command'] = ' '.join(split_command)
+
+  # Expand the contents of the response file, if any.
+  # http://llvm.org/bugs/show_bug.cgi?id=21634
+  try:
+    match = _RSP_RE.search(entry['command'])
+    if match:
+      rsp_path = os.path.join(entry['directory'], match.group(2))
+      rsp_contents = file(rsp_path).read()
+      entry['command'] = ''.join([
+          entry['command'][:match.start(1)],
+          rsp_contents,
+          entry['command'][match.end(1):]])
+  except IOError:
+    if _debugging:
+      print 'Couldn\'t read response file for %s' % entry['file']
+
+  return entry
+
+
+def _ProcessCompileDatabaseForWindows(compile_db):
+  """Make the compile db generated by ninja on Windows more clang-tool friendly.
+
+  Args:
+    compile_db: The compile database parsed as a Python dictionary.
+
+  Returns:
+    A postprocessed compile db that clang tooling can use.
+  """
+  if _debugging > 0:
+    print 'Read in %d entries from the compile db' % len(compile_db)
+  compile_db = [_ProcessEntry(e) for e in compile_db]
+  original_length = len(compile_db)
+
+  # Filter out NaCl stuff. The clang tooling chokes on them.
+  # TODO(dcheng): This doesn't appear to do anything anymore, remove?
+  compile_db = [e for e in compile_db if '_nacl.cc.pdb' not in e['command']
+      and '_nacl_win64.cc.pdb' not in e['command']]
+  if _debugging > 0:
+    print 'Filtered out %d entries...' % (original_length - len(compile_db))
+
+  # TODO(dcheng): Also filter out multiple commands for the same file. Not sure
+  # how that happens, but apparently it's an issue on Windows.
+  return compile_db
+
+
 def GenerateWithNinja(path):
   """Generates a compile database using ninja.
 
   Args:
     path: The build directory to generate a compile database for.
   """
-  # TODO(dcheng): Incorporate Windows-specific compile DB munging from
-  # https://codereview.chromium.org/718873004
-  print 'Generating compile database in %s...' % path
-  args = ['ninja', '-C', path, '-t', 'compdb', 'cc', 'cxx', 'objc', 'objcxx']
-  output = subprocess.check_output(args)
-  with file(os.path.join(path, 'compile_commands.json'), 'w') as f:
-    f.write(output)
+  # TODO(dcheng): Ensure that clang is enabled by checking the output of
+  # gn args --list=is_clang --short

[Nico, 2017/04/06 13:30:52]

I don't think that does the right thing when clang is enabled by default. On my
Mac:

$  gn args --list=is_clang --short out/gn
is_clang = false

Yet only building with clang is supported on mac.

[dcheng, 2017/04/06 18:54:46]

Doh, I thought it worked on Linux when I tested it... but I guess not. Is there
a better way to do this?

+
+  # First, generate the compile database.
+  json_compile_db = subprocess.check_output([
+      'ninja', '-C', path, '-t', 'compdb', 'cc', 'cxx', 'objc', 'objcxx'])
+  compile_db = json.loads(json_compile_db)
+
+  # TODO(dcheng): Ideally this would check target_os... but not sure there's an
+  # easy way to do that, and (for now) cross-compiles don't work without custom
+  # patches anyway.
+  if sys.platform == 'win32':
+    compile_db = _ProcessCompileDatabaseForWindows(compile_db)
+
+  f = file(os.path.join(path, 'compile_commands.json'), 'w')
+  f.write(json.dumps(compile_db, indent=2))

[Nico, 2017/04/06 13:30:52]

  with open(os.path.join(path, 'compile_commands.json'), 'w') as f:
    f.write...

1. use open, not file
(http://stackoverflow.com/questions/112970/python-when-to-use-file-vs-open)
2. use a scoper so the file gets closed again immediately after it's no longer
needed

[dcheng, 2017/04/06 18:54:46]

Done.

 
 
 def Read(path):
   """Reads a compile database into memory.
 
   Args:
     path: Directory that contains the compile database.
   """
   with open(os.path.join(path, 'compile_commands.json'), 'rb') as db:
     return json.load(db)