Implement clang tool that converts std::string("") to std::string().

tools/clang/scripts/run_tool.py

+#!/usr/bin/env python
+# Copyright (c) 2012 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.

[Nico, 2013/03/27 16:35:29]

Doesn't say why this is used instead of tooling's write-to-disk stuff (like the
comment in the previous file promised)

[dcheng, 2013/03/28 00:10:17]

Done now =)

+
+import collections
+import functools
+import multiprocessing
+import subprocess
+import sys
+
+
+"""Represents an edit to a file."""
+Edit= collections.namedtuple(
+    'Edit', ('edit_type', 'offset', 'length', 'replacement'))
+
+
+"""Gets the list of files to run the tool over from git."""
+def _GetFilesFromGit(paths = None):
+  try:
+    args = ['git', 'ls-files']
+    if paths:
+      args.extend(paths)
+    command = subprocess.Popen(args, stdout=subprocess.PIPE)
+    output, _ = command.communicate()
+    return [f for f in output.splitlines() if f[-3:] == '.cc']

[Nico, 2013/03/27 16:35:29]

f.endswith('.cc')

(there's also .m, .mm, .cpp, .h?)


Are there cases where one doesn't want to run a tool on all files in the
compilation db?

[dcheng, 2013/03/28 00:10:17]

1) I didn't really want to parse compile_commands.json. The coupling between the
Clang tool and this python script scraping the output is yucky, and parsing
compile_commands.json would add another coupling.

2) It doesn't need to run across generated files, since there's no point to
processing those anyway. Without any filtering of the output from git ls-files,
the script is only 12% complete at the 7 minute mark (implying a ~56m run time
assuming a constant rate of progress). With the current .cc filtering, the
script takes 30 minutes.

I've updated the filter to be more inclusive though and added some of the
suggested extensions since we definitely do want to rewrite those files as well.
I've left out .h (since these will be rewritten anyway if they're included
anywhere that's compiled) and .cpp (since we shouldn't have those in the Chrome
tree).

+  except OSError:
+    # TODO(dcheng): Handle errors.
+    sys.exit(1)
+
+
+"""Executes the tool.
+
+This is outside the class so it can be pickled."""
+def _ExecuteTool(toolname, build_directory, filename):
+  # TODO(dcheng): Scrape output and handle errors.
+  command = subprocess.Popen((toolname, '-p', build_directory, filename),
+                             stdout=subprocess.PIPE,
+                             stderr=subprocess.PIPE)
+  stdout, stderr = command.communicate()
+  if command.returncode != 0:
+    return {'status': False, 'filename': filename, 'stderr': stderr}
+  else:
+    return {'status': True, 'stdout': stdout}
+
+
+"""God class that does everything."""
+class _CompilerDispatcher(object):
+  def __init__(self, toolname, build_directory, filenames):
+    self.__toolname = toolname
+    self.__build_directory = build_directory
+    self.__filenames = filenames
+    self.__success_count = 0
+    self.__failed_count = 0
+    self.__edits = collections.defaultdict(list)
+
+  @property
+  def edits(self):
+    return self.__edits
+
+  """God function that does everything."""
+  def Run(self):
+    # TODO(dcheng): Add some timing.
+    pool = multiprocessing.Pool()
+    result_iterator = pool.imap_unordered(
+        functools.partial(_ExecuteTool, self.__toolname,
+                          self.__build_directory),
+        self.__filenames)
+    for result in result_iterator:
+      self.__ProcessResult(result)
+    sys.stdout.write('\n')
+    sys.stdout.flush()
+
+  """Process the result."""
+  def __ProcessResult(self, result):
+    if result['status']:
+      self.__success_count += 1
+      self.__AddEditsFromStdout(result['stdout'])
+    else:
+      self.__failed_count += 1
+      sys.stdout.write('\nFailed to process %s\n' % result['filename'])
+      sys.stdout.write(result['stderr'])
+      sys.stdout.write('\n')
+    percentage = (
+        float(self.__success_count + self.__failed_count) /
+        len(self.__filenames)) * 100
+    sys.stdout.write('Succeeded: %d, Failed: %d [%.2f%%]\r' % (
+        self.__success_count, self.__failed_count, percentage))
+    sys.stdout.flush()
+
+  """Extracts and add the list of edits generated on the tool's stdout."""
+  def __AddEditsFromStdout(self, stdout):
+    lines = stdout.splitlines()
+    start_index = lines.index('==== BEGIN EDITS ====')
+    end_index = lines.index('==== END EDITS ====')
+    for line in lines[start_index + 1:end_index]:
+      edit_type, path, offset, length, replacement = line.split(':', 4)
+      # TODO(dcheng): [6:] is a horrible hack to trim off ../../ and is fragile.
+      self.__edits[path[6:]].append(
+          Edit(edit_type, int(offset), int(length), replacement))
+
+
+"""Applies the edits!"""
+def _ApplyEdits(edits):
+  edit_count = 0
+  for k, v in edits.iteritems():
+    # Sort the edits and iterate through them in reverse order. Sorting allows
+    # duplicate edits to be quickly skipped, while reversing means that
+    # subsequent edits don't need to have their offsets updated with each edit
+    # applied.
+    v.sort()
+    last_edit = None
+    with open(k, 'r+') as f:
+      contents = f.read()
+      for edit in reversed(v):
+        if edit == last_edit:
+          continue
+        last_edit = edit
+        # TODO(dcheng): Would bytearray be better?
+        contents = (contents[:edit.offset] +
+                    edit.replacement +
+                    contents[edit.offset + edit.length:])
+        edit_count += 1
+      f.seek(0)
+      f.truncate()
+      f.write(contents)
+  print 'Applied %d edits to %d files' % (edit_count, len(edits))
+
+
+def main(argv):
+  # TODO(dcheng): Assert that we're running from chrome/src.
+  filenames = frozenset(_GetFilesFromGit(argv[2:]))
+  dispatcher = _CompilerDispatcher(argv[0], argv[1], filenames)
+  dispatcher.Run()
+  # Filter out any files that aren't in the git repository, since it's not
+  # useful to modify files that aren't under source control--typically, these
+  # are generated files or files in a git submodule that's not part of Chrome.
+  filtered_edits = {k : v for k, v in dispatcher.edits.iteritems()
+                    if k in filenames}
+  _ApplyEdits(filtered_edits)
+
+
+if __name__ == '__main__':
+  sys.exit(main(sys.argv[1:]))