Implement multi-core builds in MSVS

visual_studio/NativeClientVSAddIn/InstallerResources/NaCl/compiler_wrapper.py

+#!/usr/bin/env python
+# Copyright (c) 2012 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file
+
+"""Python wrapper around gcc to make it behave a little
+more like cl.exe WRT to parallel building.
+"""
+
+import multiprocessing
+import os
+import Queue
+import shlex
+import subprocess
+import sys
+import threading
+import time
+
+verbose = int(os.environ.get('NACL_GCC_VERBOSE', '0'))
+show_commands = int(os.environ.get('NACL_GCC_SHOW_COMMANDS', '0'))
+stop_on_error = False
+
+
+def RunGCC(cmd, basename):
+  """Run gcc and return the result along will the stdout/stderr."""
+  cmdstring = subprocess.list2cmdline(cmd)
+  p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+  stdout, stderr = p.communicate()
+  p.wait()
+  if show_commands:
+    logmsg = cmdstring
+  else:
+    logmsg = basename
+  stderr = logmsg + '\n' + stderr
+  return (p.returncode, stdout, stderr)
+
+
+def BuildSerial(base_cmd, outpath, files):
+  final_result = 0
+
+  for filename in files:
+    cmd, basename = MakeCommand(base_cmd, outpath, filename)
+    rtn, stdout, stderr = RunGCC(cmd, basename)
+    sys.stdout.write(stdout)
+    sys.stdout.flush()
+    sys.stderr.write(stderr)
+    sys.stderr.flush()
+    if rtn:
+      final_result = rtn
+      if stop_on_error:
+        break
+
+  return final_result
+
+
+def Worker(queue, out_queue):
+  """Entry point got worker threads.
+
+  Each thread will compiler jobs from the queue until
+  there are no jobs left or until the main thread signals
+  for the work to stop.
+  """
+  while not queue.empty() and Worker.running:
+    item = queue.get(False)
+    if not item:
+      break
+    results = RunGCC(item[0], item[1])
+    out_queue.put(results)
+
+
+def MakeCommand(base_cmd, outpath, filename):
+  """Build the full commandline given that output root
+  and the intput filename.
+
+  If VS passes an existing directory to -o, then we derive the
+  actual object name by combining he directory name with the
+  basename of the source file and andding ".obj"
+  """
+  basename = os.path.basename(filename)
+  out = os.path.join(outpath, os.path.splitext(basename)[0] + '.obj')
+  return (base_cmd + ['-c', filename, '-o', out], basename)
+
+
+def BuildParallel(cores, base_cmd, outpath, files):
+  Worker.running = True
+  pool = []
+  job_queue = Queue.Queue()
+  out_queue = Queue.Queue()
+
+  for filename in files:
+    cmd, basename = MakeCommand(base_cmd, outpath, filename)
+    job_queue.put((cmd, basename))
+
+  # Create worker thread pool, passing job queue
+  # and output queue to each worker.
+  args = (job_queue, out_queue)
+  for i in xrange(cores):
+    t = threading.Thread(target=Worker, args=args)
+    t.start()
+
+  results = 0
+  Trace("waiting for %d results" % len(files))
+  final_result = 0
+  while results < len(files):
+    results += 1
+    rtn, stdout, stderr = out_queue.get()
+    # stdout seem to be completely ignored by visual studio
+    # but GCC should output all useful information on stderr
+    # anyway.
+    sys.stdout.write(stdout)
+    sys.stdout.flush()
+    sys.stderr.write(stderr)
+    sys.stderr.flush()
+    if rtn:
+      final_result = rtn
+      if stop_on_error:
+        # stop all workers
+        Worker.running = False
+        break
+
+  return final_result
+
+
+def Log(msg):
+  """Log message to stderr."""
+  # Since Visual Studio basically seems to completely ignore the stdout
+  # of the compiler and only echo stderr we print everythign to stderr.
+  sys.stderr.write(str(msg) + '\n')
+  sys.stderr.flush()
+
+
+def Trace(msg):
+  if verbose:
+    Log("nacl_compiler:" + str(msg))
+
+
+def main(args):
+  if args[0][0] == '@':
+    rspfile = args[0][1:]
+    args = shlex.split(open(rspfile).read())
+
+  # find the last occurrence of '--' in the argument
+  # list and use that to signify the start of the
+  # list of sources
+  index = list(reversed(args)).index('--')
+  index = len(args) - index
+  base_cmd = args[:index-1]
+  files = args[index:]
+
+  # remove -o <path> from base_cmd
+  index = base_cmd.index('-o')
+  outpath = base_cmd[index+1]
+  del base_cmd[index+1]
+  del base_cmd[index]
+
+  cores = int(os.environ.get('NACL_GCC_CORES', '0'))
+  if not cores:
+    cores = multiprocessing.cpu_count()
+  cores = min(cores, len(files))
+
+  Trace("compiling %d sources using %d threads" % (len(files), cores))
+  rtn = BuildParallel(cores, base_cmd, outpath, files)
+  Trace("returning %d" % rtn)
+  return rtn
+
+
+if __name__ == '__main__':
+  sys.exit(main(sys.argv[1:]))