Implement multi-core builds in MSVS

visual_studio/NativeClientVSAddIn/InstallerResources/NaCl/compiler_wrapper.py

+#!/usr/bin/env python
+# Copyright (c) 2012 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file
+
+"""Python wrapper around gcc to make it behave a little
+more like cl.exe WRT to parallel building.
+"""
+
+import multiprocessing
+import os
+import Queue
+import shlex
+import subprocess
+import sys
+import threading
+import time
+
+verbose = int(os.environ.get('NACL_GCC_VERBOSE', '0'))
+show_commands = int(os.environ.get('NACL_GCC_SHOW_COMMANDS', '0'))
+cores = int(os.environ.get('NACL_GCC_CORES', '0'))
+stop_on_error = False
+
+def RunGCC(cmd, basename):
+  cmdstring = subprocess.list2cmdline(cmd)
+  p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+  stdout, stderr = p.communicate()
+  p.wait()
+  if show_commands:
+    logmsg = cmdstring
+  else:
+    logmsg = basename
+  stdout = logmsg + '\n' + stdout
+  return (p.returncode, stdout, stderr)
+
+
+def BuildSerial(base_cmd, outpath, files):
+  final_result = 0
+
+  for filename in files:
+    cmd, basename = MakeCommand(base_cmd, outpath, filename)
+    rtn, stdout, stderr = RunGCC(cmd, basename)
+    sys.stderr.write(stdout)
+    sys.stderr.write(stderr)
+    if rtn:
+      final_result = rtn
+      if stop_on_error:
+        break
+
+  return final_result
+
+
+
+
+def Worker(queue, out_queue):
+  while not queue.empty() and Worker.running:
+    item = queue.get(False)
+    if not item:
+      break
+    results = RunGCC(item[0], item[1])
+    out_queue.put(results)
+
+
+def MakeCommand(base_cmd, outpath, filename):
+  basename = os.path.basename(filename)
+  out = os.path.join(outpath, os.path.splitext(basename)[0] + '.obj')
+  return (base_cmd + ['-c', filename, '-o', out], basename)
+
+
+def BuildParallel(base_cmd, outpath, files):
+  Worker.running = True
+  pool = []
+  job_queue = Queue.Queue()
+  out_queue = Queue.Queue()
+
+  for filename in files:
+    cmd, basename = MakeCommand(base_cmd, outpath, filename)
+    job_queue.put((cmd, basename))
+
+  # Create worker thread pool, passing job queue
+  # and output queue to each worker.
+  args = (job_queue, out_queue)
+  for i in xrange(cores):
+    t = threading.Thread(target=Worker, args=args)
+    t.start()
+
+  results = 0
+  Trace("waiting for %d results" % len(files))
+  final_result = 0
+  while results < len(files):
+    results += 1
+    rtn, stdout, stderr = out_queue.get()
+    sys.stderr.write(stdout)
+    sys.stderr.write(stderr)
+    if rtn:
+      final_result = rtn
+      if stop_on_error:
+        # stop all workers
+        Worker.running = False
+        break
+
+  return final_result
+
+
+def Log(msg):
+  sys.stderr.write(str(msg) + '\n')
+  sys.stderr.flush()
+
+
+def Trace(msg):
+  if verbose:
+    Log(msg)
+
+
+def main(args):
+  global cores
+  if args[0][0] == '@':
+    rspfile = args[0][1:]
+    args = shlex.split(open(rspfile).read())
+
+  # find the last occurrence of '--' in the argument
+  # list and use that to signify the start of the
+  # list of sources
+  index = list(reversed(args)).index('--')
+  index = len(args) - index
+  base_cmd = args[:index-1]
+  files = args[index:]
+
+  # remove -o <path> from base_cmd
+  index = base_cmd.index('-o')
+  outpath = base_cmd[index+1]
+  del base_cmd[index+1]
+  del base_cmd[index]
+
+  if not cores:
+    cores = multiprocessing.cpu_count()
+
+  cores = min(cores, len(files))
+
+  Trace("compiling %d sources using %d threads" % (len(files), cores))
+
+  if cores == 1:
+    rtn = BuildSerial(base_cmd, outpath, files)
+  else:
+    rtn = BuildParallel(base_cmd, outpath, files)
+
+  Trace("done build of %d sources" % (len(files)))
+  return rtn
+
+
+if __name__ == '__main__':
+  main(sys.argv[1:])

[binji, 2012/10/04 23:27:31]

sys.exit(...) to send the errorcode (as long as you made the change above)

[Sam Clegg, 2012/10/05 21:11:19]

Done.