Make child process errors kill gyp when parallel processing is on.

pylib/gyp/input.py

 # Copyright (c) 2012 Google Inc. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 from compiler.ast import Const
 from compiler.ast import Dict
 from compiler.ast import Discard
 from compiler.ast import List
 from compiler.ast import Module
 from compiler.ast import Node
 from compiler.ast import Stmt
 import compiler
 import copy
 import gyp.common
 import multiprocessing
 import optparse
 import os.path
 import re
 import shlex
+import signal
 import subprocess
 import sys
 import threading
 import time
 from gyp.common import GypError
 
 
 # A list of types that are treated as linkable.
 linkable_types = ['executable', 'shared_library', 'loadable_module']
 
@@ skipping 419 matching lines @@
 def CallLoadTargetBuildFile(global_flags,
                             build_file_path, data,
                             aux_data, variables,
                             includes, depth, check):
   """Wrapper around LoadTargetBuildFile for parallel processing.
 
      This wrapper is used when LoadTargetBuildFile is executed in
      a worker process.
   """
 
-  # Apply globals so that the worker process behaves the same.
-  for key, value in global_flags.iteritems():
-    globals()[key] = value
+  try:
+    signal.signal(signal.SIGINT, signal.SIG_IGN)
 
-  # Save the keys so we can return data that changed.
-  data_keys = set(data)
-  aux_data_keys = set(aux_data)
+    # Apply globals so that the worker process behaves the same.
+    for key, value in global_flags.iteritems():
+      globals()[key] = value
 
-  result = LoadTargetBuildFile(build_file_path, data,
-                               aux_data, variables,
-                               includes, depth, check, False)
-  if not result:
-    return result
+    # Save the keys so we can return data that changed.
+    data_keys = set(data)
+    aux_data_keys = set(aux_data)
 
-  (build_file_path, dependencies) = result
+    result = LoadTargetBuildFile(build_file_path, data,
+                                 aux_data, variables,
+                                 includes, depth, check, False)
+    if not result:
+      return result
 
-  data_out = {}
-  for key in data:
-    if key == 'target_build_files':
-      continue
-    if key not in data_keys:
-      data_out[key] = data[key]
-  aux_data_out = {}
-  for key in aux_data:
-    if key not in aux_data_keys:
-      aux_data_out[key] = aux_data[key]
+    (build_file_path, dependencies) = result
 
-  # This gets serialized and sent back to the main process via a pipe.
-  # It's handled in LoadTargetBuildFileCallback.
-  return (build_file_path,
-          data_out,
-          aux_data_out,
-          dependencies)
+    data_out = {}
+    for key in data:
+      if key == 'target_build_files':
+        continue
+      if key not in data_keys:
+        data_out[key] = data[key]
+    aux_data_out = {}
+    for key in aux_data:
+      if key not in aux_data_keys:
+        aux_data_out[key] = aux_data[key]
+
+    # This gets serialized and sent back to the main process via a pipe.
+    # It's handled in LoadTargetBuildFileCallback.
+    return (build_file_path,
+            data_out,
+            aux_data_out,
+            dependencies)
+  except Exception, e:
+    print "Exception: ", e
+    return None
+
+
+class ParallelProcessingError(Exception):
+  pass
 
 
 class ParallelState(object):
   """Class to keep track of state when processing input files in parallel.
 
   If build files are loaded in parallel, use this to keep track of
   state during farming out and processing parallel jobs. It's stored
   in a global so that the callback function can have access to it.
   """
 
   def __init__(self):
     # The multiprocessing pool.
     self.pool = None
     # The condition variable used to protect this object and notify
     # the main loop when there might be more data to process.
     self.condition = None
     # The "data" dict that was passed to LoadTargetBuildFileParallel
     self.data = None
     # The "aux_data" dict that was passed to LoadTargetBuildFileParallel
     self.aux_data = None
     # The number of parallel calls outstanding; decremented when a response
     # was received.
     self.pending = 0
     # The set of all build files that have been scheduled, so we don't
     # schedule the same one twice.
     self.scheduled = set()
     # A list of dependency build file paths that haven't been scheduled yet.
     self.dependencies = []
+    # Flag to indicate if there was an error in a child process.
+    self.error = False
 
   def LoadTargetBuildFileCallback(self, result):
     """Handle the results of running LoadTargetBuildFile in another process.
     """
+    self.condition.acquire()
+    if not result:
+      self.error = True
+      self.condition.notify()
+      self.condition.release()
+      return
     (build_file_path0, data0, aux_data0, dependencies0) = result
-    self.condition.acquire()
     self.data['target_build_files'].add(build_file_path0)
     for key in data0:
       self.data[key] = data0[key]
     for key in aux_data0:
       self.aux_data[key] = aux_data0[key]
     for new_dependency in dependencies0:
       if new_dependency not in self.scheduled:
         self.scheduled.add(new_dependency)
         self.dependencies.append(new_dependency)
     self.pending -= 1
     self.condition.notify()
     self.condition.release()
 
 
 def LoadTargetBuildFileParallel(build_file_path, data, aux_data,
                                 variables, includes, depth, check):
   parallel_state = ParallelState()
   parallel_state.condition = threading.Condition()
   parallel_state.dependencies = [build_file_path]
   parallel_state.scheduled = set([build_file_path])
   parallel_state.pending = 0
   parallel_state.data = data
   parallel_state.aux_data = aux_data
 
-  parallel_state.condition.acquire()
-  while parallel_state.dependencies or parallel_state.pending:
-    if not parallel_state.dependencies:
-      parallel_state.condition.wait()
-      continue
+  try:
+    parallel_state.condition.acquire()
+    while parallel_state.dependencies or parallel_state.pending:
+      if parallel_state.error:
+        break
+      if not parallel_state.dependencies:
+        parallel_state.condition.wait()
+        continue
 
-    dependency = parallel_state.dependencies.pop()
+      dependency = parallel_state.dependencies.pop()
 
-    parallel_state.pending += 1
-    data_in = {}
-    data_in['target_build_files'] = data['target_build_files']
-    aux_data_in = {}
-    global_flags = {
-      'path_sections': globals()['path_sections'],
-      'non_configuration_keys': globals()['non_configuration_keys'],
-      'absolute_build_file_paths': globals()['absolute_build_file_paths'],
-      'multiple_toolsets': globals()['multiple_toolsets']}
+      parallel_state.pending += 1
+      data_in = {}
+      data_in['target_build_files'] = data['target_build_files']
+      aux_data_in = {}
+      global_flags = {
+        'path_sections': globals()['path_sections'],
+        'non_configuration_keys': globals()['non_configuration_keys'],
+        'absolute_build_file_paths': globals()['absolute_build_file_paths'],
+        'multiple_toolsets': globals()['multiple_toolsets']}
 
-    if not parallel_state.pool:
-      parallel_state.pool = multiprocessing.Pool(8)
-    parallel_state.pool.apply_async(
-        CallLoadTargetBuildFile,
-        args = (global_flags, dependency,
-                data_in, aux_data_in,
-                variables, includes, depth, check),
-        callback = parallel_state.LoadTargetBuildFileCallback)
+      if not parallel_state.pool:
+        parallel_state.pool = multiprocessing.Pool(8)
+      parallel_state.pool.apply_async(
+          CallLoadTargetBuildFile,
+          args = (global_flags, dependency,
+                  data_in, aux_data_in,
+                  variables, includes, depth, check),
+          callback = parallel_state.LoadTargetBuildFileCallback)
+  except KeyboardInterrupt, e:
+    parallel_state.pool.terminate()
+    raise e
 
   parallel_state.condition.release()
+  if parallel_state.error:
+    sys.exit()
 
 
 # Look for the bracket that matches the first bracket seen in a
 # string, and return the start and end as a tuple.  For example, if
 # the input is something like "<(foo <(bar)) blah", then it would
 # return (1, 13), indicating the entire string except for the leading
 # "<" and trailing " blah".
 def FindEnclosingBracketGroup(input):
   brackets = { '}': '{',
                ']': '[',
@@ skipping 2040 matching lines @@
     ValidateRunAsInTarget(target, target_dict, build_file)
     ValidateActionsInTarget(target, target_dict, build_file)
 
   # Generators might not expect ints.  Turn them into strs.
   TurnIntIntoStrInDict(data)
 
   # TODO(mark): Return |data| for now because the generator needs a list of
   # build files that came in.  In the future, maybe it should just accept
   # a list, and not the whole data dict.
   return [flat_list, targets, data]