Cache data for included files in the multiprocess load codepath

pylib/gyp/input.py

 # Copyright (c) 2012 Google Inc. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 from compiler.ast import Const
 from compiler.ast import Dict
 from compiler.ast import Discard
 from compiler.ast import List
 from compiler.ast import Module
 from compiler.ast import Node
@@ skipping 344 matching lines @@
   if depth:
     # TODO(dglazkov) The backslash/forward-slash replacement at the end is a
     # temporary measure. This should really be addressed by keeping all paths
     # in POSIX until actual project generation.
     d = gyp.common.RelativePath(depth, os.path.dirname(build_file_path))
     if d == '':
       variables['DEPTH'] = '.'
     else:
       variables['DEPTH'] = d.replace('\\', '/')
 
-  if build_file_path in data['target_build_files']:
-    # Already loaded.
-    return False
-  data['target_build_files'].add(build_file_path)
+  # The 'target_build_files' key is only set when loading target build files in
+  # the non-parallel code path, where LoadTargetBuildFile is called
+  # recursively.  In the parallel code path, we don't need to check whether the
+  # |build_file_path| has already been loaded, because the 'scheduled' set in
+  # ParallelState guarantees that we never load the same |build_file_path|
+  # twice.
+  if 'target_build_files' in data:
+    if build_file_path in data['target_build_files']:
+      # Already loaded.
+      return False
+    data['target_build_files'].add(build_file_path)
 
   gyp.DebugOutput(gyp.DEBUG_INCLUDES,
                   "Loading Target Build File '%s'", build_file_path)
 
   build_file_data = LoadOneBuildFile(build_file_path, data, aux_data,
                                      includes, True, check)
 
   # Store DEPTH for later use in generators.
   build_file_data['_DEPTH'] = depth
 
@@ skipping 70 matching lines @@
       try:
         LoadTargetBuildFile(dependency, data, aux_data, variables,
                             includes, depth, check, load_dependencies)
       except Exception, e:
         gyp.common.ExceptionAppend(
           e, 'while loading dependencies of %s' % build_file_path)
         raise
   else:
     return (build_file_path, dependencies)
 
-
+per_process_data = {}

[scottmg, 2014/12/02 22:50:52]

please move these up to the top

+per_process_aux_data = {}
 def CallLoadTargetBuildFile(global_flags,
-                            build_file_path, data,
-                            aux_data, variables,
+                            build_file_path, variables,
                             includes, depth, check,
                             generator_input_info):
   """Wrapper around LoadTargetBuildFile for parallel processing.
 
      This wrapper is used when LoadTargetBuildFile is executed in
      a worker process.
   """
 

[scottmg, 2014/12/02 22:50:52]

can you add

  global per_process_data
  global per_process_aux_data

here (I know it's not necessary, but just to make it a bit more obvious)

   try:
     signal.signal(signal.SIGINT, signal.SIG_IGN)
 
     # Apply globals so that the worker process behaves the same.
     for key, value in global_flags.iteritems():
       globals()[key] = value
 
-    # Save the keys so we can return data that changed.
-    data_keys = set(data)
-    aux_data_keys = set(aux_data)
-
     SetGeneratorGlobals(generator_input_info)
-    result = LoadTargetBuildFile(build_file_path, data,
-                                 aux_data, variables,
+    result = LoadTargetBuildFile(build_file_path, per_process_data,
+                                 per_process_aux_data, variables,
                                  includes, depth, check, False)
     if not result:
       return result
 
     (build_file_path, dependencies) = result
 
-    data_out = {}
-    for key in data:
-      if key == 'target_build_files':
-        continue
-      if key not in data_keys:
-        data_out[key] = data[key]
-    aux_data_out = {}
-    for key in aux_data:
-      if key not in aux_data_keys:
-        aux_data_out[key] = aux_data[key]
+    # We can safely delete the build_file_data from per_process_data because it
+    # will never be referenced by this process again, so we don't need to keep
+    # it in the cache.
+    build_file_data = per_process_data[build_file_path]

[scottmg, 2014/12/02 22:50:52]

build_file_data = per_process_data.pop(build_file_path)

+    del per_process_data[build_file_path]
 
     # This gets serialized and sent back to the main process via a pipe.
     # It's handled in LoadTargetBuildFileCallback.
     return (build_file_path,
-            data_out,
-            aux_data_out,
+            build_file_data,
             dependencies)
   except GypError, e:
     sys.stderr.write("gyp: %s\n" % e)
     return None
   except Exception, e:
     print >>sys.stderr, 'Exception:', e
     print >>sys.stderr, traceback.format_exc()
     return None
 
 
@@ skipping 10 matching lines @@
   """
 
   def __init__(self):
     # The multiprocessing pool.
     self.pool = None
     # The condition variable used to protect this object and notify
     # the main loop when there might be more data to process.
     self.condition = None
     # The "data" dict that was passed to LoadTargetBuildFileParallel
     self.data = None
-    # The "aux_data" dict that was passed to LoadTargetBuildFileParallel
-    self.aux_data = None
     # The number of parallel calls outstanding; decremented when a response
     # was received.
     self.pending = 0
     # The set of all build files that have been scheduled, so we don't
     # schedule the same one twice.
     self.scheduled = set()
     # A list of dependency build file paths that haven't been scheduled yet.
     self.dependencies = []
     # Flag to indicate if there was an error in a child process.
     self.error = False
 
   def LoadTargetBuildFileCallback(self, result):
     """Handle the results of running LoadTargetBuildFile in another process.
     """
     self.condition.acquire()
     if not result:
       self.error = True
       self.condition.notify()
       self.condition.release()
       return
-    (build_file_path0, data0, aux_data0, dependencies0) = result
+    (build_file_path0, build_file_data0, dependencies0) = result
+    self.data[build_file_path0] = build_file_data0
     self.data['target_build_files'].add(build_file_path0)
-    for key in data0:
-      self.data[key] = data0[key]
-    for key in aux_data0:
-      self.aux_data[key] = aux_data0[key]
     for new_dependency in dependencies0:
       if new_dependency not in self.scheduled:
         self.scheduled.add(new_dependency)
         self.dependencies.append(new_dependency)
     self.pending -= 1
     self.condition.notify()
     self.condition.release()
 
 
 def LoadTargetBuildFilesParallel(build_files, data, aux_data,
@@ skipping 13 matching lines @@
     while parallel_state.dependencies or parallel_state.pending:
       if parallel_state.error:
         break
       if not parallel_state.dependencies:
         parallel_state.condition.wait()
         continue
 
       dependency = parallel_state.dependencies.pop()
 
       parallel_state.pending += 1
-      data_in = {}
-      data_in['target_build_files'] = data['target_build_files']
-      aux_data_in = {}
       global_flags = {
         'path_sections': globals()['path_sections'],
         'non_configuration_keys': globals()['non_configuration_keys'],
         'multiple_toolsets': globals()['multiple_toolsets']}
 
       if not parallel_state.pool:
         parallel_state.pool = multiprocessing.Pool(multiprocessing.cpu_count())
       parallel_state.pool.apply_async(
           CallLoadTargetBuildFile,
           args = (global_flags, dependency,
-                  data_in, aux_data_in,
                   variables, includes, depth, check, generator_input_info),
           callback = parallel_state.LoadTargetBuildFileCallback)
   except KeyboardInterrupt, e:
     parallel_state.pool.terminate()
     raise e
 
   parallel_state.condition.release()
 
   parallel_state.pool.close()
   parallel_state.pool.join()
@@ skipping 2235 matching lines @@
     ValidateRunAsInTarget(target, target_dict, build_file)
     ValidateActionsInTarget(target, target_dict, build_file)
 
   # Generators might not expect ints.  Turn them into strs.
   TurnIntIntoStrInDict(data)
 
   # TODO(mark): Return |data| for now because the generator needs a list of
   # build files that came in.  In the future, maybe it should just accept
   # a list, and not the whole data dict.
   return [flat_list, targets, data]