Load target build files in parallel using Python multiprocessing.

pylib/gyp/__init__.py

 #!/usr/bin/env python
 
 # Copyright (c) 2012 Google Inc. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import copy
 import gyp.input
 import optparse
 import os.path
@@ skipping 34 matching lines @@
 
 
 class GypError(Exception):
   """Error class representing an error, which is to be presented
   to the user.  The main entry point will catch and display this.
   """
   pass
 
 
 def Load(build_files, format, default_variables={},
-         includes=[], depth='.', params=None, check=False, circular_check=True):
+         includes=[], depth='.', params=None, check=False,
+         circular_check=True, parallel=False):
   """
   Loads one or more specified build files.
   default_variables and includes will be copied before use.
   Returns the generator for the specified format and the
   data returned by loading the specified build files.
   """
   if params is None:
     params = {}
 
   flavor = None
@@ skipping 57 matching lines @@
         getattr(generator, 'generator_supports_multiple_toolsets', False),
     'generator_wants_static_library_dependencies_adjusted':
         getattr(generator,
                 'generator_wants_static_library_dependencies_adjusted', True),
     'generator_wants_sorted_dependencies':
         getattr(generator, 'generator_wants_sorted_dependencies', False),
   }
 
   # Process the input specific to this generator.
   result = gyp.input.Load(build_files, default_variables, includes[:],
-                          depth, generator_input_info, check, circular_check)
+                          depth, generator_input_info, check, circular_check,
+                          parallel)
   return [generator] + result
 
 def NameValueListToDict(name_value_list):
   """
   Takes an array of strings of the form 'NAME=VALUE' and creates a dictionary
   of the pairs.  If a string is simply NAME, then the value in the dictionary
   is set to True.  If VALUE can be converted to an integer, it is.
   """
   result = { }
   for item in name_value_list:
@@ skipping 166 matching lines @@
                     help='sets generator flag FLAG to VAL')
   parser.add_option('--generator-output', dest='generator_output',
                     action='store', default=None, metavar='DIR', type='path',
                     env_name='GYP_GENERATOR_OUTPUT',
                     help='puts generated build files under DIR')
   parser.add_option('--ignore-environment', dest='use_environment',
                     action='store_false', default=True, regenerate=False,
                     help='do not read options from environment variables')
   parser.add_option('--check', dest='check', action='store_true',
                     help='check format of gyp files')
+  parser.add_option('--parallel', dest='parallel', action='store_true',

[M-A Ruel, 2012/09/18 19:42:18]

Remove parameter dest='parallel'
it's not needed.

[dmazzoni, 2012/09/19 21:14:52]

Done.

+                    env_name='GYP_PARALLEL',
+                    regenerate=False,

[M-A Ruel, 2012/09/18 19:42:18]

What's regenerate?

[dmazzoni, 2012/09/19 21:14:52]

Looks like this isn't needed if 'dest' isn't used.

+                    help='Use multiprocessing for speed (experimental)')
   parser.add_option('--toplevel-dir', dest='toplevel_dir', action='store',
                     default=None, metavar='DIR', type='path',
                     help='directory to use as the root of the source tree')
   # --no-circular-check disables the check for circular relationships between
   # .gyp files.  These relationships should not exist, but they've only been
   # observed to be harmful with the Xcode generator.  Chromium's .gyp files
   # currently have some circular relationships on non-Mac platforms, so this
   # option allows the strict behavior to be used on Macs and the lenient
   # behavior to be used elsewhere.
   # TODO(mark): Remove this option when http://crbug.com/35878 is fixed.
@@ skipping 37 matching lines @@
       elif sys.platform in ('win32', 'cygwin'):
         options.formats = ['msvs']
       else:
         options.formats = ['make']
 
   if not options.generator_output and options.use_environment:
     g_o = os.environ.get('GYP_GENERATOR_OUTPUT')
     if g_o:
       options.generator_output = g_o
 
+  if not options.parallel and options.use_environment:
+    parallel = os.environ.get('GYP_PARALLEL')

[M-A Ruel, 2012/09/18 19:42:18]

options.parallel = bool(os.environ.get('GYP_PARALLEL'))

[dmazzoni, 2012/09/19 21:14:52]

Done.

+    if parallel:
+      options.parallel = True
+
   for mode in options.debug:
     gyp.debug[mode] = 1
 
   # Do an extra check to avoid work when we're not debugging.
   if DEBUG_GENERAL in gyp.debug.keys():
     DebugOutput(DEBUG_GENERAL, 'running with these options:')
     for option, value in sorted(options.__dict__.items()):
       if option[0] == '_':
         continue
       if isinstance(value, basestring):
@@ skipping 97 matching lines @@
               'cwd': os.getcwd(),
               'build_files_arg': build_files_arg,
               'gyp_binary': sys.argv[0],
               'home_dot_gyp': home_dot_gyp}
 
     # Start with the default variables from the command line.
     [generator, flat_list, targets, data] = Load(build_files, format,
                                                  cmdline_default_variables,
                                                  includes, options.depth,
                                                  params, options.check,
-                                                 options.circular_check)
+                                                 options.circular_check,
+                                                 options.parallel)
 
     # TODO(mark): Pass |data| for now because the generator needs a list of
     # build files that came in.  In the future, maybe it should just accept
     # a list, and not the whole data dict.
     # NOTE: flat_list is the flattened dependency graph specifying the order
     # that targets may be built.  Build systems that operate serially or that
     # need to have dependencies defined before dependents reference them should
     # generate targets in the order specified in flat_list.
     generator.GenerateOutput(flat_list, targets, data, params)
 
   # Done
   return 0
 
 
 def main(args):
   try:
     return gyp_main(args)
   except GypError, e:
     sys.stderr.write("gyp: %s\n" % e)
     return 1
 
 if __name__ == '__main__':
   sys.exit(main(sys.argv[1:]))