Instrumented libraries: improve parallel build

third_party/instrumented_libraries/scripts/download_build_install.py

 #!/usr/bin/python
 # Copyright 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Downloads, builds (with instrumentation) and installs shared libraries."""
 
 import argparse
 import ast
+import fcntl
 import os
 import platform
 import re
 import shlex
 import shutil
 import subprocess
 import sys
 
 SCRIPT_ABSOLUTE_PATH = os.path.dirname(os.path.abspath(__file__))
 
@@ skipping 22 matching lines @@
   return os.path.realpath(os.path.join(SCRIPT_ABSOLUTE_PATH, '..',
                                        path_relative_to_gyp))
 
 
 class InstrumentedPackageBuilder(object):
   """Checks out and builds a single instrumented package."""
   def __init__(self, args, clobber):
     self._cc = args.cc
     self._cxx = args.cxx
     self._extra_configure_flags = unescape_flags(args.extra_configure_flags)
-    self._jobs = args.jobs
     self._libdir = args.libdir
     self._package = args.package
     self._patch = real_path(args.patch) if args.patch else None
     self._pre_build = \
         real_path(args.pre_build) if args.pre_build else None
     self._sanitizer = args.sanitizer
     self._verbose = args.verbose
     self._clobber = clobber
     self._working_dir = os.path.join(
         real_path(args.intermediate_dir), self._package, '')
@@ skipping 28 matching lines @@
     self._build_env['LDFLAGS'] = self._ldflags
 
     if self._sanitizer == 'asan':
       # Do not report leaks during the build process.
       self._build_env['ASAN_OPTIONS'] = \
           '%s:detect_leaks=0' % self._build_env.get('ASAN_OPTIONS', '')
 
     # libappindicator1 needs this.
     self._build_env['CSC'] = '/usr/bin/mono-csc'
 
-  def shell_call(self, command, env=None, cwd=None):
+  def shell_call(self, command, env=None, cwd=None, ignore_ret_code=False):
     """Wrapper around subprocess.Popen().
 
     Calls command with specific environment and verbosity using
     subprocess.Popen().
     """
     child = subprocess.Popen(
         command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
         env=env, shell=True, cwd=cwd)
     stdout, stderr = child.communicate()
+    if ignore_ret_code:
+      if self._verbose:
+        print stdout
+      return
     if self._verbose or child.returncode:
       print stdout
     if child.returncode:
       raise Exception('Failed to run: %s' % command)
 
   def maybe_download_source(self):
     """Checks out the source code (if needed).
 
     Checks out the source code for the package, if required (i.e. unless running
     in no-clobber mode). Initializes self._source_dir and self._source_archives.
     """
     get_fresh_source = self._clobber or not os.path.exists(self._working_dir)
     if get_fresh_source:
       shutil.rmtree(self._working_dir, ignore_errors=True)
       os.makedirs(self._working_dir)
+
+      # Download one source package at a time, otherwise, there will
+      # be connection errors in gnutls_handshake().
+      self.shell_call('touch apt-source-lock')

[Evgeniy Stepanov, 2017/03/07 22:42:02]

Why not open('apt-source-lock', 'w') ?

[Tom (Use chromium acct), 2017/03/08 00:24:58]

Done.

+      lock = open('apt-source-lock')
+      fcntl.flock(lock, fcntl.LOCK_EX)
       self.shell_call('apt-get source %s' % self._package,
                       cwd=self._working_dir)
+      fcntl.flock(lock, fcntl.LOCK_UN)
 
     (dirpath, dirnames, filenames) = os.walk(self._working_dir).next()
 
     if len(dirnames) != 1:
       raise Exception(
           '`apt-get source %s\' must create exactly one subdirectory.'
               % self._package)
     self._source_dir = os.path.join(dirpath, dirnames[0], '')
 
     if len(filenames) == 0:
@@ skipping 70 matching lines @@
 
   def cleanup_after_install(self):
     """Removes unneeded files in self.temp_libdir()."""
     # .la files are not needed, nuke them.
     # In case --no-static is not supported, nuke any static libraries we built.
     self.shell_call(
         'find %s -name *.la -or -name *.a | xargs rm -f' % self.temp_libdir())
     # .pc files are not needed.
     self.shell_call('rm %s/pkgconfig -rf' % self.temp_libdir())
 
-  def make(self, args, jobs=None, env=None, cwd=None):
+  def make(self, args, env=None, cwd=None, ignore_ret_code=False):
     """Invokes `make'.
 
     Invokes `make' with the specified args, using self._build_env and
     self._source_dir by default.
     """
-    if jobs is None:
-      jobs = self._jobs
     if cwd is None:
       cwd = self._source_dir
     if env is None:
       env = self._build_env
-    cmd = ['make', '-j%s' % jobs] + args
-    self.shell_call(' '.join(cmd), env=env, cwd=cwd)
+    cmd = ['make'] + args
+    self.shell_call(' '.join(cmd), env=env, cwd=cwd,
+                    ignore_ret_code=ignore_ret_code)
 
   def make_install(self, args, **kwargs):
     """Invokes `make install'."""
     self.make(['install'] + args, **kwargs)
 
   def build_and_install(self):
     """Builds and installs the DSOs.
 
     Builds the package with ./configure + make, installs it to a temporary
-    location, then moves the relevant files to their permanent location. 
+    location, then moves the relevant files to their permanent location.
     """
     configure_cmd = './configure --libdir=/%s/ %s' % (
         self._libdir, self._extra_configure_flags)
     self.shell_call(configure_cmd, env=self._build_env, cwd=self._source_dir)
 
     # Some makefiles use BUILDROOT or INSTALL_ROOT instead of DESTDIR.
     args = ['DESTDIR', 'BUILDROOT', 'INSTALL_ROOT']
     make_args = ['%s=%s' % (name, self.temp_dir()) for name in args]
     self.make(make_args)
 
-    # Some packages don't support parallel install. Use -j1 always.
-    self.make_install(make_args, jobs=1)
+    self.make_install(make_args)
 
     self.cleanup_after_install()
 
     self.fix_rpaths(self.temp_libdir())
 
     # Now move the contents of the temporary destdir to their final place.
     # We only care for the contents of LIBDIR.
     self.shell_call('cp %s/* %s/ -rdf' % (self.temp_libdir(),
                                           self.dest_libdir()))
 
@@ skipping 100 matching lines @@
       make_args.append('USE_64=1')
 
     # Make sure we don't override the default flags in the makefile.
     for variable in ['CFLAGS', 'CXXFLAGS', 'LDFLAGS']:
       del self._build_env[variable]
 
     # Hardcoded paths.
     temp_dir = os.path.join(self._source_dir, 'nss')
     temp_libdir = os.path.join(temp_dir, 'lib')
 
-    # Parallel build is not supported. Also, the build happens in
-    # <source_dir>/nss.
-    try:
-      self.make(make_args, jobs=1, cwd=temp_dir)
-    except Exception:
-      # Building fails after all the required DSOs have been built, so ignore
-      # the error.
-      pass
+    # The build happens in <source_dir>/nss.  Building fails after all
+    # the required DSOs have been built, so ignore the error.
+    self.make(make_args, cwd=temp_dir, ignore_ret_code=True)
 
     self.fix_rpaths(temp_libdir)
 
     # 'make install' is not supported. Copy the DSOs manually.
     for (dirpath, dirnames, filenames) in os.walk(temp_libdir):
       for filename in filenames:
         if filename.endswith('.so'):
           full_path = os.path.join(dirpath, filename)
           if self._verbose:
             print 'download_build_install.py: installing %s' % full_path
           shutil.copy(full_path, self.dest_libdir())
 
 
 class StubBuilder(InstrumentedPackageBuilder):
   def download_build_install(self):
     self._touch(os.path.join(self._destdir, '%s.txt' % self._package))
+    self.shell_call('mkdir -p %s' % self.dest_libdir())
     self._touch(os.path.join(self.dest_libdir(), '%s.so.0' % self._package))
 
   def _touch(self, path):
     with open(path, 'w'):
       pass
 
 
 def main():
   parser = argparse.ArgumentParser(
       description='Download, build and install an instrumented package.')
 
-  parser.add_argument('-j', '--jobs', type=int, default=1)
   parser.add_argument('-p', '--package', required=True)
   parser.add_argument(
       '-i', '--product-dir', default='.',
       help='Relative path to the directory with chrome binaries')
   parser.add_argument(
       '-m', '--intermediate-dir', default='.',
       help='Relative path to the directory for temporary build files')
   parser.add_argument('--extra-configure-flags', default='')
   parser.add_argument('--cflags', default='')
   parser.add_argument('--ldflags', default='')
@@ skipping 31 matching lines @@
     builder = Libpci3Builder(args, clobber)
   elif args.build_method == 'stub':
     builder = StubBuilder(args, clobber)
   else:
     raise Exception('Unrecognized build method: %s' % args.build_method)
 
   builder.download_build_install()
 
 if __name__ == '__main__':
   main()