get_concurrent_links.py: give more RAM per job in LTO builds.

build/toolchain/get_concurrent_links.py

 # Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 # This script computs the number of concurrent links we want to run in the build
 # as a function of machine spec. It's based on GetDefaultConcurrentLinks in GYP.
 
+import optparse
 import os
 import re
 import subprocess
 import sys
 
-def GetDefaultConcurrentLinks():
+def GetDefaultConcurrentLinks(is_lto=False):

[Dirk Pranke, 2015/12/03 23:19:08]

nit: don't specify a default argument, since it'll always be passed in. I might
also rename it to _GetDefault to make it a bit clearer that the function is
private to the file.

[krasin, 2015/12/03 23:22:20]

Done.

   # Inherit the legacy environment variable for people that have set it in GYP.
   pool_size = int(os.getenv('GYP_LINK_CONCURRENCY', 0))
   if pool_size:
     return pool_size
 
   if sys.platform in ('win32', 'cygwin'):
     import ctypes
 
     class MEMORYSTATUSEX(ctypes.Structure):
       _fields_ = [
@@ skipping 18 matching lines @@
     return min(mem_limit, hard_cap)
   elif sys.platform.startswith('linux'):
     if os.path.exists("/proc/meminfo"):
       with open("/proc/meminfo") as meminfo:
         memtotal_re = re.compile(r'^MemTotal:\s*(\d*)\s*kB')
         for line in meminfo:
           match = memtotal_re.match(line)
           if not match:
             continue
           # Allow 8Gb per link on Linux because Gold is quite memory hungry
-          return max(1, int(match.group(1)) / (8 * (2 ** 20)))
+          # For LTO builds the RAM requirements are even higher
+          # Note: it's 15 GB for LTO build to make sure we get 4 link jobs
+          # for 64 GB, even if the system reports a couple of GBs less.
+          ram_per_link_gb = 15 if is_lto else 8
+          return max(1, int(match.group(1)) / (ram_per_link_gb * (2 ** 20)))
     return 1
   elif sys.platform == 'darwin':
     try:
       avail_bytes = int(subprocess.check_output(['sysctl', '-n', 'hw.memsize']))
       # A static library debug build of Chromium's unit_tests takes ~2.7GB, so
       # 4GB per ld process allows for some more bloat.
       return max(1, avail_bytes / (4 * (2 ** 30)))  # total / 4GB
     except Exception:
       return 1
   else:
     # TODO(scottmg): Implement this for other platforms.
     return 1
 
-print GetDefaultConcurrentLinks()
+def main():
+  parser = optparse.OptionParser()
+  parser.add_option('--lto', action="store_true", default=False,
+                    help='This is an LTO build with higher memory requirements')
+  parser.disable_interspersed_args()
+  options, args = parser.parse_args()
+
+  print GetDefaultConcurrentLinks(is_lto=options.lto)
+  return 0
+
+if __name__ == '__main__':
+  sys.exit(main())