| OLD | NEW |
| 1 # Copyright 2014 The Chromium Authors. All rights reserved. | 1 # Copyright 2014 The Chromium Authors. All rights reserved. |
| 2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
| 3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
| 4 | 4 |
| 5 # This script computs the number of concurrent links we want to run in the build | 5 # This script computs the number of concurrent links we want to run in the build |
| 6 # as a function of machine spec. It's based on GetDefaultConcurrentLinks in GYP. | 6 # as a function of machine spec. It's based on GetDefaultConcurrentLinks in GYP. |
| 7 | 7 |
| 8 import os | 8 import os |
| 9 import re | 9 import re |
| 10 import subprocess |
| 10 import sys | 11 import sys |
| 11 | 12 |
| 12 def GetDefaultConcurrentLinks(): | 13 def GetDefaultConcurrentLinks(): |
| 13 # Inherit the legacy environment variable for people that have set it in GYP. | 14 # Inherit the legacy environment variable for people that have set it in GYP. |
| 14 pool_size = int(os.getenv('GYP_LINK_CONCURRENCY', 0)) | 15 pool_size = int(os.getenv('GYP_LINK_CONCURRENCY', 0)) |
| 15 if pool_size: | 16 if pool_size: |
| 16 return pool_size | 17 return pool_size |
| 17 | 18 |
| 18 if sys.platform in ('win32', 'cygwin'): | 19 if sys.platform in ('win32', 'cygwin'): |
| 19 import ctypes | 20 import ctypes |
| 20 | 21 |
| 21 class MEMORYSTATUSEX(ctypes.Structure): | 22 class MEMORYSTATUSEX(ctypes.Structure): |
| 22 _fields_ = [ | 23 _fields_ = [ |
| 23 ("dwLength", ctypes.c_ulong), | 24 ("dwLength", ctypes.c_ulong), |
| 24 ("dwMemoryLoad", ctypes.c_ulong), | 25 ("dwMemoryLoad", ctypes.c_ulong), |
| 25 ("ullTotalPhys", ctypes.c_ulonglong), | 26 ("ullTotalPhys", ctypes.c_ulonglong), |
| 26 ("ullAvailPhys", ctypes.c_ulonglong), | 27 ("ullAvailPhys", ctypes.c_ulonglong), |
| 27 ("ullTotalPageFile", ctypes.c_ulonglong), | 28 ("ullTotalPageFile", ctypes.c_ulonglong), |
| 28 ("ullAvailPageFile", ctypes.c_ulonglong), | 29 ("ullAvailPageFile", ctypes.c_ulonglong), |
| 29 ("ullTotalVirtual", ctypes.c_ulonglong), | 30 ("ullTotalVirtual", ctypes.c_ulonglong), |
| 30 ("ullAvailVirtual", ctypes.c_ulonglong), | 31 ("ullAvailVirtual", ctypes.c_ulonglong), |
| 31 ("sullAvailExtendedVirtual", ctypes.c_ulonglong), | 32 ("sullAvailExtendedVirtual", ctypes.c_ulonglong), |
| 32 ] | 33 ] |
| 33 | 34 |
| 34 stat = MEMORYSTATUSEX() | 35 stat = MEMORYSTATUSEX(dwLength=ctypes.sizeof(MEMORYSTATUSEX)) |
| 35 stat.dwLength = ctypes.sizeof(stat) | |
| 36 ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(stat)) | 36 ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(stat)) |
| 37 | 37 |
| 38 mem_limit = max(1, stat.ullTotalPhys / (4 * (2 ** 30))) # total / 4GB | 38 mem_limit = max(1, stat.ullTotalPhys / (4 * (2 ** 30))) # total / 4GB |
| 39 hard_cap = max(1, int(os.getenv('GYP_LINK_CONCURRENCY_MAX', 2**32))) | 39 hard_cap = max(1, int(os.getenv('GYP_LINK_CONCURRENCY_MAX', 2**32))) |
| 40 return min(mem_limit, hard_cap) | 40 return min(mem_limit, hard_cap) |
| 41 elif sys.platform.startswith('linux'): | 41 elif sys.platform.startswith('linux'): |
| 42 if os.path.exists("/proc/meminfo"): | 42 if os.path.exists("/proc/meminfo"): |
| 43 with open("/proc/meminfo") as meminfo: | 43 with open("/proc/meminfo") as meminfo: |
| 44 memtotal_re = re.compile(r'^MemTotal:\s*(\d*)\s*kB') | 44 memtotal_re = re.compile(r'^MemTotal:\s*(\d*)\s*kB') |
| 45 for line in meminfo: | 45 for line in meminfo: |
| 46 match = memtotal_re.match(line) | 46 match = memtotal_re.match(line) |
| 47 if not match: | 47 if not match: |
| 48 continue | 48 continue |
| 49 # Allow 8Gb per link on Linux because Gold is quite memory hungry | 49 # Allow 8Gb per link on Linux because Gold is quite memory hungry |
| 50 return max(1, int(match.group(1)) / (8 * (2 ** 20))) | 50 return max(1, int(match.group(1)) / (8 * (2 ** 20))) |
| 51 return 1 | 51 return 1 |
| 52 elif sys.platform == 'darwin': | 52 elif sys.platform == 'darwin': |
| 53 try: | 53 try: |
| 54 avail_bytes = int(subprocess.check_output(['sysctl', '-n', 'hw.memsize'])) | 54 avail_bytes = int(subprocess.check_output(['sysctl', '-n', 'hw.memsize'])) |
| 55 # A static library debug build of Chromium's unit_tests takes ~2.7GB, so | 55 # A static library debug build of Chromium's unit_tests takes ~2.7GB, so |
| 56 # 4GB per ld process allows for some more bloat. | 56 # 4GB per ld process allows for some more bloat. |
| 57 return max(1, avail_bytes / (4 * (2 ** 30))) # total / 4GB | 57 return max(1, avail_bytes / (4 * (2 ** 30))) # total / 4GB |
| 58 except: | 58 except Exception: |
| 59 return 1 | 59 return 1 |
| 60 else: | 60 else: |
| 61 # TODO(scottmg): Implement this for other platforms. | 61 # TODO(scottmg): Implement this for other platforms. |
| 62 return 1 | 62 return 1 |
| 63 | 63 |
| 64 print GetDefaultConcurrentLinks() | 64 print GetDefaultConcurrentLinks() |
| OLD | NEW |