Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(564)

Side by Side Diff: build/toolchain/get_concurrent_links.py

Issue 1509733004: Give more RAM per ld process. (Closed) Base URL: https://chromium.googlesource.com/chromium/src.git@master
Patch Set: reserve 20 GB for spikes Created 5 years ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
« no previous file with comments | « no previous file | no next file » | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 # Copyright 2014 The Chromium Authors. All rights reserved. 1 # Copyright 2014 The Chromium Authors. All rights reserved.
2 # Use of this source code is governed by a BSD-style license that can be 2 # Use of this source code is governed by a BSD-style license that can be
3 # found in the LICENSE file. 3 # found in the LICENSE file.
4 4
5 # This script computs the number of concurrent links we want to run in the build 5 # This script computs the number of concurrent links we want to run in the build
6 # as a function of machine spec. It's based on GetDefaultConcurrentLinks in GYP. 6 # as a function of machine spec. It's based on GetDefaultConcurrentLinks in GYP.
7 7
8 import optparse 8 import optparse
9 import os 9 import os
10 import re 10 import re
(...skipping 31 matching lines...) Expand 10 before | Expand all | Expand 10 after
42 hard_cap = max(1, int(os.getenv('GYP_LINK_CONCURRENCY_MAX', 2**32))) 42 hard_cap = max(1, int(os.getenv('GYP_LINK_CONCURRENCY_MAX', 2**32)))
43 return min(mem_limit, hard_cap) 43 return min(mem_limit, hard_cap)
44 elif sys.platform.startswith('linux'): 44 elif sys.platform.startswith('linux'):
45 if os.path.exists("/proc/meminfo"): 45 if os.path.exists("/proc/meminfo"):
46 with open("/proc/meminfo") as meminfo: 46 with open("/proc/meminfo") as meminfo:
47 memtotal_re = re.compile(r'^MemTotal:\s*(\d*)\s*kB') 47 memtotal_re = re.compile(r'^MemTotal:\s*(\d*)\s*kB')
48 for line in meminfo: 48 for line in meminfo:
49 match = memtotal_re.match(line) 49 match = memtotal_re.match(line)
50 if not match: 50 if not match:
51 continue 51 continue
52 mem_total_gb = float(match.group(1)) / (2 ** 20)
52 # Allow 8Gb per link on Linux because Gold is quite memory hungry 53 # Allow 8Gb per link on Linux because Gold is quite memory hungry
53 # For LTO builds the RAM requirements are even higher 54 mem_per_link_gb = 8
54 # Note: it's 15 GB for LTO build to make sure we get 4 link jobs 55 if is_lto:
55 # for 64 GB, even if the system reports a couple of GBs less. 56 mem_total_gb -= 20 # Reserve
56 ram_per_link_gb = 15 if is_lto else 8 57 # For LTO builds the RAM requirements are even higher
57 return max(1, int(match.group(1)) / (ram_per_link_gb * (2 ** 20))) 58 mem_per_link_gb = 20
59 return int(max(1, mem_total_gb / mem_per_link_gb))
58 return 1 60 return 1
59 elif sys.platform == 'darwin': 61 elif sys.platform == 'darwin':
60 try: 62 try:
61 avail_bytes = int(subprocess.check_output(['sysctl', '-n', 'hw.memsize'])) 63 avail_bytes = int(subprocess.check_output(['sysctl', '-n', 'hw.memsize']))
62 # A static library debug build of Chromium's unit_tests takes ~2.7GB, so 64 # A static library debug build of Chromium's unit_tests takes ~2.7GB, so
63 # 4GB per ld process allows for some more bloat. 65 # 4GB per ld process allows for some more bloat.
64 return max(1, avail_bytes / (4 * (2 ** 30))) # total / 4GB 66 return max(1, avail_bytes / (4 * (2 ** 30))) # total / 4GB
65 except Exception: 67 except Exception:
66 return 1 68 return 1
67 else: 69 else:
68 # TODO(scottmg): Implement this for other platforms. 70 # TODO(scottmg): Implement this for other platforms.
69 return 1 71 return 1
70 72
71 def main(): 73 def main():
72 parser = optparse.OptionParser() 74 parser = optparse.OptionParser()
73 parser.add_option('--lto', action="store_true", default=False, 75 parser.add_option('--lto', action="store_true", default=False,
74 help='This is an LTO build with higher memory requirements') 76 help='This is an LTO build with higher memory requirements')
75 parser.disable_interspersed_args() 77 parser.disable_interspersed_args()
76 options, args = parser.parse_args() 78 options, args = parser.parse_args()
77 79
78 print _GetDefaultConcurrentLinks(is_lto=options.lto) 80 print _GetDefaultConcurrentLinks(is_lto=options.lto)
79 return 0 81 return 0
80 82
81 if __name__ == '__main__': 83 if __name__ == '__main__':
82 sys.exit(main()) 84 sys.exit(main())
OLDNEW
« no previous file with comments | « no previous file | no next file » | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698