OLD | NEW |
---|---|
1 # Copyright (c) 2013 Google Inc. All rights reserved. | 1 # Copyright (c) 2013 Google Inc. All rights reserved. |
2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
4 | 4 |
5 import copy | 5 import copy |
6 import hashlib | 6 import hashlib |
7 import multiprocessing | 7 import multiprocessing |
8 import os.path | 8 import os.path |
9 import re | 9 import re |
10 import signal | 10 import signal |
(...skipping 1488 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
1499 ("ullAvailPageFile", ctypes.c_ulonglong), | 1499 ("ullAvailPageFile", ctypes.c_ulonglong), |
1500 ("ullTotalVirtual", ctypes.c_ulonglong), | 1500 ("ullTotalVirtual", ctypes.c_ulonglong), |
1501 ("ullAvailVirtual", ctypes.c_ulonglong), | 1501 ("ullAvailVirtual", ctypes.c_ulonglong), |
1502 ("sullAvailExtendedVirtual", ctypes.c_ulonglong), | 1502 ("sullAvailExtendedVirtual", ctypes.c_ulonglong), |
1503 ] | 1503 ] |
1504 | 1504 |
1505 stat = MEMORYSTATUSEX() | 1505 stat = MEMORYSTATUSEX() |
1506 stat.dwLength = ctypes.sizeof(stat) | 1506 stat.dwLength = ctypes.sizeof(stat) |
1507 ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(stat)) | 1507 ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(stat)) |
1508 | 1508 |
1509 return max(1, stat.ullTotalPhys / (4 * (2 ** 30))) # total / 4GB | 1509 mem_limit = max(1, stat.ullTotalPhys / (4 * (2 ** 30))) # total / 4GB |
1510 cpu_limit = multiprocessing.cpu_count() | |
scottmg
2013/09/11 20:09:06
that does
if sys.platform == 'win32':
| |
1511 return min(mem_limit, cpu_limit) | |
1510 elif sys.platform.startswith('linux'): | 1512 elif sys.platform.startswith('linux'): |
1511 with open("/proc/meminfo") as meminfo: | 1513 with open("/proc/meminfo") as meminfo: |
1512 memtotal_re = re.compile(r'^MemTotal:\s*(\d*)\s*kB') | 1514 memtotal_re = re.compile(r'^MemTotal:\s*(\d*)\s*kB') |
1513 for line in meminfo: | 1515 for line in meminfo: |
1514 match = memtotal_re.match(line) | 1516 match = memtotal_re.match(line) |
1515 if not match: | 1517 if not match: |
1516 continue | 1518 continue |
1517 # Allow 8Gb per link on Linux because Gold is quite memory hungry | 1519 # Allow 8Gb per link on Linux because Gold is quite memory hungry |
1518 return max(1, int(match.group(1)) / (8 * (2 ** 20))) | 1520 return max(1, int(match.group(1)) / (8 * (2 ** 20))) |
1519 return 1 | 1521 return 1 |
(...skipping 582 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
2102 arglists.append( | 2104 arglists.append( |
2103 (target_list, target_dicts, data, params, config_name)) | 2105 (target_list, target_dicts, data, params, config_name)) |
2104 pool.map(CallGenerateOutputForConfig, arglists) | 2106 pool.map(CallGenerateOutputForConfig, arglists) |
2105 except KeyboardInterrupt, e: | 2107 except KeyboardInterrupt, e: |
2106 pool.terminate() | 2108 pool.terminate() |
2107 raise e | 2109 raise e |
2108 else: | 2110 else: |
2109 for config_name in config_names: | 2111 for config_name in config_names: |
2110 GenerateOutputForConfig(target_list, target_dicts, data, params, | 2112 GenerateOutputForConfig(target_list, target_dicts, data, params, |
2111 config_name) | 2113 config_name) |
OLD | NEW |