| OLD | NEW |
|---|
| (Empty) |
| 1 #!/usr/bin/env python | |
| 2 # Copyright 2017 The Chromium Authors. All rights reserved. | |
| 3 # Use of this source code is governed by a BSD-style license that can be | |
| 4 # found in the LICENSE file. | |
| 5 | |
| 6 """Merge the PGC files generated during the profiling step to the PGD database. | |
| 7 | |
| 8 This is required to workaround a flakyness in pgomgr.exe where it can run out | |
| 9 of address space while trying to merge all the PGC files at the same time. | |
| 10 """ | |
| 11 | |
| 12 import glob | |
| 13 import json | |
| 14 import optparse | |
| 15 import os | |
| 16 import subprocess | |
| 17 import sys | |
| 18 | |
| 19 | |
| 20 def find_pgomgr(chrome_checkout_dir, target_cpu): | |
| 21 """Find pgomgr.exe.""" | |
| 22 if target_cpu not in ('x86', 'x64'): | |
| 23 raise Exception('target_cpu should be x86 or x64.') | |
| 24 | |
| 25 win_toolchain_json_file = os.path.join(chrome_checkout_dir, 'build', | |
| 26 'win_toolchain.json') | |
| 27 if not os.path.exists(win_toolchain_json_file): | |
| 28 raise Exception('The toolchain JSON file is missing.') | |
| 29 with open(win_toolchain_json_file) as temp_f: | |
| 30 toolchain_data = json.load(temp_f) | |
| 31 if not os.path.isdir(toolchain_data['path']): | |
| 32 raise Exception('The toolchain JSON file is invalid.') | |
| 33 | |
| 34 pgomgr_dir = os.path.join(toolchain_data['path'], 'VC', 'bin') | |
| 35 if target_cpu == 'x64': | |
| 36 pgomgr_dir = os.path.join(pgomgr_dir, 'amd64') | |
| 37 | |
| 38 pgomgr_path = os.path.join(pgomgr_dir, 'pgomgr.exe') | |
| 39 if not os.path.exists(pgomgr_path): | |
| 40 raise Exception('pgomgr.exe is missing from %s.' % pgomgr_dir) | |
| 41 | |
| 42 return pgomgr_path | |
| 43 | |
| 44 | |
| 45 def main(): | |
| 46 parser = optparse.OptionParser(usage='%prog [options]') | |
| 47 parser.add_option('--checkout-dir', help='The Chrome checkout directory.') | |
| 48 parser.add_option('--target-cpu', help='The target\'s bitness.') | |
| 49 parser.add_option('--build-dir', help='Chrome build directory.') | |
| 50 parser.add_option('--binary-name', help='The binary for which the PGC files ' | |
| 51 'should be merged, without extension.') | |
| 52 options, _ = parser.parse_args() | |
| 53 | |
| 54 if not options.checkout_dir: | |
| 55 parser.error('--checkout-dir is required') | |
| 56 if not options.target_cpu: | |
| 57 parser.error('--target-cpu is required') | |
| 58 if not options.build_dir: | |
| 59 parser.error('--build-dir is required') | |
| 60 if not options.binary_name: | |
| 61 parser.error('--binary-name is required') | |
| 62 | |
| 63 # Starts by finding pgomgr.exe. | |
| 64 pgomgr_path = find_pgomgr(options.checkout_dir, options.target_cpu) | |
| 65 | |
| 66 pgc_files = glob.glob(os.path.join(options.build_dir, | |
| 67 '%s*.pgc' % options.binary_name)) | |
| 68 | |
| 69 # Number of PGC files that should be merged in each iterations, merging all | |
| 70 # the files one by one is really slow but merging more to 10 at a time doesn't | |
| 71 # really seem to impact the total time. | |
| 72 # | |
| 73 # Number of pgc merged per iteration | Time (in min) | |
| 74 # 1 | 27.2 | |
| 75 # 10 | 12.8 | |
| 76 # 20 | 12.0 | |
| 77 # 30 | 11.5 | |
| 78 # 40 | 11.4 | |
| 79 # 50 | 11.5 | |
| 80 # 60 | 11.6 | |
| 81 # 70 | 11.6 | |
| 82 # 80 | 11.7 | |
| 83 # | |
| 84 # TODO(sebmarchand): Measure the memory usage of pgomgr.exe to see how it get | |
| 85 # affected by the number of pgc files. | |
| 86 pgc_per_iter = 20 | |
| 87 | |
| 88 def _split_in_chunks(items, chunk_size): | |
| 89 """Split |items| in chunks of size |chunk_size|. | |
| 90 | |
| 91 Source: http://stackoverflow.com/a/312464 | |
| 92 """ | |
| 93 for i in xrange(0, len(items), chunk_size): | |
| 94 yield items[i:i + chunk_size] | |
| 95 | |
| 96 for chunk in _split_in_chunks(pgc_files, pgc_per_iter): | |
| 97 merge_command = [ | |
| 98 pgomgr_path, | |
| 99 '/merge' | |
| 100 ] | |
| 101 for pgc_file in chunk: | |
| 102 merge_command.append([ | |
| 103 os.path.join(options.build_dir, pgc_file) | |
| 104 ]) | |
| 105 | |
| 106 merge_command.append([ | |
| 107 os.path.join(options.build_dir, '%s.pgd' % options.binary_name) | |
| 108 ]) | |
| 109 proc = subprocess.Popen(merge_command, stdout=subprocess.PIPE) | |
| 110 stdout, _ = proc.communicate() | |
| 111 if proc.returncode != 0: | |
| 112 raise Exception('Error while trying to merge the PGC files:\n%s' % stdout) | |
| 113 | |
| 114 | |
| 115 if __name__ == '__main__': | |
| 116 sys.exit(main()) | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 2680683002:
Revert of Add a script to merge the PGC files (Closed)
