Add a script to merge the PGC files

build/win/merge_pgc_files.py

+#!/usr/bin/env python
+# Copyright 2017 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Merge the PGC files generated during the profiling step to the PGD database.
+
+This is required to workaround a flakyness in pgomgr.exe where it can run out
+of address space while trying to merge all the PGC files at the same time.
+"""
+
+import glob
+import json
+import optparse
+import os
+import subprocess
+import sys
+
+
+def find_pgomgr(chrome_checkout_dir, target_cpu):
+  """Find pgomgr.exe."""
+  if target_cpu not in ('x86', 'x64'):
+    raise Exception('target_cpu should be x86 or x64.')
+
+  win_toolchain_json_file = os.path.join(chrome_checkout_dir, 'build',
+      'win_toolchain.json')
+  if not os.path.exists(win_toolchain_json_file):
+    raise Exception('The toolchain JSON file is missing.')
+  with open(win_toolchain_json_file) as temp_f:
+    toolchain_data = json.load(temp_f)
+  if not os.path.isdir(toolchain_data['path']):
+    raise Exception('The toolchain JSON file is invalid.')
+
+  pgomgr_dir = os.path.join(toolchain_data['path'], 'VC', 'bin')
+  if target_cpu == 'x64':
+    pgomgr_dir = os.path.join(pgomgr_dir, 'amd64')
+
+  pgomgr_path = os.path.join(pgomgr_dir, 'pgomgr.exe')
+  if not os.path.exists(pgomgr_path):
+    raise Exception('pgomgr.exe is missing from %s.' % pgomgr_dir)
+
+  return pgomgr_path
+
+
+def main():
+  parser = optparse.OptionParser(usage='%prog [options]')
+  parser.add_option('--checkout-dir', help='The Chrome checkout directory.')
+  parser.add_option('--target-cpu', help='The target\'s bitness.')
+  parser.add_option('--build-dir', help='Chrome build directory.')
+  parser.add_option('--binary-name', help='The binary for which the PGC files '
+                    'should be merged, without extension.')
+  options, _ = parser.parse_args()
+
+  if not options.checkout_dir:
+    parser.error('--checkout-dir is required')
+  if not options.target_cpu:
+    parser.error('--target-cpu is required')
+  if not options.build_dir:
+    parser.error('--build-dir is required')
+  if not options.binary_name:
+    parser.error('--binary-name is required')
+
+  # Starts by finding pgomgr.exe.
+  pgomgr_path = find_pgomgr(options.checkout_dir, options.target_cpu)
+
+  pgc_files = glob.glob(os.path.join(options.build_dir,
+                                     '%s*.pgc' % options.binary_name))
+
+  # Number of PGC files that should be merged in each iterations, merging all
+  # the files one by one is really slow but merging more to 10 at a time doesn't
+  # really seem to impact the total time.
+  #
+  # Number of pgc merged per iteration  |  Time (in min)
+  # 1                                   |  27.2
+  # 10                                  |  12.8
+  # 20                                  |  12.0
+  # 30                                  |  11.5
+  # 40                                  |  11.4
+  # 50                                  |  11.5
+  # 60                                  |  11.6
+  # 70                                  |  11.6
+  # 80                                  |  11.7
+  #
+  # TODO(sebmarchand): Measure the memory usage of pgomgr.exe to see how it get
+  #     affected by the number of pgc files.
+  pgc_per_iter = 20
+  iterations = int(len(pgc_files) / pgc_per_iter) + (

[scottmg, 2017/02/04 02:44:09]

This makes my head hurt. :) How about http://stackoverflow.com/a/312464 on
pgc_files?

[Sébastien Marchand, 2017/02/06 16:48:37]

Done, it's much cleaner! Should I link to this post in my code?

+      1 if (len(pgc_files) % pgc_per_iter) != 0 else 0)
+  for i in xrange(0, iterations):
+    merge_command = [
+        pgomgr_path,
+        '/merge'
+    ]
+    for j in xrange(i * pgc_per_iter,
+                    min((i + 1) * pgc_per_iter, len(pgc_files))):
+      merge_command.append([
+          os.path.join(options.build_dir, pgc_files[j])
+      ])
+
+    merge_command.append([
+        os.path.join(options.build_dir, '%s.pgd' % options.binary_name)
+    ])
+    proc = subprocess.Popen(merge_command, stdout=subprocess.PIPE)
+    stdout, _ = proc.communicate()
+    if proc.returncode != 0:
+      raise Exception('Error while trying to merge the PGC files:\n%s' % stdout)
+
+
+if __name__ == '__main__':
+  sys.exit(main())