Add a script to merge the PGC files

build/win/merge_pgc_files.py

Index: build/win/merge_pgc_files.py
diff --git a/build/win/merge_pgc_files.py b/build/win/merge_pgc_files.py
new file mode 100644
index 0000000000000000000000000000000000000000..1498d98777f9833cd5c53980cb0ac559448d9e55
--- /dev/null
+++ b/build/win/merge_pgc_files.py
@@ -0,0 +1,113 @@
+#!/usr/bin/env python
+# Copyright 2017 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Merge the PGC files generated during the profiling step to the PGD database.
+
+This is required to workaround a flakyness in pgomgr.exe where it can run out
+of address space while trying to merge all the PGC files at the same time.
+"""
+
+import glob
+import json
+import optparse
+import os
+import subprocess
+import sys
+
+
+def find_pgomgr(chrome_checkout_dir, target_cpu):
+  """Find pgomgr.exe."""
+  if target_cpu not in ('x86', 'x64'):
+    raise Exception('target_cpu should be x86 or x64.')
+
+  win_toolchain_json_file = os.path.join(chrome_checkout_dir, 'build',
+      'win_toolchain.json')
+  if not os.path.exists(win_toolchain_json_file):
+    raise Exception('The toolchain JSON file is missing.')
+  with open(win_toolchain_json_file) as temp_f:
+    toolchain_data = json.load(temp_f)
+  if not os.path.isdir(toolchain_data['path']):
+    raise Exception('The toolchain JSON file is invalid.')
+
+  pgomgr_dir = os.path.join(toolchain_data['path'], 'VC', 'bin')
+  if target_cpu == 'x64':
+    pgomgr_dir = os.path.join(pgomgr_dir, 'amd64')
+
+  pgomgr_path = os.path.join(pgomgr_dir, 'pgomgr.exe')
+  if not os.path.exists(pgomgr_path):
+    raise Exception('pgomgr.exe is missing from %s.' % pgomgr_dir)
+
+  return pgomgr_path
+
+
+def main():
+  parser = optparse.OptionParser(usage='%prog [options]')
+  parser.add_option('--checkout-dir', help='The Chrome checkout directory.')
+  parser.add_option('--target-cpu', help='The target\'s bitness.')
+  parser.add_option('--build-dir', help='Chrome build directory.')
+  parser.add_option('--binary-name', help='The binary for which the PGC files '
+                    'should be merged, without extension.')
+  options, _ = parser.parse_args()
+
+  if not options.checkout_dir:
+    parser.error('--checkout-dir is required')
+  if not options.target_cpu:
+    parser.error('--target-cpu is required')
+  if not options.build_dir:
+    parser.error('--build-dir is required')
+  if not options.binary_name:
+    parser.error('--binary-name is required')
+
+  # Starts by finding pgomgr.exe.
+  pgomgr_path = find_pgomgr(options.checkout_dir, options.target_cpu)
+
+  pgc_files = glob.glob(os.path.join(options.build_dir,
+                                     '%s*.pgc' % options.binary_name))
+
+  # Number of PGC files that should be merged in each iterations, merging all
+  # the files one by one is really slow but merging more to 10 at a time doesn't
+  # really seem to impact the total time.
+  #
+  # Number of pgc merged per iteration  |  Time (in min)
+  # 1                                   |  27.2
+  # 10                                  |  12.8
+  # 20                                  |  12.0
+  # 30                                  |  11.5
+  # 40                                  |  11.4
+  # 50                                  |  11.5
+  # 60                                  |  11.6
+  # 70                                  |  11.6
+  # 80                                  |  11.7
+  #
+  # TODO(sebmarchand): Measure the memory usage of pgomgr.exe to see how it get
+  #     affected by the number of pgc files.
+  pgc_per_iter = 20
+
+  def _split_in_chunks(items, chunk_size):

[scottmg, 2017/02/06 17:57:14]

You could just use pgc_per_iter directly in the body of the function if you
think that's clearer. (But this is fine too if you like the generic version
better.)

[Sébastien Marchand, 2017/02/06 18:35:50]

I'll stick with the generic version so it can be extracted and re-used elsewhere
directly.

+    """Split |items| in chunks of size |chunk_size|."""
+    for i in xrange(0, len(items), chunk_size):
+      yield items[i:i + chunk_size]
+
+  for chunk in _split_in_chunks(pgc_files, pgc_per_iter):
+    merge_command = [
+        pgomgr_path,
+        '/merge'
+    ]
+    for pgc_file in chunk:
+      merge_command.append([
+          os.path.join(options.build_dir, pgc_file)
+      ])
+
+    merge_command.append([
+        os.path.join(options.build_dir, '%s.pgd' % options.binary_name)
+    ])
+    proc = subprocess.Popen(merge_command, stdout=subprocess.PIPE)
+    stdout, _ = proc.communicate()
+    if proc.returncode != 0:
+      raise Exception('Error while trying to merge the PGC files:\n%s' % stdout)
+
+
+if __name__ == '__main__':
+  sys.exit(main())