Make the PGO profiling step work with VS2017

build/vs_toolchain.py

 #!/usr/bin/env python
 # Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import glob
 import json
 import os
 import pipes
 import platform
+import re
 import shutil
 import stat
 import subprocess
 import sys
 
 
 script_dir = os.path.dirname(os.path.realpath(__file__))
 chrome_src = os.path.abspath(os.path.join(script_dir, os.pardir))
 SRC_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 sys.path.insert(0, os.path.join(chrome_src, 'tools', 'gyp', 'pylib'))
@@ skipping 23 matching lines @@
     with open(json_data_file, 'r') as tempf:
       toolchain_data = json.load(tempf)
 
     toolchain = toolchain_data['path']
     version = toolchain_data['version']
     win_sdk = toolchain_data.get('win_sdk')
     if not win_sdk:
       win_sdk = toolchain_data['win8sdk']
     wdk = toolchain_data['wdk']
     # TODO(scottmg): The order unfortunately matters in these. They should be
-    # split into separate keys for x86 and x64. (See CopyVsRuntimeDlls call
-    # below). http://crbug.com/345992
+    # split into separate keys for x86 and x64. (See CopyDlls call below).
+    # http://crbug.com/345992
     vs_runtime_dll_dirs = toolchain_data['runtime_dirs']
 
     os.environ['GYP_MSVS_OVERRIDE_PATH'] = toolchain
     os.environ['GYP_MSVS_VERSION'] = version
 
     # Limit the scope of the gyp import to only where it is used. This
     # potentially lets build configs that never execute this block to drop
     # their GYP checkout.
     import gyp
 
@@ skipping 97 matching lines @@
       path = _RegistryGetValue(key, 'InstallDir')
       if not path:
         continue
       path = os.path.normpath(os.path.join(path, '..', '..'))
       return path
 
   raise Exception(('Visual Studio Version %s (from GYP_MSVS_VERSION)'
                    ' not found.') % (version_as_year))
 
 
-def _VersionNumber():
-  """Gets the standard version number ('120', '140', etc.) based on
-  GYP_MSVS_VERSION."""
-  vs_version = GetVisualStudioVersion()
-  if vs_version == '2015':
-    return '140'
-  if vs_version == '2017':
-    return '150'
-  raise ValueError('Unexpected GYP_MSVS_VERSION')
-
-
 def _CopyRuntimeImpl(target, source, verbose=True):
   """Copy |source| to |target| if it doesn't already exist or if it needs to be
   updated (comparing last modified time as an approximate float match as for
   some reason the values tend to differ by ~1e-07 despite being copies of the
   same file... https://crbug.com/603603).
   """
   if (os.path.isdir(os.path.dirname(target)) and
       (not os.path.isfile(target) or
        abs(os.stat(target).st_mtime - os.stat(source).st_mtime) >= 0.01)):
     if verbose:
@@ skipping 27 matching lines @@
   ucrt_files = glob.glob(os.path.join(ucrt_dll_dirs, 'api-ms-win-*.dll'))
   assert len(ucrt_files) > 0
   for ucrt_src_file in ucrt_files:
     file_part = os.path.basename(ucrt_src_file)
     ucrt_dst_file = os.path.join(target_dir, file_part)
     _CopyRuntimeImpl(ucrt_dst_file, ucrt_src_file, False)
   _CopyRuntimeImpl(os.path.join(target_dir, 'ucrtbase' + suffix),
                     os.path.join(source_dir, 'ucrtbase' + suffix))
 
 
+def _CopyPGORuntime(target_dir, target_cpu):
+  """Copy the runtime dependencies required during a PGO build.
+  """
+  env_version = GetVisualStudioVersion()
+  # These dependencies will be in a different location depending on the version
+  # of the toolchain.
+  if env_version == '2015':
+    pgo_x86_runtime_dir = os.path.join(os.environ.get('GYP_MSVS_OVERRIDE_PATH'),
+                                       'VC', 'bin')
+    pgo_x64_runtime_dir = os.path.join(pgo_x86_runtime_dir, 'amd64')
+  elif env_version == '2017':
+    # In VS2017 the PGO runtime dependencies are located in
+    # {toolchain_root}/VC/Tools/MSVC/{x.y.z}/Host{target_cpu}/{target_cpu}, the

[scottmg, 2017/05/11 19:30:28]

This comment is missing a "bin/" I think.

[Sébastien Marchand, 2017/05/11 20:15:35]

yep, fixed.

+    # {version_number} part is likely to change in case of a minor update of the
+    # toolchain so we don't hardcode this value here (except for the major
+    # number).
+    vc_tool_msvc_root = os.path.join(os.environ.get('GYP_MSVS_OVERRIDE_PATH'),

[scottmg, 2017/05/11 19:30:28]

vc_tools_msvc_root (with an 'S') to match the path name.

[Sébastien Marchand, 2017/05/11 20:15:35]

Done.

+        'VC', 'Tools', 'MSVC')
+    pgo_runtime_root = None
+    for directory in os.listdir(vc_tool_msvc_root):
+      if not os.path.isdir(os.path.join(vc_tool_msvc_root, directory)):
+        continue
+      if re.match('14\.\d+\.\d+', directory):
+        pgo_runtime_root = os.path.join(vc_tool_msvc_root, directory, 'bin')
+        break
+    assert pgo_runtime_root
+    pgo_x86_runtime_dir = os.path.join(pgo_runtime_root, 'HostX86', 'x86')

[scottmg, 2017/05/11 19:30:28]

HostX86 should be HostX64, as we only want target x86, right?

... Oh, apparently there's no pgosweep in HostX64\x86. Maybe a comment to that
effect would be good.

[Sébastien Marchand, 2017/05/11 20:15:35]

Done.

+    pgo_x64_runtime_dir = os.path.join(pgo_runtime_root, 'HostX64', 'x64')
+  else:
+    raise Exception('Unexpected toolchain version: %s.' % env_version)
+
+  # We need to copy 2 runtime dependencies used during the profiling step:
+  #     - pgort140.dll: runtime library required to run the instrumented image.
+  #     - pgosweep.exe: executable used to collect the profiling data
+  pgo_runtimes = ['pgort140.dll', 'pgosweep.exe']
+  for runtime in pgo_runtimes:
+    if target_cpu == 'x86':
+      source = os.path.join(pgo_x86_runtime_dir, runtime)
+    elif target_cpu == 'x64':
+      source = os.path.join(pgo_x64_runtime_dir, runtime)
+    else:
+      raise NotImplementedError("Unexpected target_cpu value: " + target_cpu)
+    if not os.path.exists(source):
+      raise Exception('Unable to find %s.' % source)
+    _CopyRuntimeImpl(os.path.join(target_dir, runtime), source)
+
+
 def _CopyRuntime(target_dir, source_dir, target_cpu, debug):
   """Copy the VS runtime DLLs, only if the target doesn't exist, but the target
   directory does exist. Handles VS 2015 and VS 2017."""
   suffix = "d.dll" if debug else ".dll"
   # VS 2017 uses the same CRT DLLs as VS 2015.
   _CopyUCRTRuntime(target_dir, source_dir, target_cpu, '%s140' + suffix,
                     suffix)
 
-  # Copy the PGO runtime library to the release directories.
-  if not debug and os.environ.get('GYP_MSVS_OVERRIDE_PATH'):
-    pgo_x86_runtime_dir = os.path.join(os.environ.get('GYP_MSVS_OVERRIDE_PATH'),
-                                        'VC', 'bin')
-    pgo_x64_runtime_dir = os.path.join(pgo_x86_runtime_dir, 'amd64')
-    pgo_runtime_dll = 'pgort' + _VersionNumber() + '.dll'
-    if target_cpu == "x86":
-      source_x86 = os.path.join(pgo_x86_runtime_dir, pgo_runtime_dll)
-      if os.path.exists(source_x86):
-        _CopyRuntimeImpl(os.path.join(target_dir, pgo_runtime_dll), source_x86)
-    elif target_cpu == "x64":
-      source_x64 = os.path.join(pgo_x64_runtime_dir, pgo_runtime_dll)
-      if os.path.exists(source_x64):
-        _CopyRuntimeImpl(os.path.join(target_dir, pgo_runtime_dll),
-                          source_x64)
-    else:
-      raise NotImplementedError("Unexpected target_cpu value:" + target_cpu)
-
-
-def CopyVsRuntimeDlls(output_dir, runtime_dirs):

[scottmg, 2017/05/11 19:30:28]

https://cs.chromium.org/search/?q=CopyVsRuntimeDlls+package:%5Echromium$&type=cs
:/ (I think it's fine to delete this though, v8 has its own copy, and I don't
think libyuv's is used except in a standalone build so they'll have to update
something.

[Sébastien Marchand, 2017/05/11 20:15:35]

Yeah, and the call to this function is behind a "if vs2013_runtime_dll_dirs", so
I really doubt that it's still in use?

-  """Copies the VS runtime DLLs from the given |runtime_dirs| to the output
-  directory so that even if not system-installed, built binaries are likely to
-  be able to run.
-
-  This needs to be run after gyp has been run so that the expected target
-  output directories are already created.
-
-  This is used for the GYP build and gclient runhooks.
-  """
-  x86, x64 = runtime_dirs
-  out_debug = os.path.join(output_dir, 'Debug')
-  out_debug_nacl64 = os.path.join(output_dir, 'Debug', 'x64')
-  out_release = os.path.join(output_dir, 'Release')
-  out_release_nacl64 = os.path.join(output_dir, 'Release', 'x64')
-  out_debug_x64 = os.path.join(output_dir, 'Debug_x64')
-  out_release_x64 = os.path.join(output_dir, 'Release_x64')
-
-  if os.path.exists(out_debug) and not os.path.exists(out_debug_nacl64):
-    os.makedirs(out_debug_nacl64)
-  if os.path.exists(out_release) and not os.path.exists(out_release_nacl64):
-    os.makedirs(out_release_nacl64)
-  _CopyRuntime(out_debug,          x86, "x86", debug=True)
-  _CopyRuntime(out_release,        x86, "x86", debug=False)
-  _CopyRuntime(out_debug_x64,      x64, "x64", debug=True)
-  _CopyRuntime(out_release_x64,    x64, "x64", debug=False)
-  _CopyRuntime(out_debug_nacl64,   x64, "x64", debug=True)
-  _CopyRuntime(out_release_nacl64, x64, "x64", debug=False)
-
 
 def CopyDlls(target_dir, configuration, target_cpu):
   """Copy the VS runtime DLLs into the requested directory as needed.
 
   configuration is one of 'Debug' or 'Release'.
   target_cpu is one of 'x86' or 'x64'.
 
   The debug configuration gets both the debug and release DLLs; the
   release config only the latter.
-
-  This is used for the GN build.
   """
   vs_runtime_dll_dirs = SetEnvironmentAndGetRuntimeDllDirs()
   if not vs_runtime_dll_dirs:
     return
 
   x64_runtime, x86_runtime = vs_runtime_dll_dirs
   runtime_dir = x64_runtime if target_cpu == 'x64' else x86_runtime
   _CopyRuntime(target_dir, runtime_dir, target_cpu, debug=False)
   if configuration == 'Debug':
     _CopyRuntime(target_dir, runtime_dir, target_cpu, debug=True)
+  else:
+    _CopyPGORuntime(target_dir, target_cpu)
 
   _CopyDebugger(target_dir, target_cpu)
 
 
 def _CopyDebugger(target_dir, target_cpu):
   """Copy dbghelp.dll into the requested directory as needed.
 
   target_cpu is one of 'x86' or 'x64'.
 
   dbghelp.dll is used when Chrome needs to symbolize stacks. Copying this file
@@ skipping 121 matching lines @@
       'copy_dlls': CopyDlls,
   }
   if len(sys.argv) < 2 or sys.argv[1] not in commands:
     print >>sys.stderr, 'Expected one of: %s' % ', '.join(commands)
     return 1
   return commands[sys.argv[1]](*sys.argv[2:])
 
 
 if __name__ == '__main__':
   sys.exit(main())