Add --enable-pgo option to build clang with PGO

tools/clang/scripts/update.py

 #!/usr/bin/env python
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """This script is used to download prebuilt clang binaries.
 
 It is also used by package.py to build the prebuilt clang binaries."""
 
 import argparse
@@ skipping 265 matching lines @@
   tool detection logic munges them in a weird way."""
   assert not any(i in os.path.basename(CHROME_TOOLS_SHIM_DIR) for i in '-_')
   os.mkdir(CHROME_TOOLS_SHIM_DIR)
   with file(os.path.join(CHROME_TOOLS_SHIM_DIR, 'CMakeLists.txt'), 'w') as f:
     f.write('# Automatically generated by tools/clang/scripts/update.py. ' +
             'Do not edit.\n')
     f.write('# Since tools/clang is located in another directory, use the \n')
     f.write('# two arg version to specify where build artifacts go. CMake\n')
     f.write('# disallows reuse of the same binary dir for multiple source\n')
     f.write('# dirs, so the build artifacts need to go into a subdirectory.\n')
-    f.write('# dirs, so the build artifacts need to go into a subdirectory.\n')

[Nico, 2017/04/07 15:58:53]

Also a good change, but also unrelated.

     f.write('if (CHROMIUM_TOOLS_SRC)\n')
     f.write('  add_subdirectory(${CHROMIUM_TOOLS_SRC} ' +
               '${CMAKE_CURRENT_BINARY_DIR}/a)\n')
     f.write('endif (CHROMIUM_TOOLS_SRC)\n')
 
 
 def DownloadHostGcc(args):
   """Downloads gcc 4.8.5 and makes sure args.gcc_toolchain is set."""
   if not sys.platform.startswith('linux') or args.gcc_toolchain:
     return
@@ skipping 72 matching lines @@
   return vs_version
 
 
 def CopyDiaDllTo(target_dir):
   # This script always wants to use the 64-bit msdia*.dll.
   dia_path = os.path.join(GetVSVersion().Path(), 'DIA SDK', 'bin', 'amd64')
   dia_dll = os.path.join(dia_path, DIA_DLL[GetVSVersion().ShortName()])
   CopyFile(dia_dll, target_dir)
 
 
+def GetNewCCAndCXX(install_dir):
+  if sys.platform == 'win32':
+    cc = os.path.join(install_dir, 'bin', 'clang-cl.exe')
+    cxx = os.path.join(install_dir, 'bin', 'clang-cl.exe')
+    # CMake has a hard time with backslashes in compiler paths:
+    # https://stackoverflow.com/questions/13050827
+    cc = cc.replace('\\', '/')
+    cxx = cxx.replace('\\', '/')
+  else:
+    cc = os.path.join(install_dir, 'bin', 'clang')
+    cxx = os.path.join(install_dir, 'bin', 'clang++')
+
+  return cc, cxx
+
+
+def BuildLLVMgold(args, install_dir, base_cmake_args, binutils_incdir, cc, cxx):
+  # Build LLVM gold plugin with LTO. That speeds up the linker by ~10%.
+  # We only use LTO for Linux now.
+  print 'Building LTO LLVM Gold plugin'
+  if os.path.exists(LLVM_LTO_GOLD_PLUGIN_DIR):
+    RmTree(LLVM_LTO_GOLD_PLUGIN_DIR)
+  EnsureDirExists(LLVM_LTO_GOLD_PLUGIN_DIR)
+  os.chdir(LLVM_LTO_GOLD_PLUGIN_DIR)
+
+  # Create a symlink to LLVMgold.so build in the previous step so that ar
+  # and ranlib could find it while linking LLVMgold.so with LTO.
+  EnsureDirExists(BFD_PLUGINS_DIR)
+  RunCommand(['ln', '-sf',
+              os.path.join(install_dir, 'lib', 'LLVMgold.so'),
+              os.path.join(BFD_PLUGINS_DIR, 'LLVMgold.so')])
+
+  lto_cflags = ['-flto=thin']
+  lto_ldflags = ['-fuse-ld=lld']
+  if args.gcc_toolchain:
+    # Tell the bootstrap compiler to use a specific gcc prefix to search
+    # for standard library headers and shared object files.
+    lto_cflags += ['--gcc-toolchain=' + args.gcc_toolchain]
+  lto_cmake_args = base_cmake_args + [
+      '-DLLVM_BINUTILS_INCDIR=' + binutils_incdir,
+      '-DCMAKE_C_COMPILER=' + cc,
+      '-DCMAKE_CXX_COMPILER=' + cxx,
+      '-DCMAKE_C_FLAGS=' + ' '.join(lto_cflags),
+      '-DCMAKE_CXX_FLAGS=' + ' '.join(lto_cflags),
+      '-DCMAKE_EXE_LINKER_FLAGS=' + ' '.join(lto_ldflags),
+      '-DCMAKE_SHARED_LINKER_FLAGS=' + ' '.join(lto_ldflags),
+      '-DCMAKE_MODULE_LINKER_FLAGS=' + ' '.join(lto_ldflags)]
+
+  # We need to use the proper binutils which support LLVM Gold plugin.
+  lto_env = os.environ.copy()
+  lto_env['PATH'] = BINUTILS_BIN_DIR + os.pathsep + lto_env.get('PATH', '')
+
+  RmCmakeCache('.')
+  RunCommand(['cmake'] + lto_cmake_args + [LLVM_DIR], env=lto_env)
+  RunCommand(['ninja', 'LLVMgold', 'lld'], env=lto_env)
+
+
 def VeryifyVersionOfBuiltClangMatchesVERSION():
   """Checks that `clang --version` outputs VERSION.  If this fails, VERSION
   in this file is out-of-date and needs to be updated (possibly in an
   `if use_head_revision:` block in main() first)."""
   clang = os.path.join(LLVM_BUILD_DIR, 'bin', 'clang')
   if sys.platform == 'win32':
     # TODO: Parse `clang-cl /?` output for built clang's version and check that
     # to check the binary we're actually shipping? But clang-cl.exe is just
     # a copy of clang.exe, so this does check the same thing.
     clang += '.exe'
@@ skipping 68 matching lines @@
 
   Checkout('LLVM', LLVM_REPO_URL + '/llvm/trunk', LLVM_DIR)
   Checkout('Clang', LLVM_REPO_URL + '/cfe/trunk', CLANG_DIR)
   if sys.platform != 'darwin':
     Checkout('LLD', LLVM_REPO_URL + '/lld/trunk', LLD_DIR)
   elif os.path.exists(LLD_DIR):
     # In case someone sends a tryjob that temporary adds lld to the checkout,
     # make sure it's not around on future builds.
     RmTree(LLD_DIR)
   Checkout('compiler-rt', LLVM_REPO_URL + '/compiler-rt/trunk', COMPILER_RT_DIR)
+  if args.enable_pgo and sys.platform != 'win32':
+    # PGO requires compiler-rt.
+    Checkout('compiler-rt', LLVM_REPO_URL + '/compiler-rt/trunk',
+             os.path.join(LLVM_DIR, 'projects', 'compiler-rt'))

[Nico, 2017/04/07 15:58:53]

IIRC package.py packages based on blacklists instead of whitelists – have you
checked we don't bundle more stuff due to this being present?

[matthewtff.asm, 2017/04/11 12:14:43]

The only diff I've got is one header file:
lib/clang/5.0.0/include/xray/xray_interface.h

I beleive it's ok to bundle this one since it's size is 3KB

   if sys.platform == 'darwin':
     # clang needs a libc++ checkout, else -stdlib=libc++ won't find includes
     # (i.e. this is needed for bootstrap builds).
     Checkout('libcxx', LLVM_REPO_URL + '/libcxx/trunk', LIBCXX_DIR)
     # We used to check out libcxxabi on OS X; we no longer need that.
     if os.path.exists(LIBCXXABI_DIR):
       RmTree(LIBCXXABI_DIR)
 
   cc, cxx = None, None
   libstdcpp = None
@@ skipping 46 matching lines @@
     RunCommand(['ninja'], msvc_arch='x64')
     if args.run_tests:
       if sys.platform == 'win32':
         CopyDiaDllTo(os.path.join(LLVM_BOOTSTRAP_DIR, 'bin'))
       RunCommand(['ninja', 'check-all'], msvc_arch='x64')
     RunCommand(['ninja', 'install'], msvc_arch='x64')
     if args.gcc_toolchain:
       # Copy that gcc's stdlibc++.so.6 to the build dir, so the bootstrap
       # compiler can start.
       CopyFile(libstdcpp, os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'lib'))
-
-    if sys.platform == 'win32':
-      cc = os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'bin', 'clang-cl.exe')
-      cxx = os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'bin', 'clang-cl.exe')
-      # CMake has a hard time with backslashes in compiler paths:
-      # https://stackoverflow.com/questions/13050827
-      cc = cc.replace('\\', '/')
-      cxx = cxx.replace('\\', '/')
-    else:
-      cc = os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'bin', 'clang')
-      cxx = os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'bin', 'clang++')
+    cc, cxx = GetNewCCAndCXX(LLVM_BOOTSTRAP_INSTALL_DIR)
 
     if args.gcc_toolchain:
       # Tell the bootstrap compiler to use a specific gcc prefix to search
       # for standard library headers and shared object files.
       cflags = ['--gcc-toolchain=' + args.gcc_toolchain]
       cxxflags = ['--gcc-toolchain=' + args.gcc_toolchain]
     print 'Building final compiler'
 
-  # Build LLVM gold plugin with LTO. That speeds up the linker by ~10%.
-  # We only use LTO for Linux now.
   if args.bootstrap and args.lto_gold_plugin:

[Nico, 2017/04/07 15:58:53]

This is now never true (assuming we default to PGO) since we only pass
--bootstrap on Windows and lto_gold_plugin is only true on Linux.

-    print 'Building LTO LLVM Gold plugin'
-    if os.path.exists(LLVM_LTO_GOLD_PLUGIN_DIR):
-      RmTree(LLVM_LTO_GOLD_PLUGIN_DIR)
-    EnsureDirExists(LLVM_LTO_GOLD_PLUGIN_DIR)
-    os.chdir(LLVM_LTO_GOLD_PLUGIN_DIR)
-
-    # Create a symlink to LLVMgold.so build in the previous step so that ar
-    # and ranlib could find it while linking LLVMgold.so with LTO.
-    EnsureDirExists(BFD_PLUGINS_DIR)
-    RunCommand(['ln', '-sf',
-                os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'lib', 'LLVMgold.so'),
-                os.path.join(BFD_PLUGINS_DIR, 'LLVMgold.so')])
-
-    lto_cflags = ['-flto=thin']
-    lto_ldflags = ['-fuse-ld=lld']
-    if args.gcc_toolchain:
-      # Tell the bootstrap compiler to use a specific gcc prefix to search
-      # for standard library headers and shared object files.
-      lto_cflags += ['--gcc-toolchain=' + args.gcc_toolchain]
-    lto_cmake_args = base_cmake_args + [
-        '-DLLVM_BINUTILS_INCDIR=' + binutils_incdir,
-        '-DCMAKE_C_COMPILER=' + cc,
-        '-DCMAKE_CXX_COMPILER=' + cxx,
-        '-DCMAKE_C_FLAGS=' + ' '.join(lto_cflags),
-        '-DCMAKE_CXX_FLAGS=' + ' '.join(lto_cflags),
-        '-DCMAKE_EXE_LINKER_FLAGS=' + ' '.join(lto_ldflags),
-        '-DCMAKE_SHARED_LINKER_FLAGS=' + ' '.join(lto_ldflags),
-        '-DCMAKE_MODULE_LINKER_FLAGS=' + ' '.join(lto_ldflags)]
-
-    # We need to use the proper binutils which support LLVM Gold plugin.
-    lto_env = os.environ.copy()
-    lto_env['PATH'] = BINUTILS_BIN_DIR + os.pathsep + lto_env.get('PATH', '')
-
-    RmCmakeCache('.')
-    RunCommand(['cmake'] + lto_cmake_args + [LLVM_DIR], env=lto_env)
-    RunCommand(['ninja', 'LLVMgold', 'lld'], env=lto_env)
-
+    BuildLLVMgold(args, LLVM_BOOTSTRAP_INSTALL_DIR, binutils_incdir,
+                  base_cmake_args, cc, cxx)
 
   # LLVM uses C++11 starting in llvm 3.5. On Linux, this means libstdc++4.7+ is
   # needed, on OS X it requires libc++. clang only automatically links to libc++
   # when targeting OS X 10.9+, so add stdlib=libc++ explicitly so clang can run
   # on OS X versions as old as 10.7.
   deployment_target = ''
 
-  if sys.platform == 'darwin' and args.bootstrap:
+  if sys.platform == 'darwin' and (args.bootstrap or args.enable_pgo):
     # When building on 10.9, /usr/include usually doesn't exist, and while
     # Xcode's clang automatically sets a sysroot, self-built clangs don't.
     cflags = ['-isysroot', subprocess.check_output(
         ['xcrun', '--show-sdk-path']).rstrip()]
     cxxflags = ['-stdlib=libc++'] + cflags
     ldflags += ['-stdlib=libc++']
     deployment_target = '10.7'
-    # Running libc++ tests takes a long time. Since it was only needed for
-    # the install step above, don't build it as part of the main build.
-    # This makes running package.py over 10% faster (30 min instead of 34 min)
-    RmTree(LIBCXX_DIR)
+    if not args.enable_pgo:

[Nico, 2017/04/07 15:58:53]

Why this change?

[matthewtff.asm, 2017/04/11 12:14:43]

We need to build clang with libcxx checked out in tree to get standart include
files. Previously those used to be copied from bootstrap build, but now it's no
longer possible.
Actually 4 minutes are no longer an issue as with PGO build takes much more than
30min :(

+      # Running libc++ tests takes a long time. Since it was only needed for
+      # the install step above, don't build it as part of the main build.
+      # This makes running package.py over 10% faster (30 min instead of 34 min)
+      RmTree(LIBCXX_DIR)
 
   # Build clang.
 
   # If building at head, define a macro that plugins can use for #ifdefing
   # out code that builds at head, but not at CLANG_REVISION or vice versa.
   if use_head_revision:
     cflags += ['-DLLVM_FORCE_HEAD_REVISION']
     cxxflags += ['-DLLVM_FORCE_HEAD_REVISION']
 
   # Build PDBs for archival on Windows.  Don't use RelWithDebInfo since it
@@ skipping 10 matching lines @@
     deployment_env = os.environ.copy()
     deployment_env['MACOSX_DEPLOYMENT_TARGET'] = deployment_target
 
   cmake_args = []
   # TODO(thakis): Unconditionally append this to base_cmake_args instead once
   # compiler-rt can build with clang-cl on Windows (http://llvm.org/PR23698)
   cc_args = base_cmake_args if sys.platform != 'win32' else cmake_args
   if cc is not None:  cc_args.append('-DCMAKE_C_COMPILER=' + cc)
   if cxx is not None: cc_args.append('-DCMAKE_CXX_COMPILER=' + cxx)
   chrome_tools = list(set(['plugins', 'blink_gc_plugin'] + args.extra_tools))
+
+
+  if args.enable_pgo:
+    pgo_cache_path = os.path.join(
+        LLVM_DIR, 'tools', 'clang', 'cmake', 'caches', 'PGO.cmake')
+    pgo_stage2_cache = os.path.join(
+        CHROMIUM_DIR, 'tools', 'clang', 'PGO-stage2.cmake')
+    cmake_args += [
+        '-DBOOTSTRAP_BOOTSTRAP_CHROMIUM_TOOLS=%s' % ','.join(chrome_tools),
+        '-DCMAKE_VERBOSE_MAKEFILE=ON',
+        '-DBOOTSTRAP_CMAKE_VERBOSE_MAKEFILE=ON',
+        '-DBOOTSTRAP_BOOTSTRAP_CMAKE_VERBOSE_MAKEFILE=ON',
+        '-DPGO_BUILD_CONFIGURATION=%s' % pgo_stage2_cache,
+        '-C', pgo_cache_path,
+        '-DBOOTSTRAP_LLVM_BINUTILS_INCDIR=' + binutils_incdir,
+        '-DBOOTSTRAP_CMAKE_C_FLAGS=' + ' '.join(cflags),
+        '-DBOOTSTRAP_CMAKE_CXX_FLAGS=' + ' '.join(cxxflags),
+        '-DBOOTSTRAP_BOOTSTRAP_LLVM_BINUTILS_INCDIR=' + binutils_incdir,
+        '-DBOOTSTRAP_BOOTSTRAP_CMAKE_C_FLAGS=' + ' '.join(cflags),
+        '-DBOOTSTRAP_BOOTSTRAP_CMAKE_CXX_FLAGS=' + ' '.join(cxxflags),
+        '-DBOOTSTRAP_BOOTSTRAP_CMAKE_EXE_LINKER_FLAGS=' + ' '.join(ldflags),
+        '-DBOOTSTRAP_BOOTSTRAP_CMAKE_SHARED_LINKER_FLAGS=' + ' '.join(ldflags),
+        '-DBOOTSTRAP_BOOTSTRAP_CMAKE_MODULE_LINKER_FLAGS=' + ' '.join(ldflags),
+    ]
   cmake_args += base_cmake_args + [
       '-DLLVM_ENABLE_THREADS=OFF',
       '-DLLVM_BINUTILS_INCDIR=' + binutils_incdir,
       '-DCMAKE_C_FLAGS=' + ' '.join(cflags),
       '-DCMAKE_CXX_FLAGS=' + ' '.join(cxxflags),
       '-DCMAKE_EXE_LINKER_FLAGS=' + ' '.join(ldflags),
       '-DCMAKE_SHARED_LINKER_FLAGS=' + ' '.join(ldflags),
       '-DCMAKE_MODULE_LINKER_FLAGS=' + ' '.join(ldflags),
       '-DCMAKE_INSTALL_PREFIX=' + LLVM_BUILD_DIR,
       # TODO(thakis): Remove this once official builds pass -Wl,--build-id
       # explicitly, https://crbug.com/622775
       '-DENABLE_LINKER_BUILD_ID=ON',
       '-DCHROMIUM_TOOLS_SRC=%s' % os.path.join(CHROMIUM_DIR, 'tools', 'clang'),
-      '-DCHROMIUM_TOOLS=%s' % ';'.join(chrome_tools)]
+      '-DCHROMIUM_TOOLS=%s' % ','.join(chrome_tools)]
 
   EnsureDirExists(LLVM_BUILD_DIR)
   os.chdir(LLVM_BUILD_DIR)
   RmCmakeCache('.')
   RunCommand(['cmake'] + cmake_args + [LLVM_DIR],
              msvc_arch='x64', env=deployment_env)
 
+  ninja_target = 'stage2' if args.enable_pgo else 'all'
+  RunCommand(['ninja', ninja_target], env=deployment_env, msvc_arch='x64')
+
+  if args.enable_pgo:
+    global LLVM_BUILD_DIR
+    LLVM_BUILD_DIR = os.path.join(LLVM_BUILD_DIR, 'tools', 'clang',
+        'stage2-instrumented-bins', 'tools', 'clang', 'stage2-bins')
+
   if args.gcc_toolchain:
     # Copy in the right stdlibc++.so.6 so clang can start.
     if not os.path.exists(os.path.join(LLVM_BUILD_DIR, 'lib')):
       os.mkdir(os.path.join(LLVM_BUILD_DIR, 'lib'))
     libstdcpp = subprocess.check_output(
         [cxx] + cxxflags + ['-print-file-name=libstdc++.so.6']).rstrip()
     CopyFile(libstdcpp, os.path.join(LLVM_BUILD_DIR, 'lib'))
 
-  RunCommand(['ninja'], msvc_arch='x64')
-
-  # Copy LTO-optimized lld, if any.
-  if args.bootstrap and args.lto_gold_plugin:
-    CopyFile(os.path.join(LLVM_LTO_GOLD_PLUGIN_DIR, 'bin', 'lld'),
-             os.path.join(LLVM_BUILD_DIR, 'bin'))
-
   if chrome_tools:
     # If any Chromium tools were built, install those now.
     RunCommand(['ninja', 'cr-install'], msvc_arch='x64')
 
+  # Copy LTO-optimized lld, if any.
+  if args.lto_gold_plugin:
+    if args.enable_pgo:
+      cc, cxx = GetNewCCAndCXX(LLVM_BUILD_DIR)
+      BuildLLVMgold(args, LLVM_BUILD_DIR, base_cmake_args,
+                    binutils_incdir, cc, cxx)
+    if args.bootstrap or args.enable_pgo:
+      CopyFile(os.path.join(LLVM_LTO_GOLD_PLUGIN_DIR, 'bin', 'lld'),
+               os.path.join(LLVM_BUILD_DIR, 'bin'))
+
   if sys.platform == 'darwin':
     # See http://crbug.com/256342
     RunCommand(['strip', '-x', os.path.join(LLVM_BUILD_DIR, 'bin', 'clang')])
   elif sys.platform.startswith('linux'):
     RunCommand(['strip', os.path.join(LLVM_BUILD_DIR, 'bin', 'clang')])
 
   VeryifyVersionOfBuiltClangMatchesVERSION()
 
   # Do an out-of-tree build of compiler-rt.
   # On Windows, this is used to get the 32-bit ASan run-time.
@@ skipping 167 matching lines @@
   parser.add_argument('--print-clang-version', action='store_true',
                       help='print current clang version (e.g. x.y.z) and exit.')
   parser.add_argument('--run-tests', action='store_true',
                       help='run tests after building; only for local builds')
   parser.add_argument('--extra-tools', nargs='*', default=[],
                       help='select additional chrome tools to build')
   parser.add_argument('--without-android', action='store_false',
                       help='don\'t build Android ASan runtime (linux only)',
                       dest='with_android',
                       default=sys.platform.startswith('linux'))
+  parser.add_argument('--enable-pgo', action='store_true',
+                      help='build clang in 2 stages using PGO')
   args = parser.parse_args()
 
-  if args.lto_gold_plugin and not args.bootstrap:
-    print '--lto-gold-plugin requires --bootstrap'
+  if args.lto_gold_plugin and (not args.bootstrap and not args.enable_pgo):
+    print '--lto-gold-plugin requires --bootstrap or --enable-pgo'
     return 1
   if args.lto_gold_plugin and not sys.platform.startswith('linux'):
     print '--lto-gold-plugin is only effective on Linux. Ignoring the option.'
     args.lto_gold_plugin = False
+  if args.enable_pgo:
+    if args.bootstrap:
+      print '--bootstrap makes no sense with --enable-pgo specified'
+      return 1
+    if not args.force_local_build:
+      print '--enable-pgo makes no sense without --force-local-build'
+      return 1
+    if sys.platform == 'win32':
+      print '--enable-pgo not supported on Windows'
 
   if args.if_needed:
     # TODO(thakis): Can probably remove this and --if-needed altogether.
     if re.search(r'\b(make_clang_dir)=', os.environ.get('GYP_DEFINES', '')):
       print 'Skipping Clang update (make_clang_dir= was set in GYP_DEFINES).'
       return 0
 
   # Get svn if we're going to use it to check the revision or do a local build.
   if (use_head_revision or args.llvm_force_head_revision or
       args.force_local_build):
@@ skipping 25 matching lines @@
     args.force_local_build = True
     if 'OS=android' not in os.environ.get('GYP_DEFINES', ''):
       # Only build the Android ASan rt on ToT bots when targetting Android.
       args.with_android = False
 
   return UpdateClang(args)
 
 
 if __name__ == '__main__':
   sys.exit(main())