Clang toolchain: use lld to compile LLVM Gold plugin and lld with LTO.

tools/clang/scripts/update.py

 #!/usr/bin/env python
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """This script is used to download prebuilt clang binaries.
 
 It is also used by package.py to build the prebuilt clang binaries."""
 
 import argparse
@@ skipping 419 matching lines @@
     return 1
 
   DownloadHostGcc(args)
   AddCMakeToPath()
   AddGnuWinToPath()
 
   DeleteChromeToolsShim()
 
   Checkout('LLVM', LLVM_REPO_URL + '/llvm/trunk', LLVM_DIR)
   Checkout('Clang', LLVM_REPO_URL + '/cfe/trunk', CLANG_DIR)
-  if sys.platform == 'win32' or use_head_revision:
+  if (sys.platform == 'win32' or sys.platform.startswith('linux')
+      or use_head_revision):
     Checkout('LLD', LLVM_REPO_URL + '/lld/trunk', LLD_DIR)

[hans, 2016/07/22 15:33:05]

I wonder if we should just always build it, to simplify the logic here.
package.py still decides if it's included in the tarball.

[krasin1, 2016/07/22 20:19:57]

This is a good idea. Done.

   Checkout('compiler-rt', LLVM_REPO_URL + '/compiler-rt/trunk', COMPILER_RT_DIR)
   if sys.platform == 'darwin':
     # clang needs a libc++ checkout, else -stdlib=libc++ won't find includes
     # (i.e. this is needed for bootstrap builds).
     Checkout('libcxx', LLVM_REPO_URL + '/libcxx/trunk', LIBCXX_DIR)
     # We used to check out libcxxabi on OS X; we no longer need that.
     if os.path.exists(LIBCXXABI_DIR):
       RmTree(LIBCXXABI_DIR)
 
   cc, cxx = None, None
@@ skipping 70 matching lines @@
 
     if args.gcc_toolchain:
       # Tell the bootstrap compiler to use a specific gcc prefix to search
       # for standard library headers and shared object files.
       cflags = ['--gcc-toolchain=' + args.gcc_toolchain]
       cxxflags = ['--gcc-toolchain=' + args.gcc_toolchain]
     print 'Building final compiler'
 
   # Build LLVM gold plugin with LTO. That speeds up the linker by ~10%.
   # We only use LTO for Linux now.
-  if args.bootstrap and args.lto_gold_plugin:
+  if args.bootstrap and args.lto:
     print 'Building LTO LLVM Gold plugin'
     if os.path.exists(LLVM_LTO_GOLD_PLUGIN_DIR):
       RmTree(LLVM_LTO_GOLD_PLUGIN_DIR)
     EnsureDirExists(LLVM_LTO_GOLD_PLUGIN_DIR)
     os.chdir(LLVM_LTO_GOLD_PLUGIN_DIR)
 
-    # Create a symlink to LLVMgold.so build in the previous step so that ar
-    # and ranlib could find it while linking LLVMgold.so with LTO.
-    EnsureDirExists(BFD_PLUGINS_DIR)
-    RunCommand(['ln', '-sf',
-                os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'lib', 'LLVMgold.so'),
-                os.path.join(BFD_PLUGINS_DIR, 'LLVMgold.so')])
-
     lto_cflags = ['-flto']
-    lto_ldflags = ['-fuse-ld=gold']
+    lto_cxxflags = ['-flto']
+    lto_ldflags = ['-fuse-ld=lld']
     if args.gcc_toolchain:
       # Tell the bootstrap compiler to use a specific gcc prefix to search
       # for standard library headers and shared object files.
       lto_cflags += ['--gcc-toolchain=' + args.gcc_toolchain]
     lto_cmake_args = base_cmake_args + [
         '-DLLVM_BINUTILS_INCDIR=' + binutils_incdir,
         '-DCMAKE_C_COMPILER=' + cc,
         '-DCMAKE_CXX_COMPILER=' + cxx,
         '-DCMAKE_C_FLAGS=' + ' '.join(lto_cflags),
-        '-DCMAKE_CXX_FLAGS=' + ' '.join(lto_cflags),
+        '-DCMAKE_CXX_FLAGS=' + ' '.join(lto_cxxflags),
         '-DCMAKE_EXE_LINKER_FLAGS=' + ' '.join(lto_ldflags),
         '-DCMAKE_SHARED_LINKER_FLAGS=' + ' '.join(lto_ldflags),
         '-DCMAKE_MODULE_LINKER_FLAGS=' + ' '.join(lto_ldflags)]
 
-    # We need to use the proper binutils which support LLVM Gold plugin.
+    # We need to use the proper binutils which support LTO.
     lto_env = os.environ.copy()
     lto_env['PATH'] = BINUTILS_BIN_DIR + os.pathsep + lto_env.get('PATH', '')
-
     RmCmakeCache('.')
     RunCommand(['cmake'] + lto_cmake_args + [LLVM_DIR], env=lto_env)
     RunCommand(['ninja', 'LLVMgold'], env=lto_env)
 
 
   # LLVM uses C++11 starting in llvm 3.5. On Linux, this means libstdc++4.7+ is
   # needed, on OS X it requires libc++. clang only automatically links to libc++
   # when targeting OS X 10.9+, so add stdlib=libc++ explicitly so clang can run
   # on OS X versions as old as 10.7.
   deployment_target = ''
@@ skipping 14 matching lines @@
   # Build clang.
 
   # If building at head, define a macro that plugins can use for #ifdefing
   # out code that builds at head, but not at CLANG_REVISION or vice versa.
   if use_head_revision:
     cflags += ['-DLLVM_FORCE_HEAD_REVISION']
     cxxflags += ['-DLLVM_FORCE_HEAD_REVISION']
 
   CreateChromeToolsShim()
 
-  deployment_env = None
+  llvm_build_cflags = cflags
+  llvm_build_cxxflags = cxxflags
+  llvm_build_ldflags = ldflags
+  llvm_build_env = None
+  if sys.platform != 'win32':
+    # On Windows, the line below might add some Unicode entries
+    # and that will fail if passed into popen. See
+    # http://stackoverflow.com/questions/12253014/why-does-popen-fail-on-windows-if-the-env-parameter-contains-a-unicode-object
+    # Since we don't need to add anything into environment on Windows,
+    # just workaround this, until we moved into Python 3, where it's
+    # supposedly fixed.
+    llvm_build_env = os.environ.copy()

[hans, 2016/07/22 15:33:05]

Pretty scary. But now we'll have an empty llvm_build_env on Windows. Does that
work?

[krasin1, 2016/07/22 20:19:58]

This is a non-functional change for windows, as passing None as env into 
subprocess.call means "use the system env".

This is also exactly what we passed before my change, as deployment_env was
initialized to None as well.

   if deployment_target:
-    deployment_env = os.environ.copy()
-    deployment_env['MACOSX_DEPLOYMENT_TARGET'] = deployment_target
+    llvm_build_env['MACOSX_DEPLOYMENT_TARGET'] = deployment_target
+  if args.lto:
+    # Put proper binutils for building with LTO
+    llvm_build_env['PATH'] = (BINUTILS_BIN_DIR + os.pathsep +
+                              llvm_build_env.get('PATH', ''))
+    llvm_build_cflags += ['-flto']
+    llvm_build_cxxflags += ['-flto']
+    llvm_build_ldflags += ['-fuse-ld=lld']
 
   cmake_args = []
   # TODO(thakis): Unconditionally append this to base_cmake_args instead once
   # compiler-rt can build with clang-cl on Windows (http://llvm.org/PR23698)
   cc_args = base_cmake_args if sys.platform != 'win32' else cmake_args
   if cc is not None:  cc_args.append('-DCMAKE_C_COMPILER=' + cc)
   if cxx is not None: cc_args.append('-DCMAKE_CXX_COMPILER=' + cxx)
   cmake_args += base_cmake_args + [
       '-DLLVM_BINUTILS_INCDIR=' + binutils_incdir,
-      '-DCMAKE_C_FLAGS=' + ' '.join(cflags),
-      '-DCMAKE_CXX_FLAGS=' + ' '.join(cxxflags),
-      '-DCMAKE_EXE_LINKER_FLAGS=' + ' '.join(ldflags),
-      '-DCMAKE_SHARED_LINKER_FLAGS=' + ' '.join(ldflags),
-      '-DCMAKE_MODULE_LINKER_FLAGS=' + ' '.join(ldflags),
+      '-DCMAKE_C_FLAGS=' + ' '.join(llvm_build_cflags),
+      '-DCMAKE_CXX_FLAGS=' + ' '.join(llvm_build_cxxflags),
+      '-DCMAKE_EXE_LINKER_FLAGS=' + ' '.join(llvm_build_ldflags),
+      '-DCMAKE_SHARED_LINKER_FLAGS=' + ' '.join(llvm_build_ldflags),
+      '-DCMAKE_MODULE_LINKER_FLAGS=' + ' '.join(llvm_build_ldflags),
       '-DCMAKE_INSTALL_PREFIX=' + LLVM_BUILD_DIR,
       # TODO(thakis): Remove this once official builds pass -Wl,--build-id
       # explicitly, https://crbug.com/622775
       '-DENABLE_LINKER_BUILD_ID=ON',
       '-DCHROMIUM_TOOLS_SRC=%s' % os.path.join(CHROMIUM_DIR, 'tools', 'clang'),
       '-DCHROMIUM_TOOLS=%s' % ';'.join(args.tools)]
 
   EnsureDirExists(LLVM_BUILD_DIR)
   os.chdir(LLVM_BUILD_DIR)
   RmCmakeCache('.')
   RunCommand(['cmake'] + cmake_args + [LLVM_DIR],
-             msvc_arch='x64', env=deployment_env)
+             msvc_arch='x64', env=llvm_build_env)
 
   if args.gcc_toolchain:
     # Copy in the right stdlibc++.so.6 so clang can start.
     if not os.path.exists(os.path.join(LLVM_BUILD_DIR, 'lib')):
       os.mkdir(os.path.join(LLVM_BUILD_DIR, 'lib'))
     libstdcpp = subprocess.check_output(
         [cxx] + cxxflags + ['-print-file-name=libstdc++.so.6']).rstrip()
     CopyFile(libstdcpp, os.path.join(LLVM_BUILD_DIR, 'lib'))
 
   RunCommand(['ninja'], msvc_arch='x64')
@@ skipping 19 matching lines @@
     RmTree(COMPILER_RT_BUILD_DIR)
   os.makedirs(COMPILER_RT_BUILD_DIR)
   os.chdir(COMPILER_RT_BUILD_DIR)
   # TODO(thakis): Add this once compiler-rt can build with clang-cl (see
   # above).
   #if args.bootstrap and sys.platform == 'win32':
     # The bootstrap compiler produces 64-bit binaries by default.
     #cflags += ['-m32']
     #cxxflags += ['-m32']
   compiler_rt_args = base_cmake_args + [
-      '-DCMAKE_C_FLAGS=' + ' '.join(cflags),
-      '-DCMAKE_CXX_FLAGS=' + ' '.join(cxxflags)]
+      '-DCMAKE_C_FLAGS=' + ' '.join(llvm_build_cflags),
+      '-DCMAKE_CXX_FLAGS=' + ' '.join(llvm_build_cxxflags)]
   if sys.platform != 'win32':
     compiler_rt_args += ['-DLLVM_CONFIG_PATH=' +
                          os.path.join(LLVM_BUILD_DIR, 'bin', 'llvm-config'),
                         '-DSANITIZER_MIN_OSX_VERSION="10.7"']
   # compiler-rt is part of the llvm checkout on Windows but a stand-alone
   # directory elsewhere, see the TODO above COMPILER_RT_DIR.
   RmCmakeCache('.')
   RunCommand(['cmake'] + compiler_rt_args +
              [LLVM_DIR if sys.platform == 'win32' else COMPILER_RT_DIR],
-             msvc_arch='x86', env=deployment_env)
+             msvc_arch='x86', env=llvm_build_env)
   RunCommand(['ninja', 'compiler-rt'], msvc_arch='x86')
 
   # Copy select output to the main tree.
   # TODO(hans): Make this (and the .gypi and .isolate files) version number
   # independent.
   if sys.platform == 'win32':
     platform = 'windows'
   elif sys.platform == 'darwin':
     platform = 'darwin'
   else:
@@ skipping 110 matching lines @@
   parser = argparse.ArgumentParser(description='Build Clang.')
   parser.add_argument('--bootstrap', action='store_true',
                       help='first build clang with CC, then with itself.')
   parser.add_argument('--if-needed', action='store_true',
                       help="run only if the script thinks clang is needed")
   parser.add_argument('--force-local-build', action='store_true',
                       help="don't try to download prebuild binaries")
   parser.add_argument('--gcc-toolchain', help='set the version for which gcc '
                       'version be used for building; --gcc-toolchain=/opt/foo '
                       'picks /opt/foo/bin/gcc')
-  parser.add_argument('--lto-gold-plugin', action='store_true',
-                      help='build LLVM Gold plugin with LTO')
+  parser.add_argument('--lto', action='store_true',
+                      help='build Clang toolchain with LTO')

[hans, 2016/07/22 15:33:05]

The flag also controls whether to build the Gold plugin. Maybe a little bit
confusing..

[krasin1, 2016/07/22 20:19:57]

Yes, but I hope we can remove Gold plugin altogether very soon. If not, we can
refactor it later.

   parser.add_argument('--llvm-force-head-revision', action='store_true',
                       help=('use the revision in the repo when printing '
                             'the revision'))
   parser.add_argument('--print-revision', action='store_true',
                       help='print current clang revision and exit.')
   parser.add_argument('--print-clang-version', action='store_true',
                       help='print current clang version (e.g. x.y.z) and exit.')
   parser.add_argument('--run-tests', action='store_true',
                       help='run tests after building; only for local builds')
   parser.add_argument('--tools', nargs='*',
                       help='select which chrome tools to build',
                       default=['plugins', 'blink_gc_plugin'])
   parser.add_argument('--without-android', action='store_false',
                       help='don\'t build Android ASan runtime (linux only)',
                       dest='with_android',
                       default=sys.platform.startswith('linux'))
   args = parser.parse_args()
 
-  if args.lto_gold_plugin and not args.bootstrap:
-    print '--lto-gold-plugin requires --bootstrap'
+  if args.lto and not args.bootstrap:
+    print '--lto requires --bootstrap'
     return 1
-  if args.lto_gold_plugin and not sys.platform.startswith('linux'):
-    print '--lto-gold-plugin is only effective on Linux. Ignoring the option.'
-    args.lto_gold_plugin = False
+  if args.lto and not sys.platform.startswith('linux'):
+    print '--lto is only effective on Linux. Ignoring the option.'
+    args.lto = False
 
   if args.if_needed:
     is_clang_required = False
     # clang is always used on Mac and Linux.
     if sys.platform == 'darwin' or sys.platform.startswith('linux'):
       is_clang_required = True
     # clang requested via $GYP_DEFINES.
     if re.search(r'\b(clang|asan|lsan|msan|tsan)=1',
                  os.environ.get('GYP_DEFINES', '')):
       is_clang_required = True
@@ skipping 33 matching lines @@
     args.force_local_build = True
     if 'OS=android' not in os.environ.get('GYP_DEFINES', ''):
       # Only build the Android ASan rt on ToT bots when targetting Android.
       args.with_android = False
 
   return UpdateClang(args)
 
 
 if __name__ == '__main__':
   sys.exit(main())