Index: tools/clang/scripts/update.py |
diff --git a/tools/clang/scripts/update.py b/tools/clang/scripts/update.py |
index 3571d3159462e951e913a72eb89ca1e4cba98652..c4729a31fb8dc6e967cd6c0330dfe319577c03df 100755 |
--- a/tools/clang/scripts/update.py |
+++ b/tools/clang/scripts/update.py |
@@ -375,6 +375,69 @@ def CopyDiaDllTo(target_dir): |
CopyFile(dia_dll, target_dir) |
+def GetNewCCAndCXX(install_dir): |
+ if sys.platform == 'win32': |
+ cc = os.path.join(install_dir, 'bin', 'clang-cl.exe') |
+ cxx = os.path.join(install_dir, 'bin', 'clang-cl.exe') |
+ # CMake has a hard time with backslashes in compiler paths: |
+ # https://stackoverflow.com/questions/13050827 |
+ cc = cc.replace('\\', '/') |
+ cxx = cxx.replace('\\', '/') |
+ else: |
+ cc = os.path.join(install_dir, 'bin', 'clang') |
+ cxx = os.path.join(install_dir, 'bin', 'clang++') |
+ |
+ return cc, cxx |
+ |
+ |
+def BuildLLVMgold(args, install_dir, base_cmake_args, binutils_incdir, cc, cxx): |
+ # Build LLVM gold plugin with LTO. That speeds up the linker by ~10%. |
+ # We only use LTO for Linux now. |
+ print 'Building LTO LLVM Gold plugin' |
+ if os.path.exists(LLVM_LTO_GOLD_PLUGIN_DIR): |
+ RmTree(LLVM_LTO_GOLD_PLUGIN_DIR) |
+ EnsureDirExists(LLVM_LTO_GOLD_PLUGIN_DIR) |
+ os.chdir(LLVM_LTO_GOLD_PLUGIN_DIR) |
+ |
+ # Create a symlink to LLVMgold.so build in the previous step so that ar |
+ # and ranlib could find it while linking LLVMgold.so with LTO. |
+ EnsureDirExists(BFD_PLUGINS_DIR) |
+ RunCommand(['ln', '-sf', |
+ os.path.join(install_dir, 'lib', 'LLVMgold.so'), |
+ os.path.join(BFD_PLUGINS_DIR, 'LLVMgold.so')]) |
+ |
+ # Link against binutils's copy of tcmalloc to speed up the linker by ~10%. |
+ # In package.py we copy the .so into our package. |
+ tcmalloc_ldflags = ['-L' + BINUTILS_LIB_DIR, '-ltcmalloc_minimal'] |
+ # Make sure that tblgen and the test suite can find tcmalloc. |
+ os.environ['LD_LIBRARY_PATH'] = \ |
+ BINUTILS_LIB_DIR + os.pathsep + os.environ.get('LD_LIBRARY_PATH', '') |
+ |
+ lto_cflags = ['-flto=thin'] |
+ lto_ldflags = ['-fuse-ld=lld'] |
+ if args.gcc_toolchain: |
+ # Tell the bootstrap compiler to use a specific gcc prefix to search |
+ # for standard library headers and shared object files. |
+ lto_cflags += ['--gcc-toolchain=' + args.gcc_toolchain] |
+ lto_cmake_args = base_cmake_args + [ |
+ '-DLLVM_BINUTILS_INCDIR=' + binutils_incdir, |
+ '-DCMAKE_C_COMPILER=' + cc, |
+ '-DCMAKE_CXX_COMPILER=' + cxx, |
+ '-DCMAKE_C_FLAGS=' + ' '.join(lto_cflags), |
+ '-DCMAKE_CXX_FLAGS=' + ' '.join(lto_cflags), |
+ '-DCMAKE_EXE_LINKER_FLAGS=' + ' '.join(lto_ldflags + tcmalloc_ldflags), |
+ '-DCMAKE_SHARED_LINKER_FLAGS=' + ' '.join(lto_ldflags), |
+ '-DCMAKE_MODULE_LINKER_FLAGS=' + ' '.join(lto_ldflags)] |
+ |
+ # We need to use the proper binutils which support LLVM Gold plugin. |
+ lto_env = os.environ.copy() |
+ lto_env['PATH'] = BINUTILS_BIN_DIR + os.pathsep + lto_env.get('PATH', '') |
+ |
+ RmCmakeCache('.') |
+ RunCommand(['cmake'] + lto_cmake_args + [LLVM_DIR], env=lto_env) |
+ RunCommand(['ninja', 'LLVMgold', 'lld'], env=lto_env) |
+ |
+ |
def VeryifyVersionOfBuiltClangMatchesVERSION(): |
"""Checks that `clang --version` outputs VERSION. If this fails, VERSION |
in this file is out-of-date and needs to be updated (possibly in an |
@@ -462,7 +525,10 @@ def UpdateClang(args): |
# In case someone sends a tryjob that temporary adds lld to the checkout, |
# make sure it's not around on future builds. |
RmTree(LLD_DIR) |
- Checkout('compiler-rt', LLVM_REPO_URL + '/compiler-rt/trunk', COMPILER_RT_DIR) |
+ if args.enable_pgo: |
+ # PGO requires compiler-rt. |
+ Checkout('compiler-rt', LLVM_REPO_URL + '/compiler-rt/trunk', |
+ os.path.join(LLVM_DIR, 'projects', 'compiler-rt')) |
if sys.platform == 'darwin': |
# clang needs a libc++ checkout, else -stdlib=libc++ won't find includes |
# (i.e. this is needed for bootstrap builds). |
@@ -504,105 +570,13 @@ def UpdateClang(args): |
if sys.platform.startswith('linux'): |
binutils_incdir = os.path.join(BINUTILS_DIR, 'include') |
- if args.bootstrap: |
- print 'Building bootstrap compiler' |
- EnsureDirExists(LLVM_BOOTSTRAP_DIR) |
- os.chdir(LLVM_BOOTSTRAP_DIR) |
- bootstrap_args = base_cmake_args + [ |
- '-DLLVM_BINUTILS_INCDIR=' + binutils_incdir, |
- '-DLLVM_TARGETS_TO_BUILD=host', |
- '-DCMAKE_INSTALL_PREFIX=' + LLVM_BOOTSTRAP_INSTALL_DIR, |
- '-DCMAKE_C_FLAGS=' + ' '.join(cflags), |
- '-DCMAKE_CXX_FLAGS=' + ' '.join(cxxflags), |
- ] |
- if cc is not None: bootstrap_args.append('-DCMAKE_C_COMPILER=' + cc) |
- if cxx is not None: bootstrap_args.append('-DCMAKE_CXX_COMPILER=' + cxx) |
- RmCmakeCache('.') |
- RunCommand(['cmake'] + bootstrap_args + [LLVM_DIR], msvc_arch='x64') |
- RunCommand(['ninja'], msvc_arch='x64') |
- if args.run_tests: |
- if sys.platform == 'win32': |
- CopyDiaDllTo(os.path.join(LLVM_BOOTSTRAP_DIR, 'bin')) |
- RunCommand(['ninja', 'check-all'], msvc_arch='x64') |
- RunCommand(['ninja', 'install'], msvc_arch='x64') |
- if args.gcc_toolchain: |
- # Copy that gcc's stdlibc++.so.6 to the build dir, so the bootstrap |
- # compiler can start. |
- CopyFile(libstdcpp, os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'lib')) |
- |
- if sys.platform == 'win32': |
- cc = os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'bin', 'clang-cl.exe') |
- cxx = os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'bin', 'clang-cl.exe') |
- # CMake has a hard time with backslashes in compiler paths: |
- # https://stackoverflow.com/questions/13050827 |
- cc = cc.replace('\\', '/') |
- cxx = cxx.replace('\\', '/') |
- else: |
- cc = os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'bin', 'clang') |
- cxx = os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'bin', 'clang++') |
- |
- if args.gcc_toolchain: |
- # Tell the bootstrap compiler to use a specific gcc prefix to search |
- # for standard library headers and shared object files. |
- cflags = ['--gcc-toolchain=' + args.gcc_toolchain] |
- cxxflags = ['--gcc-toolchain=' + args.gcc_toolchain] |
- print 'Building final compiler' |
- |
- # Build LLVM gold plugin with LTO. That speeds up the linker by ~10%. |
- # We only use LTO for Linux now. |
- if args.bootstrap and args.lto_gold_plugin: |
- print 'Building LTO LLVM Gold plugin' |
- if os.path.exists(LLVM_LTO_GOLD_PLUGIN_DIR): |
- RmTree(LLVM_LTO_GOLD_PLUGIN_DIR) |
- EnsureDirExists(LLVM_LTO_GOLD_PLUGIN_DIR) |
- os.chdir(LLVM_LTO_GOLD_PLUGIN_DIR) |
- |
- # Create a symlink to LLVMgold.so build in the previous step so that ar |
- # and ranlib could find it while linking LLVMgold.so with LTO. |
- EnsureDirExists(BFD_PLUGINS_DIR) |
- RunCommand(['ln', '-sf', |
- os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'lib', 'LLVMgold.so'), |
- os.path.join(BFD_PLUGINS_DIR, 'LLVMgold.so')]) |
- |
- # Link against binutils's copy of tcmalloc to speed up the linker by ~10%. |
- # In package.py we copy the .so into our package. |
- tcmalloc_ldflags = ['-L' + BINUTILS_LIB_DIR, '-ltcmalloc_minimal'] |
- # Make sure that tblgen and the test suite can find tcmalloc. |
- os.environ['LD_LIBRARY_PATH'] = \ |
- BINUTILS_LIB_DIR + os.pathsep + os.environ.get('LD_LIBRARY_PATH', '') |
- |
- lto_cflags = ['-flto=thin'] |
- lto_ldflags = ['-fuse-ld=lld'] |
- if args.gcc_toolchain: |
- # Tell the bootstrap compiler to use a specific gcc prefix to search |
- # for standard library headers and shared object files. |
- lto_cflags += ['--gcc-toolchain=' + args.gcc_toolchain] |
- lto_cmake_args = base_cmake_args + [ |
- '-DLLVM_BINUTILS_INCDIR=' + binutils_incdir, |
- '-DCMAKE_C_COMPILER=' + cc, |
- '-DCMAKE_CXX_COMPILER=' + cxx, |
- '-DCMAKE_C_FLAGS=' + ' '.join(lto_cflags), |
- '-DCMAKE_CXX_FLAGS=' + ' '.join(lto_cflags), |
- '-DCMAKE_EXE_LINKER_FLAGS=' + ' '.join(lto_ldflags + tcmalloc_ldflags), |
- '-DCMAKE_SHARED_LINKER_FLAGS=' + ' '.join(lto_ldflags), |
- '-DCMAKE_MODULE_LINKER_FLAGS=' + ' '.join(lto_ldflags)] |
- |
- # We need to use the proper binutils which support LLVM Gold plugin. |
- lto_env = os.environ.copy() |
- lto_env['PATH'] = BINUTILS_BIN_DIR + os.pathsep + lto_env.get('PATH', '') |
- |
- RmCmakeCache('.') |
- RunCommand(['cmake'] + lto_cmake_args + [LLVM_DIR], env=lto_env) |
- RunCommand(['ninja', 'LLVMgold', 'lld'], env=lto_env) |
- |
- |
# LLVM uses C++11 starting in llvm 3.5. On Linux, this means libstdc++4.7+ is |
# needed, on OS X it requires libc++. clang only automatically links to libc++ |
# when targeting OS X 10.9+, so add stdlib=libc++ explicitly so clang can run |
# on OS X versions as old as 10.7. |
deployment_target = '' |
- if sys.platform == 'darwin' and args.bootstrap: |
+ if sys.platform == 'darwin' and args.enable_pgo: |
# When building on 10.9, /usr/include usually doesn't exist, and while |
# Xcode's clang automatically sets a sysroot, self-built clangs don't. |
cflags = ['-isysroot', subprocess.check_output( |
@@ -610,10 +584,6 @@ def UpdateClang(args): |
cxxflags = ['-stdlib=libc++'] + cflags |
ldflags += ['-stdlib=libc++'] |
deployment_target = '10.7' |
- # Running libc++ tests takes a long time. Since it was only needed for |
- # the install step above, don't build it as part of the main build. |
- # This makes running package.py over 10% faster (30 min instead of 34 min) |
- RmTree(LIBCXX_DIR) |
# Build clang. |
@@ -644,17 +614,49 @@ def UpdateClang(args): |
if cc is not None: cc_args.append('-DCMAKE_C_COMPILER=' + cc) |
if cxx is not None: cc_args.append('-DCMAKE_CXX_COMPILER=' + cxx) |
chrome_tools = list(set(['plugins', 'blink_gc_plugin'] + args.extra_tools)) |
- cmake_args += base_cmake_args + [ |
- '-DLLVM_ENABLE_THREADS=OFF', |
+ |
+ |
+ stage2_cache = os.path.join(CHROMIUM_DIR, 'tools', 'clang', 'stage2.cmake') |
+ if args.bootstrap: |
+ additional_cmake_args = [ |
+ '-DBOOTSTRAP_CHROMIUM_TOOLS=%s' % ','.join(chrome_tools), |
+ '-DBOOTSTRAP_CMAKE_EXE_LINKER_FLAGS=' + ' '.join(ldflags), |
+ '-DBOOTSTRAP_CMAKE_SHARED_LINKER_FLAGS=' + ' '.join(ldflags), |
+ '-DBOOTSTRAP_CMAKE_MODULE_LINKER_FLAGS=' + ' '.join(ldflags), |
+ '-DSTAGE2_CACHE_FILE=%s' % stage2_cache, |
+ '-C', os.path.join(LLVM_DIR, 'tools', 'clang', 'cmake', 'caches', |
+ 'DistributionExample.cmake'), |
+ ] |
+ elif args.enable_pgo: |
+ additional_cmake_args = [ |
+ '-DBOOTSTRAP_BOOTSTRAP_CHROMIUM_TOOLS=%s' % ','.join(chrome_tools), |
+ '-DBOOTSTRAP_BOOTSTRAP_LLVM_BINUTILS_INCDIR=' + binutils_incdir, |
+ '-DBOOTSTRAP_BOOTSTRAP_CMAKE_C_FLAGS=' + ' '.join(cflags), |
+ '-DBOOTSTRAP_BOOTSTRAP_CMAKE_CXX_FLAGS=' + ' '.join(cxxflags), |
+ '-DBOOTSTRAP_BOOTSTRAP_CMAKE_EXE_LINKER_FLAGS=' + ' '.join(ldflags), |
+ '-DBOOTSTRAP_BOOTSTRAP_CMAKE_SHARED_LINKER_FLAGS=' + ' '.join(ldflags), |
+ '-DBOOTSTRAP_BOOTSTRAP_CMAKE_MODULE_LINKER_FLAGS=' + ' '.join(ldflags), |
+ '-DPGO_BUILD_CONFIGURATION=%s' % stage2_cache, |
+ '-C', os.path.join(LLVM_DIR, 'tools', 'clang', 'cmake', 'caches', |
+ 'PGO.cmake') |
+ ] |
+ |
+ cmake_args += base_cmake_args + additional_cmake_args + [ |
+ # Override lto setting from DistributionExample.cmake cache, if used. |
+ '-DPACKAGE_VENDOR=', |
+ '-DBOOTSTRAP_LLVM_ENABLE_LTO=OFF', |
+ '-DCMAKE_VERBOSE_MAKEFILE=ON', |
+ '-DBOOTSTRAP_CMAKE_VERBOSE_MAKEFILE=ON', |
+ '-DBOOTSTRAP_BOOTSTRAP_CMAKE_VERBOSE_MAKEFILE=ON', |
'-DLLVM_BINUTILS_INCDIR=' + binutils_incdir, |
+ '-DBOOTSTRAP_LLVM_BINUTILS_INCDIR=' + binutils_incdir, |
'-DCMAKE_C_FLAGS=' + ' '.join(cflags), |
+ '-DBOOTSTRAP_CMAKE_C_FLAGS=' + ' '.join(cflags), |
'-DCMAKE_CXX_FLAGS=' + ' '.join(cxxflags), |
- '-DCMAKE_EXE_LINKER_FLAGS=' + ' '.join(ldflags), |
- '-DCMAKE_SHARED_LINKER_FLAGS=' + ' '.join(ldflags), |
- '-DCMAKE_MODULE_LINKER_FLAGS=' + ' '.join(ldflags), |
+ '-DBOOTSTRAP_CMAKE_CXX_FLAGS=' + ' '.join(cxxflags), |
'-DCMAKE_INSTALL_PREFIX=' + LLVM_BUILD_DIR, |
- '-DCHROMIUM_TOOLS_SRC=%s' % os.path.join(CHROMIUM_DIR, 'tools', 'clang'), |
- '-DCHROMIUM_TOOLS=%s' % ';'.join(chrome_tools)] |
+ '-DCHROMIUM_TOOLS=%s' % ','.join(chrome_tools), |
+ ] |
EnsureDirExists(LLVM_BUILD_DIR) |
os.chdir(LLVM_BUILD_DIR) |
@@ -662,6 +664,18 @@ def UpdateClang(args): |
RunCommand(['cmake'] + cmake_args + [LLVM_DIR], |
msvc_arch='x64', env=deployment_env) |
+ RunCommand(['ninja', 'stage2'], env=deployment_env, msvc_arch='x64') |
+ |
+ if args.bootstrap: |
+ global LLVM_BUILD_DIR |
+ LLVM_BUILD_DIR = os.path.join( |
+ LLVM_BUILD_DIR, 'tools', 'clang', 'stage2-bins') |
+ elif args.enable_pgo: |
+ global LLVM_BUILD_DIR |
+ LLVM_BUILD_DIR = os.path.join(LLVM_BUILD_DIR, 'tools', 'clang', |
+ 'stage2-instrumented-bins', 'tools', 'clang', 'stage2-bins') |
+ os.chdir(LLVM_BUILD_DIR) |
+ |
if args.gcc_toolchain: |
# Copy in the right stdlibc++.so.6 so clang can start. |
if not os.path.exists(os.path.join(LLVM_BUILD_DIR, 'lib')): |
@@ -670,17 +684,16 @@ def UpdateClang(args): |
[cxx] + cxxflags + ['-print-file-name=libstdc++.so.6']).rstrip() |
CopyFile(libstdcpp, os.path.join(LLVM_BUILD_DIR, 'lib')) |
- RunCommand(['ninja'], msvc_arch='x64') |
+ RunCommand(['ninja', 'cr-install'], msvc_arch='x64') |
- # Copy LTO-optimized lld, if any. |
- if args.bootstrap and args.lto_gold_plugin: |
+ # Build & copy LTO-optimized lld, if any. |
+ if args.lto_gold_plugin: |
+ cc, cxx = GetNewCCAndCXX(LLVM_BUILD_DIR) |
+ BuildLLVMgold(args, LLVM_BUILD_DIR, base_cmake_args, |
+ binutils_incdir, cc, cxx) |
CopyFile(os.path.join(LLVM_LTO_GOLD_PLUGIN_DIR, 'bin', 'lld'), |
os.path.join(LLVM_BUILD_DIR, 'bin')) |
- if chrome_tools: |
- # If any Chromium tools were built, install those now. |
- RunCommand(['ninja', 'cr-install'], msvc_arch='x64') |
- |
if sys.platform == 'darwin': |
# See http://crbug.com/256342 |
RunCommand(['strip', '-x', os.path.join(LLVM_BUILD_DIR, 'bin', 'clang')]) |
@@ -694,6 +707,7 @@ def UpdateClang(args): |
# TODO(hans): Remove once the regular build above produces this. |
# On Mac and Linux, this is used to get the regular 64-bit run-time. |
# Do a clobbered build due to cmake changes. |
+ Checkout('compiler-rt', LLVM_REPO_URL + '/compiler-rt/trunk', COMPILER_RT_DIR) |
if os.path.isdir(COMPILER_RT_BUILD_DIR): |
RmTree(COMPILER_RT_BUILD_DIR) |
os.makedirs(COMPILER_RT_BUILD_DIR) |
@@ -868,14 +882,25 @@ def main(): |
help='don\'t build Android ASan runtime (linux only)', |
dest='with_android', |
default=sys.platform.startswith('linux')) |
+ parser.add_argument('--enable-pgo', action='store_true', |
+ help='build clang in 2 stages using PGO') |
args = parser.parse_args() |
- if args.lto_gold_plugin and not args.bootstrap: |
- print '--lto-gold-plugin requires --bootstrap' |
+ if args.lto_gold_plugin and not args.enable_pgo: |
+ print '--lto-gold-plugin requires --enable-pgo' |
return 1 |
if args.lto_gold_plugin and not sys.platform.startswith('linux'): |
print '--lto-gold-plugin is only effective on Linux. Ignoring the option.' |
args.lto_gold_plugin = False |
+ if args.enable_pgo: |
+ if args.bootstrap: |
+ print '--bootstrap makes no sense with --enable-pgo specified' |
+ return 1 |
+ if not args.force_local_build: |
+ print '--enable-pgo makes no sense without --force-local-build' |
+ return 1 |
+ if sys.platform == 'win32': |
+ print '--enable-pgo not supported on Windows' |
if args.if_needed: |
# TODO(thakis): Can probably remove this and --if-needed altogether. |