Index: tools/clang/scripts/update.py |
diff --git a/tools/clang/scripts/update.py b/tools/clang/scripts/update.py |
index 55b64eeffa9bfd261e97f8d551eb8036efabf203..100bdec5cc88dd60957affff4f16c1ffb2d6eda4 100755 |
--- a/tools/clang/scripts/update.py |
+++ b/tools/clang/scripts/update.py |
@@ -3,12 +3,12 @@ |
# Use of this source code is governed by a BSD-style license that can be |
# found in the LICENSE file. |
-"""Windows can't run .sh files, so this is a Python implementation of |
-update.sh. This script should replace update.sh on all platforms eventually.""" |
+"""This script is used to download prebuilt clang binaries. |
+ |
+It is also used by package.py to build the prebuilt clang binaries.""" |
import argparse |
-import contextlib |
-import cStringIO |
+import distutils.spawn |
import glob |
import os |
import pipes |
@@ -18,26 +18,25 @@ import subprocess |
import stat |
import sys |
import tarfile |
+import tempfile |
import time |
import urllib2 |
import zipfile |
+ |
# Do NOT CHANGE this if you don't know what you're doing -- see |
-# https://code.google.com/p/chromium/wiki/UpdatingClang |
+# https://chromium.googlesource.com/chromium/src/+/master/docs/updating_clang.md |
# Reverting problematic clang rolls is safe, though. |
-# Note: this revision is only used for Windows. Other platforms use update.sh. |
-# TODO(thakis): Use the same revision on Windows and non-Windows. |
-# TODO(thakis): Remove update.sh, use update.py everywhere. |
-LLVM_WIN_REVISION = '242415' |
+CLANG_REVISION = '282487' |
use_head_revision = 'LLVM_FORCE_HEAD_REVISION' in os.environ |
if use_head_revision: |
- LLVM_WIN_REVISION = 'HEAD' |
+ CLANG_REVISION = 'HEAD' |
# This is incremented when pushing a new build of Clang at the same revision. |
CLANG_SUB_REVISION=1 |
-PACKAGE_VERSION = "%s-%s" % (LLVM_WIN_REVISION, CLANG_SUB_REVISION) |
+PACKAGE_VERSION = "%s-%s" % (CLANG_REVISION, CLANG_SUB_REVISION) |
# Path constants. (All of these should be absolute paths.) |
THIS_DIR = os.path.abspath(os.path.dirname(__file__)) |
@@ -47,71 +46,135 @@ LLVM_DIR = os.path.join(THIRD_PARTY_DIR, 'llvm') |
LLVM_BOOTSTRAP_DIR = os.path.join(THIRD_PARTY_DIR, 'llvm-bootstrap') |
LLVM_BOOTSTRAP_INSTALL_DIR = os.path.join(THIRD_PARTY_DIR, |
'llvm-bootstrap-install') |
+LLVM_LTO_GOLD_PLUGIN_DIR = os.path.join(THIRD_PARTY_DIR, 'llvm-lto-gold-plugin') |
CHROME_TOOLS_SHIM_DIR = os.path.join(LLVM_DIR, 'tools', 'chrometools') |
LLVM_BUILD_DIR = os.path.join(CHROMIUM_DIR, 'third_party', 'llvm-build', |
'Release+Asserts') |
-COMPILER_RT_BUILD_DIR = os.path.join(LLVM_BUILD_DIR, '32bit-compiler-rt') |
+COMPILER_RT_BUILD_DIR = os.path.join(LLVM_BUILD_DIR, 'compiler-rt') |
CLANG_DIR = os.path.join(LLVM_DIR, 'tools', 'clang') |
LLD_DIR = os.path.join(LLVM_DIR, 'tools', 'lld') |
-COMPILER_RT_DIR = os.path.join(LLVM_DIR, 'projects', 'compiler-rt') |
+# compiler-rt is built as part of the regular LLVM build on Windows to get |
+# the 64-bit runtime, and out-of-tree elsewhere. |
+# TODO(thakis): Try to unify this. |
+if sys.platform == 'win32': |
+ COMPILER_RT_DIR = os.path.join(LLVM_DIR, 'projects', 'compiler-rt') |
+else: |
+ COMPILER_RT_DIR = os.path.join(LLVM_DIR, 'compiler-rt') |
LIBCXX_DIR = os.path.join(LLVM_DIR, 'projects', 'libcxx') |
LIBCXXABI_DIR = os.path.join(LLVM_DIR, 'projects', 'libcxxabi') |
LLVM_BUILD_TOOLS_DIR = os.path.abspath( |
os.path.join(LLVM_DIR, '..', 'llvm-build-tools')) |
-STAMP_FILE = os.path.join(LLVM_DIR, '..', 'llvm-build', 'cr_build_revision') |
+STAMP_FILE = os.path.normpath( |
+ os.path.join(LLVM_DIR, '..', 'llvm-build', 'cr_build_revision')) |
BINUTILS_DIR = os.path.join(THIRD_PARTY_DIR, 'binutils') |
-VERSION = '3.8.0' |
+BINUTILS_BIN_DIR = os.path.join(BINUTILS_DIR, BINUTILS_DIR, |
+ 'Linux_x64', 'Release', 'bin') |
+BFD_PLUGINS_DIR = os.path.join(BINUTILS_DIR, 'Linux_x64', 'Release', |
+ 'lib', 'bfd-plugins') |
+VERSION = '4.0.0' |
+ANDROID_NDK_DIR = os.path.join( |
+ CHROMIUM_DIR, 'third_party', 'android_tools', 'ndk') |
# URL for pre-built binaries. |
-CDS_URL = 'https://commondatastorage.googleapis.com/chromium-browser-clang' |
+CDS_URL = os.environ.get('CDS_CLANG_BUCKET_OVERRIDE', |
+ 'https://commondatastorage.googleapis.com/chromium-browser-clang') |
LLVM_REPO_URL='https://llvm.org/svn/llvm-project' |
if 'LLVM_REPO_URL' in os.environ: |
LLVM_REPO_URL = os.environ['LLVM_REPO_URL'] |
+# Bump after VC updates. |
+DIA_DLL = { |
+ '2013': 'msdia120.dll', |
+ '2015': 'msdia140.dll', |
+} |
+ |
def DownloadUrl(url, output_file): |
"""Download url into output_file.""" |
CHUNK_SIZE = 4096 |
TOTAL_DOTS = 10 |
- sys.stdout.write('Downloading %s ' % url) |
- sys.stdout.flush() |
- response = urllib2.urlopen(url) |
- total_size = int(response.info().getheader('Content-Length').strip()) |
- bytes_done = 0 |
- dots_printed = 0 |
+ num_retries = 3 |
+ retry_wait_s = 5 # Doubled at each retry. |
+ |
while True: |
- chunk = response.read(CHUNK_SIZE) |
- if not chunk: |
- break |
- output_file.write(chunk) |
- bytes_done += len(chunk) |
- num_dots = TOTAL_DOTS * bytes_done / total_size |
- sys.stdout.write('.' * (num_dots - dots_printed)) |
- sys.stdout.flush() |
- dots_printed = num_dots |
- print ' Done.' |
- |
- |
-def ReadStampFile(): |
+ try: |
+ sys.stdout.write('Downloading %s ' % url) |
+ sys.stdout.flush() |
+ response = urllib2.urlopen(url) |
+ total_size = int(response.info().getheader('Content-Length').strip()) |
+ bytes_done = 0 |
+ dots_printed = 0 |
+ while True: |
+ chunk = response.read(CHUNK_SIZE) |
+ if not chunk: |
+ break |
+ output_file.write(chunk) |
+ bytes_done += len(chunk) |
+ num_dots = TOTAL_DOTS * bytes_done / total_size |
+ sys.stdout.write('.' * (num_dots - dots_printed)) |
+ sys.stdout.flush() |
+ dots_printed = num_dots |
+ if bytes_done != total_size: |
+ raise urllib2.URLError("only got %d of %d bytes" % |
+ (bytes_done, total_size)) |
+ print ' Done.' |
+ return |
+ except urllib2.URLError as e: |
+ sys.stdout.write('\n') |
+ print e |
+ if num_retries == 0 or isinstance(e, urllib2.HTTPError) and e.code == 404: |
+ raise e |
+ num_retries -= 1 |
+ print 'Retrying in %d s ...' % retry_wait_s |
+ time.sleep(retry_wait_s) |
+ retry_wait_s *= 2 |
+ |
+ |
+def EnsureDirExists(path): |
+ if not os.path.exists(path): |
+ print "Creating directory %s" % path |
+ os.makedirs(path) |
+ |
+ |
+def DownloadAndUnpack(url, output_dir): |
+ with tempfile.TemporaryFile() as f: |
+ DownloadUrl(url, f) |
+ f.seek(0) |
+ EnsureDirExists(output_dir) |
+ if url.endswith('.zip'): |
+ zipfile.ZipFile(f).extractall(path=output_dir) |
+ else: |
+ tarfile.open(mode='r:gz', fileobj=f).extractall(path=output_dir) |
+ |
+ |
+def ReadStampFile(path=STAMP_FILE): |
"""Return the contents of the stamp file, or '' if it doesn't exist.""" |
try: |
- with open(STAMP_FILE, 'r') as f: |
- return f.read() |
+ with open(path, 'r') as f: |
+ return f.read().rstrip() |
except IOError: |
return '' |
-def WriteStampFile(s): |
+def WriteStampFile(s, path=STAMP_FILE): |
"""Write s to the stamp file.""" |
- if not os.path.exists(os.path.dirname(STAMP_FILE)): |
- os.makedirs(os.path.dirname(STAMP_FILE)) |
- with open(STAMP_FILE, 'w') as f: |
+ EnsureDirExists(os.path.dirname(path)) |
+ with open(path, 'w') as f: |
f.write(s) |
+ f.write('\n') |
def GetSvnRevision(svn_repo): |
"""Returns current revision of the svn repo at svn_repo.""" |
+ if sys.platform == 'darwin': |
+ # mac_files toolchain must be set for hermetic builds. |
+ root = os.path.dirname(os.path.dirname(os.path.dirname( |
+ os.path.dirname(__file__)))) |
+ sys.path.append(os.path.join(root, 'build')) |
+ import mac_toolchain |
+ |
+ mac_toolchain.SetToolchainEnvironment() |
svn_info = subprocess.check_output('svn info ' + svn_repo, shell=True) |
m = re.search(r'Revision: (\d+)', svn_info) |
return m.group(1) |
@@ -129,6 +192,15 @@ def RmTree(dir): |
shutil.rmtree(dir, onerror=ChmodAndRetry) |
+def RmCmakeCache(dir): |
+ """Delete CMake cache related files from dir.""" |
+ for dirpath, dirs, files in os.walk(dir): |
+ if 'CMakeCache.txt' in files: |
+ os.remove(os.path.join(dirpath, 'CMakeCache.txt')) |
+ if 'CMakeFiles' in dirs: |
+ RmTree(os.path.join(dirpath, 'CMakeFiles')) |
+ |
+ |
def RunCommand(command, msvc_arch=None, env=None, fail_hard=True): |
"""Run command and return success (True) or failure; or if fail_hard is |
True, exit on failure. If msvc_arch is set, runs the command in a |
@@ -163,15 +235,15 @@ def RunCommand(command, msvc_arch=None, env=None, fail_hard=True): |
def CopyFile(src, dst): |
"""Copy a file from src to dst.""" |
- shutil.copy(src, dst) |
print "Copying %s to %s" % (src, dst) |
+ shutil.copy(src, dst) |
def CopyDirectoryContents(src, dst, filename_filter=None): |
"""Copy the files from directory src to dst |
with an optional filename filter.""" |
- if not os.path.exists(dst): |
- os.makedirs(dst) |
+ dst = os.path.realpath(dst) # realpath() in case dst ends in /.. |
+ EnsureDirExists(dst) |
for root, _, files in os.walk(src): |
for f in files: |
if filename_filter and not re.match(filename_filter, f): |
@@ -181,9 +253,9 @@ def CopyDirectoryContents(src, dst, filename_filter=None): |
def Checkout(name, url, dir): |
"""Checkout the SVN module at url into dir. Use name for the log message.""" |
- print "Checking out %s r%s into '%s'" % (name, LLVM_WIN_REVISION, dir) |
+ print "Checking out %s r%s into '%s'" % (name, CLANG_REVISION, dir) |
- command = ['svn', 'checkout', '--force', url + '@' + LLVM_WIN_REVISION, dir] |
+ command = ['svn', 'checkout', '--force', url + '@' + CLANG_REVISION, dir] |
if RunCommand(command, fail_hard=False): |
return |
@@ -195,120 +267,9 @@ def Checkout(name, url, dir): |
RunCommand(command) |
-def RevertPreviouslyPatchedFiles(): |
- print 'Reverting previously patched files' |
- files = [ |
- '%(clang)s/test/Index/crash-recovery-modules.m', |
- '%(clang)s/unittests/libclang/LibclangTest.cpp', |
- '%(compiler_rt)s/lib/asan/asan_rtl.cc', |
- '%(compiler_rt)s/test/asan/TestCases/Linux/new_array_cookie_test.cc', |
- '%(llvm)s/test/DebugInfo/gmlt.ll', |
- '%(llvm)s/lib/CodeGen/SpillPlacement.cpp', |
- '%(llvm)s/lib/CodeGen/SpillPlacement.h', |
- '%(llvm)s/lib/Transforms/Instrumentation/MemorySanitizer.cpp', |
- '%(clang)s/test/Driver/env.c', |
- '%(clang)s/lib/Frontend/InitPreprocessor.cpp', |
- '%(clang)s/test/Frontend/exceptions.c', |
- '%(clang)s/test/Preprocessor/predefined-exceptions.m', |
- '%(llvm)s/test/Bindings/Go/go.test', |
- '%(clang)s/lib/Parse/ParseExpr.cpp', |
- '%(clang)s/lib/Parse/ParseTemplate.cpp', |
- '%(clang)s/lib/Sema/SemaDeclCXX.cpp', |
- '%(clang)s/lib/Sema/SemaExprCXX.cpp', |
- '%(clang)s/test/SemaCXX/default2.cpp', |
- '%(clang)s/test/SemaCXX/typo-correction-delayed.cpp', |
- '%(compiler_rt)s/lib/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cc', |
- '%(compiler_rt)s/test/tsan/signal_segv_handler.cc', |
- '%(compiler_rt)s/lib/sanitizer_common/sanitizer_coverage_libcdep.cc', |
- '%(compiler_rt)s/cmake/config-ix.cmake', |
- '%(compiler_rt)s/CMakeLists.txt', |
- '%(compiler_rt)s/lib/ubsan/ubsan_platform.h', |
- ] |
- for f in files: |
- f = f % { |
- 'clang': CLANG_DIR, |
- 'compiler_rt': COMPILER_RT_DIR, |
- 'llvm': LLVM_DIR, |
- } |
- if os.path.exists(f): |
- os.remove(f) # For unversioned files. |
- RunCommand(['svn', 'revert', f]) |
- |
- |
-def ApplyLocalPatches(): |
- # There's no patch program on Windows by default. We don't need patches on |
- # Windows yet, and maybe this not working on Windows will motivate us to |
- # remove patches over time. |
- assert sys.platform != 'win32' |
- |
- # Apply patch for tests failing with --disable-pthreads (llvm.org/PR11974) |
- clang_patches = [ r"""\ |
---- test/Index/crash-recovery-modules.m (revision 202554) |
-+++ test/Index/crash-recovery-modules.m (working copy) |
-@@ -12,6 +12,8 @@ |
- |
- // REQUIRES: crash-recovery |
- // REQUIRES: shell |
-+// XFAIL: * |
-+// (PR11974) |
- |
- @import Crash; |
-""", r"""\ |
---- unittests/libclang/LibclangTest.cpp (revision 215949) |
-+++ unittests/libclang/LibclangTest.cpp (working copy) |
-@@ -431,7 +431,7 @@ |
- EXPECT_EQ(0U, clang_getNumDiagnostics(ClangTU)); |
- } |
- |
--TEST_F(LibclangReparseTest, ReparseWithModule) { |
-+TEST_F(LibclangReparseTest, DISABLED_ReparseWithModule) { |
- const char *HeaderTop = "#ifndef H\n#define H\nstruct Foo { int bar;"; |
- const char *HeaderBottom = "\n};\n#endif\n"; |
- const char *MFile = "#include \"HeaderFile.h\"\nint main() {" |
-""" |
- ] |
- |
- # This Go bindings test doesn't work after bootstrap on Linux, PR21552. |
- llvm_patches = [ r"""\ |
---- test/Bindings/Go/go.test (revision 223109) |
-+++ test/Bindings/Go/go.test (working copy) |
-@@ -1,3 +1,3 @@ |
--; RUN: llvm-go test llvm.org/llvm/bindings/go/llvm |
-+; RUN: true |
- |
- ; REQUIRES: shell |
-""" |
- ] |
- |
- # The UBSan run-time, which is now bundled with the ASan run-time, doesn't |
- # work on Mac OS X 10.8 (PR23539). |
- compiler_rt_patches = [ r"""\ |
---- CMakeLists.txt (revision 241602) |
-+++ CMakeLists.txt (working copy) |
-@@ -305,6 +305,7 @@ |
- list(APPEND SANITIZER_COMMON_SUPPORTED_OS iossim) |
- endif() |
- endif() |
-+ set(SANITIZER_MIN_OSX_VERSION "10.7") |
- if(SANITIZER_MIN_OSX_VERSION VERSION_LESS "10.7") |
- message(FATAL_ERROR "Too old OS X version: ${SANITIZER_MIN_OSX_VERSION}") |
- endif() |
-""" |
- ] |
- |
- for path, patches in [(LLVM_DIR, llvm_patches), |
- (CLANG_DIR, clang_patches), |
- (COMPILER_RT_DIR, compiler_rt_patches)]: |
- print 'Applying patches in', path |
- for patch in patches: |
- print patch |
- p = subprocess.Popen( ['patch', '-p0', '-d', path], stdin=subprocess.PIPE) |
- (stdout, stderr) = p.communicate(input=patch) |
- if p.returncode != 0: |
- raise RuntimeError('stdout %s, stderr %s' % (stdout, stderr)) |
- |
- |
def DeleteChromeToolsShim(): |
+ OLD_SHIM_DIR = os.path.join(LLVM_DIR, 'tools', 'zzz-chrometools') |
+ shutil.rmtree(OLD_SHIM_DIR, ignore_errors=True) |
shutil.rmtree(CHROME_TOOLS_SHIM_DIR, ignore_errors=True) |
@@ -337,30 +298,53 @@ def CreateChromeToolsShim(): |
f.write('endif (CHROMIUM_TOOLS_SRC)\n') |
+def DownloadHostGcc(args): |
+ """Downloads gcc 4.8.5 and makes sure args.gcc_toolchain is set.""" |
+ if not sys.platform.startswith('linux') or args.gcc_toolchain: |
+ return |
+ # Unconditionally download a prebuilt gcc to guarantee the included libstdc++ |
+ # works on Ubuntu Precise. |
+ gcc_dir = os.path.join(LLVM_BUILD_TOOLS_DIR, 'gcc485precise') |
+ if not os.path.exists(gcc_dir): |
+ print 'Downloading pre-built GCC 4.8.5...' |
+ DownloadAndUnpack( |
+ CDS_URL + '/tools/gcc485precise.tgz', LLVM_BUILD_TOOLS_DIR) |
+ args.gcc_toolchain = gcc_dir |
+ |
+ |
def AddCMakeToPath(): |
"""Download CMake and add it to PATH.""" |
if sys.platform == 'win32': |
- zip_name = 'cmake-3.2.2-win32-x86.zip' |
+ zip_name = 'cmake-3.4.3-win32-x86.zip' |
cmake_dir = os.path.join(LLVM_BUILD_TOOLS_DIR, |
- 'cmake-3.2.2-win32-x86', 'bin') |
+ 'cmake-3.4.3-win32-x86', 'bin') |
else: |
suffix = 'Darwin' if sys.platform == 'darwin' else 'Linux' |
- zip_name = 'cmake310_%s.tgz' % suffix |
- cmake_dir = os.path.join(LLVM_BUILD_TOOLS_DIR, 'cmake310', 'bin') |
+ zip_name = 'cmake343_%s.tgz' % suffix |
+ cmake_dir = os.path.join(LLVM_BUILD_TOOLS_DIR, 'cmake343', 'bin') |
if not os.path.exists(cmake_dir): |
- if not os.path.exists(LLVM_BUILD_TOOLS_DIR): |
- os.makedirs(LLVM_BUILD_TOOLS_DIR) |
- # The cmake archive is smaller than 20 MB, small enough to keep in memory: |
- with contextlib.closing(cStringIO.StringIO()) as f: |
- DownloadUrl(CDS_URL + '/tools/' + zip_name, f) |
- f.seek(0) |
- if zip_name.endswith('.zip'): |
- zipfile.ZipFile(f).extractall(path=LLVM_BUILD_TOOLS_DIR) |
- else: |
- tarfile.open(mode='r:gz', fileobj=f).extractall(path= |
- LLVM_BUILD_TOOLS_DIR) |
+ DownloadAndUnpack(CDS_URL + '/tools/' + zip_name, LLVM_BUILD_TOOLS_DIR) |
os.environ['PATH'] = cmake_dir + os.pathsep + os.environ.get('PATH', '') |
+ |
+def AddGnuWinToPath(): |
+ """Download some GNU win tools and add them to PATH.""" |
+ if sys.platform != 'win32': |
+ return |
+ |
+ gnuwin_dir = os.path.join(LLVM_BUILD_TOOLS_DIR, 'gnuwin') |
+ GNUWIN_VERSION = '5' |
+ GNUWIN_STAMP = os.path.join(gnuwin_dir, 'stamp') |
+ if ReadStampFile(GNUWIN_STAMP) == GNUWIN_VERSION: |
+ print 'GNU Win tools already up to date.' |
+ else: |
+ zip_name = 'gnuwin-%s.zip' % GNUWIN_VERSION |
+ DownloadAndUnpack(CDS_URL + '/tools/' + zip_name, LLVM_BUILD_TOOLS_DIR) |
+ WriteStampFile(GNUWIN_VERSION, GNUWIN_STAMP) |
+ |
+ os.environ['PATH'] = gnuwin_dir + os.pathsep + os.environ.get('PATH', '') |
+ |
+ |
vs_version = None |
def GetVSVersion(): |
global vs_version |
@@ -377,99 +361,152 @@ def GetVSVersion(): |
# or a system-wide installation otherwise. |
sys.path.append(os.path.join(CHROMIUM_DIR, 'tools', 'gyp', 'pylib')) |
import gyp.MSVSVersion |
- vs_version = gyp.MSVSVersion.SelectVisualStudioVersion('2013') |
+ vs_version = gyp.MSVSVersion.SelectVisualStudioVersion( |
+ vs_toolchain.GetVisualStudioVersion()) |
return vs_version |
+def CopyDiaDllTo(target_dir): |
+ # This script always wants to use the 64-bit msdia*.dll. |
+ dia_path = os.path.join(GetVSVersion().Path(), 'DIA SDK', 'bin', 'amd64') |
+ dia_dll = os.path.join(dia_path, DIA_DLL[GetVSVersion().ShortName()]) |
+ CopyFile(dia_dll, target_dir) |
+ |
+ |
+def VeryifyVersionOfBuiltClangMatchesVERSION(): |
+ """Checks that `clang --version` outputs VERSION. If this fails, VERSION |
+ in this file is out-of-date and needs to be updated (possibly in an |
+ `if use_head_revision:` block in main() first).""" |
+ clang = os.path.join(LLVM_BUILD_DIR, 'bin', 'clang') |
+ if sys.platform == 'win32': |
+ # TODO: Parse `clang-cl /?` output for built clang's version and check that |
+ # to check the binary we're actually shipping? But clang-cl.exe is just |
+ # a copy of clang.exe, so this does check the same thing. |
+ clang += '.exe' |
+ version_out = subprocess.check_output([clang, '--version']) |
+ version_out = re.match(r'clang version ([0-9.]+)', version_out).group(1) |
+ if version_out != VERSION: |
+ print ('unexpected clang version %s (not %s), update VERSION in update.py' |
+ % (version_out, VERSION)) |
+ sys.exit(1) |
+ |
+ |
def UpdateClang(args): |
print 'Updating Clang to %s...' % PACKAGE_VERSION |
- if ReadStampFile() == PACKAGE_VERSION: |
- print 'Already up to date.' |
- return 0 |
+ |
+ need_gold_plugin = 'LLVM_DOWNLOAD_GOLD_PLUGIN' in os.environ or ( |
+ sys.platform.startswith('linux') and |
+ 'buildtype=Official' in os.environ.get('GYP_DEFINES', '') and |
+ 'branding=Chrome' in os.environ.get('GYP_DEFINES', '')) |
+ |
+ if ReadStampFile() == PACKAGE_VERSION and not args.force_local_build: |
+ print 'Clang is already up to date.' |
+ if not need_gold_plugin or os.path.exists( |
+ os.path.join(LLVM_BUILD_DIR, "lib/LLVMgold.so")): |
+ return 0 |
# Reset the stamp file in case the build is unsuccessful. |
WriteStampFile('') |
if not args.force_local_build: |
cds_file = "clang-%s.tgz" % PACKAGE_VERSION |
- cds_full_url = CDS_URL + '/Win/' + cds_file |
- |
- # Check if there's a prebuilt binary and if so just fetch that. That's |
- # faster, and goma relies on having matching binary hashes on client and |
- # server too. |
- print 'Trying to download prebuilt clang' |
- |
- # clang packages are smaller than 50 MB, small enough to keep in memory. |
- with contextlib.closing(cStringIO.StringIO()) as f: |
- try: |
- DownloadUrl(cds_full_url, f) |
- f.seek(0) |
- tarfile.open(mode='r:gz', fileobj=f).extractall(path=LLVM_BUILD_DIR) |
- print 'clang %s unpacked' % PACKAGE_VERSION |
- # Download the gold plugin if requested to by an environment variable. |
- # This is used by the CFI ClusterFuzz bot. |
- if 'LLVM_DOWNLOAD_GOLD_PLUGIN' in os.environ: |
- RunCommand(['python', CHROMIUM_DIR+'/build/download_gold_plugin.py']) |
- WriteStampFile(PACKAGE_VERSION) |
- return 0 |
- except urllib2.HTTPError: |
- print 'Did not find prebuilt clang %s, building locally' % cds_file |
- |
+ if sys.platform == 'win32' or sys.platform == 'cygwin': |
+ cds_full_url = CDS_URL + '/Win/' + cds_file |
+ elif sys.platform == 'darwin': |
+ cds_full_url = CDS_URL + '/Mac/' + cds_file |
+ else: |
+ assert sys.platform.startswith('linux') |
+ cds_full_url = CDS_URL + '/Linux_x64/' + cds_file |
+ |
+ print 'Downloading prebuilt clang' |
+ if os.path.exists(LLVM_BUILD_DIR): |
+ RmTree(LLVM_BUILD_DIR) |
+ try: |
+ DownloadAndUnpack(cds_full_url, LLVM_BUILD_DIR) |
+ print 'clang %s unpacked' % PACKAGE_VERSION |
+ if sys.platform == 'win32': |
+ CopyDiaDllTo(os.path.join(LLVM_BUILD_DIR, 'bin')) |
+ # Download the gold plugin if requested to by an environment variable. |
+ # This is used by the CFI ClusterFuzz bot, and it's required for official |
+ # builds on linux. |
+ if need_gold_plugin: |
+ RunCommand(['python', CHROMIUM_DIR+'/build/download_gold_plugin.py']) |
+ WriteStampFile(PACKAGE_VERSION) |
+ return 0 |
+ except urllib2.URLError: |
+ print 'Failed to download prebuilt clang %s' % cds_file |
+ print 'Use --force-local-build if you want to build locally.' |
+ print 'Exiting.' |
+ return 1 |
+ |
+ if args.with_android and not os.path.exists(ANDROID_NDK_DIR): |
+ print 'Android NDK not found at ' + ANDROID_NDK_DIR |
+ print 'The Android NDK is needed to build a Clang whose -fsanitize=address' |
+ print 'works on Android. See ' |
+ print 'https://www.chromium.org/developers/how-tos/android-build-instructions' |
+ print 'for how to install the NDK, or pass --without-android.' |
+ return 1 |
+ |
+ DownloadHostGcc(args) |
AddCMakeToPath() |
+ AddGnuWinToPath() |
- RevertPreviouslyPatchedFiles() |
DeleteChromeToolsShim() |
Checkout('LLVM', LLVM_REPO_URL + '/llvm/trunk', LLVM_DIR) |
Checkout('Clang', LLVM_REPO_URL + '/cfe/trunk', CLANG_DIR) |
- if sys.platform == 'win32': |
+ if sys.platform == 'win32' or use_head_revision: |
Checkout('LLD', LLVM_REPO_URL + '/lld/trunk', LLD_DIR) |
Checkout('compiler-rt', LLVM_REPO_URL + '/compiler-rt/trunk', COMPILER_RT_DIR) |
if sys.platform == 'darwin': |
# clang needs a libc++ checkout, else -stdlib=libc++ won't find includes |
# (i.e. this is needed for bootstrap builds). |
Checkout('libcxx', LLVM_REPO_URL + '/libcxx/trunk', LIBCXX_DIR) |
- # While we're bundling our own libc++ on OS X, we need to compile libc++abi |
- # into it too (since OS X 10.6 doesn't have libc++abi.dylib either). |
- Checkout('libcxxabi', LLVM_REPO_URL + '/libcxxabi/trunk', LIBCXXABI_DIR) |
- |
- if args.with_patches and sys.platform != 'win32': |
- ApplyLocalPatches() |
+ # We used to check out libcxxabi on OS X; we no longer need that. |
+ if os.path.exists(LIBCXXABI_DIR): |
+ RmTree(LIBCXXABI_DIR) |
cc, cxx = None, None |
- cflags = cxxflags = ldflags = [] |
- |
- # LLVM uses C++11 starting in llvm 3.5. On Linux, this means libstdc++4.7+ is |
- # needed, on OS X it requires libc++. clang only automatically links to libc++ |
- # when targeting OS X 10.9+, so add stdlib=libc++ explicitly so clang can run |
- # on OS X versions as old as 10.7. |
- # TODO(thakis): Some bots are still on 10.6 (nacl...), so for now bundle |
- # libc++.dylib. Remove this once all bots are on 10.7+, then use |
- # -DLLVM_ENABLE_LIBCXX=ON and change deployment_target to 10.7. |
- deployment_target = '' |
- |
- if sys.platform == 'darwin': |
- # When building on 10.9, /usr/include usually doesn't exist, and while |
- # Xcode's clang automatically sets a sysroot, self-built clangs don't. |
- cflags = ['-isysroot', subprocess.check_output( |
- ['xcrun', '--show-sdk-path']).rstrip()] |
- cxxflags = ['-stdlib=libc++', '-nostdinc++', |
- '-I' + os.path.join(LIBCXX_DIR, 'include')] + cflags |
- if args.bootstrap: |
- deployment_target = '10.6' |
+ libstdcpp = None |
+ if args.gcc_toolchain: # This option is only used on Linux. |
+ # Use the specified gcc installation for building. |
+ cc = os.path.join(args.gcc_toolchain, 'bin', 'gcc') |
+ cxx = os.path.join(args.gcc_toolchain, 'bin', 'g++') |
+ |
+ if not os.access(cc, os.X_OK): |
+ print 'Invalid --gcc-toolchain: "%s"' % args.gcc_toolchain |
+ print '"%s" does not appear to be valid.' % cc |
+ return 1 |
+ |
+ # Set LD_LIBRARY_PATH to make auxiliary targets (tablegen, bootstrap |
+ # compiler, etc.) find the .so. |
+ libstdcpp = subprocess.check_output( |
+ [cxx, '-print-file-name=libstdc++.so.6']).rstrip() |
+ os.environ['LD_LIBRARY_PATH'] = os.path.dirname(libstdcpp) |
+ |
+ cflags = [] |
+ cxxflags = [] |
+ ldflags = [] |
base_cmake_args = ['-GNinja', |
'-DCMAKE_BUILD_TYPE=Release', |
'-DLLVM_ENABLE_ASSERTIONS=ON', |
'-DLLVM_ENABLE_THREADS=OFF', |
+ '-DLLVM_ENABLE_TIMESTAMPS=OFF', |
+ # Statically link MSVCRT to avoid DLL dependencies. |
+ '-DLLVM_USE_CRT_RELEASE=MT', |
] |
+ binutils_incdir = '' |
+ if sys.platform.startswith('linux'): |
+ binutils_incdir = os.path.join(BINUTILS_DIR, 'Linux_x64/Release/include') |
+ |
if args.bootstrap: |
print 'Building bootstrap compiler' |
- if not os.path.exists(LLVM_BOOTSTRAP_DIR): |
- os.makedirs(LLVM_BOOTSTRAP_DIR) |
+ EnsureDirExists(LLVM_BOOTSTRAP_DIR) |
os.chdir(LLVM_BOOTSTRAP_DIR) |
bootstrap_args = base_cmake_args + [ |
+ '-DLLVM_BINUTILS_INCDIR=' + binutils_incdir, |
'-DLLVM_TARGETS_TO_BUILD=host', |
'-DCMAKE_INSTALL_PREFIX=' + LLVM_BOOTSTRAP_INSTALL_DIR, |
'-DCMAKE_C_FLAGS=' + ' '.join(cflags), |
@@ -477,11 +514,18 @@ def UpdateClang(args): |
] |
if cc is not None: bootstrap_args.append('-DCMAKE_C_COMPILER=' + cc) |
if cxx is not None: bootstrap_args.append('-DCMAKE_CXX_COMPILER=' + cxx) |
+ RmCmakeCache('.') |
RunCommand(['cmake'] + bootstrap_args + [LLVM_DIR], msvc_arch='x64') |
RunCommand(['ninja'], msvc_arch='x64') |
if args.run_tests: |
+ if sys.platform == 'win32': |
+ CopyDiaDllTo(os.path.join(LLVM_BOOTSTRAP_DIR, 'bin')) |
RunCommand(['ninja', 'check-all'], msvc_arch='x64') |
RunCommand(['ninja', 'install'], msvc_arch='x64') |
+ if args.gcc_toolchain: |
+ # Copy that gcc's stdlibc++.so.6 to the build dir, so the bootstrap |
+ # compiler can start. |
+ CopyFile(libstdcpp, os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'lib')) |
if sys.platform == 'win32': |
cc = os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'bin', 'clang-cl.exe') |
@@ -493,61 +537,78 @@ def UpdateClang(args): |
else: |
cc = os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'bin', 'clang') |
cxx = os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'bin', 'clang++') |
+ |
+ if args.gcc_toolchain: |
+ # Tell the bootstrap compiler to use a specific gcc prefix to search |
+ # for standard library headers and shared object files. |
+ cflags = ['--gcc-toolchain=' + args.gcc_toolchain] |
+ cxxflags = ['--gcc-toolchain=' + args.gcc_toolchain] |
print 'Building final compiler' |
- if sys.platform == 'darwin': |
- # Build libc++.dylib while some bots are still on OS X 10.6. |
- libcxxbuild = os.path.join(LLVM_BUILD_DIR, 'libcxxbuild') |
- if os.path.isdir(libcxxbuild): |
- RmTree(libcxxbuild) |
- libcxxflags = ['-O3', '-std=c++11', '-fstrict-aliasing'] |
- |
- # libcxx and libcxxabi both have a file stdexcept.cpp, so put their .o files |
- # into different subdirectories. |
- os.makedirs(os.path.join(libcxxbuild, 'libcxx')) |
- os.chdir(os.path.join(libcxxbuild, 'libcxx')) |
- RunCommand(['c++', '-c'] + cxxflags + libcxxflags + |
- glob.glob(os.path.join(LIBCXX_DIR, 'src', '*.cpp'))) |
- |
- os.makedirs(os.path.join(libcxxbuild, 'libcxxabi')) |
- os.chdir(os.path.join(libcxxbuild, 'libcxxabi')) |
- RunCommand(['c++', '-c'] + cxxflags + libcxxflags + |
- glob.glob(os.path.join(LIBCXXABI_DIR, 'src', '*.cpp')) + |
- ['-I' + os.path.join(LIBCXXABI_DIR, 'include')]) |
- |
- os.chdir(libcxxbuild) |
- libdir = os.path.join(LIBCXX_DIR, 'lib') |
- RunCommand(['cc'] + glob.glob('libcxx/*.o') + glob.glob('libcxxabi/*.o') + |
- ['-o', 'libc++.1.dylib', '-dynamiclib', '-nodefaultlibs', |
- '-current_version', '1', '-compatibility_version', '1', '-lSystem', |
- '-install_name', '@executable_path/libc++.dylib', |
- '-Wl,-unexported_symbols_list,' + libdir + '/libc++unexp.exp', |
- '-Wl,-force_symbols_not_weak_list,' + libdir + '/notweak.exp', |
- '-Wl,-force_symbols_weak_list,' + libdir + '/weak.exp']) |
- if os.path.exists('libc++.dylib'): |
- os.remove('libc++.dylib') |
- os.symlink('libc++.1.dylib', 'libc++.dylib') |
- ldflags += ['-stdlib=libc++', '-L' + libcxxbuild] |
- |
- if args.bootstrap: |
- # Now that the libc++ headers have been installed and libc++.dylib is |
- # built, delete the libc++ checkout again so that it's not part of the |
- # main build below -- the libc++(abi) tests don't pass on OS X in |
- # bootstrap builds (http://llvm.org/PR24068) |
- RmTree(LIBCXX_DIR) |
- RmTree(LIBCXXABI_DIR) |
- cxxflags = ['-stdlib=libc++', '-nostdinc++', |
- '-I' + os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, |
- 'include/c++/v1') |
- ] + cflags |
+ # Build LLVM gold plugin with LTO. That speeds up the linker by ~10%. |
+ # We only use LTO for Linux now. |
+ if args.bootstrap and args.lto_gold_plugin: |
+ print 'Building LTO LLVM Gold plugin' |
+ if os.path.exists(LLVM_LTO_GOLD_PLUGIN_DIR): |
+ RmTree(LLVM_LTO_GOLD_PLUGIN_DIR) |
+ EnsureDirExists(LLVM_LTO_GOLD_PLUGIN_DIR) |
+ os.chdir(LLVM_LTO_GOLD_PLUGIN_DIR) |
+ |
+ # Create a symlink to LLVMgold.so build in the previous step so that ar |
+ # and ranlib could find it while linking LLVMgold.so with LTO. |
+ EnsureDirExists(BFD_PLUGINS_DIR) |
+ RunCommand(['ln', '-sf', |
+ os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'lib', 'LLVMgold.so'), |
+ os.path.join(BFD_PLUGINS_DIR, 'LLVMgold.so')]) |
+ |
+ lto_cflags = ['-flto'] |
+ lto_ldflags = ['-fuse-ld=gold'] |
+ if args.gcc_toolchain: |
+ # Tell the bootstrap compiler to use a specific gcc prefix to search |
+ # for standard library headers and shared object files. |
+ lto_cflags += ['--gcc-toolchain=' + args.gcc_toolchain] |
+ lto_cmake_args = base_cmake_args + [ |
+ '-DLLVM_BINUTILS_INCDIR=' + binutils_incdir, |
+ '-DCMAKE_C_COMPILER=' + cc, |
+ '-DCMAKE_CXX_COMPILER=' + cxx, |
+ '-DCMAKE_C_FLAGS=' + ' '.join(lto_cflags), |
+ '-DCMAKE_CXX_FLAGS=' + ' '.join(lto_cflags), |
+ '-DCMAKE_EXE_LINKER_FLAGS=' + ' '.join(lto_ldflags), |
+ '-DCMAKE_SHARED_LINKER_FLAGS=' + ' '.join(lto_ldflags), |
+ '-DCMAKE_MODULE_LINKER_FLAGS=' + ' '.join(lto_ldflags)] |
+ |
+ # We need to use the proper binutils which support LLVM Gold plugin. |
+ lto_env = os.environ.copy() |
+ lto_env['PATH'] = BINUTILS_BIN_DIR + os.pathsep + lto_env.get('PATH', '') |
+ |
+ RmCmakeCache('.') |
+ RunCommand(['cmake'] + lto_cmake_args + [LLVM_DIR], env=lto_env) |
+ RunCommand(['ninja', 'LLVMgold'], env=lto_env) |
+ |
+ |
+ # LLVM uses C++11 starting in llvm 3.5. On Linux, this means libstdc++4.7+ is |
+ # needed, on OS X it requires libc++. clang only automatically links to libc++ |
+ # when targeting OS X 10.9+, so add stdlib=libc++ explicitly so clang can run |
+ # on OS X versions as old as 10.7. |
+ deployment_target = '' |
+ |
+ if sys.platform == 'darwin' and args.bootstrap: |
+ # When building on 10.9, /usr/include usually doesn't exist, and while |
+ # Xcode's clang automatically sets a sysroot, self-built clangs don't. |
+ cflags = ['-isysroot', subprocess.check_output( |
+ ['xcrun', '--show-sdk-path']).rstrip()] |
+ cxxflags = ['-stdlib=libc++'] + cflags |
+ ldflags += ['-stdlib=libc++'] |
+ deployment_target = '10.7' |
+ # Running libc++ tests takes a long time. Since it was only needed for |
+ # the install step above, don't build it as part of the main build. |
+ # This makes running package.py over 10% faster (30 min instead of 34 min) |
+ RmTree(LIBCXX_DIR) |
# Build clang. |
- binutils_incdir = '' |
- if sys.platform.startswith('linux'): |
- binutils_incdir = os.path.join(BINUTILS_DIR, 'Linux_x64/Release/include') |
# If building at head, define a macro that plugins can use for #ifdefing |
- # out code that builds at head, but not at LLVM_WIN_REVISION or vice versa. |
+ # out code that builds at head, but not at CLANG_REVISION or vice versa. |
if use_head_revision: |
cflags += ['-DLLVM_FORCE_HEAD_REVISION'] |
cxxflags += ['-DLLVM_FORCE_HEAD_REVISION'] |
@@ -559,7 +620,13 @@ def UpdateClang(args): |
deployment_env = os.environ.copy() |
deployment_env['MACOSX_DEPLOYMENT_TARGET'] = deployment_target |
- cmake_args = base_cmake_args + [ |
+ cmake_args = [] |
+ # TODO(thakis): Unconditionally append this to base_cmake_args instead once |
+ # compiler-rt can build with clang-cl on Windows (http://llvm.org/PR23698) |
+ cc_args = base_cmake_args if sys.platform != 'win32' else cmake_args |
+ if cc is not None: cc_args.append('-DCMAKE_C_COMPILER=' + cc) |
+ if cxx is not None: cc_args.append('-DCMAKE_CXX_COMPILER=' + cxx) |
+ cmake_args += base_cmake_args + [ |
'-DLLVM_BINUTILS_INCDIR=' + binutils_incdir, |
'-DCMAKE_C_FLAGS=' + ' '.join(cflags), |
'-DCMAKE_CXX_FLAGS=' + ' '.join(cxxflags), |
@@ -567,19 +634,26 @@ def UpdateClang(args): |
'-DCMAKE_SHARED_LINKER_FLAGS=' + ' '.join(ldflags), |
'-DCMAKE_MODULE_LINKER_FLAGS=' + ' '.join(ldflags), |
'-DCMAKE_INSTALL_PREFIX=' + LLVM_BUILD_DIR, |
+ # TODO(thakis): Remove this once official builds pass -Wl,--build-id |
+ # explicitly, https://crbug.com/622775 |
+ '-DENABLE_LINKER_BUILD_ID=ON', |
'-DCHROMIUM_TOOLS_SRC=%s' % os.path.join(CHROMIUM_DIR, 'tools', 'clang'), |
'-DCHROMIUM_TOOLS=%s' % ';'.join(args.tools)] |
- # TODO(thakis): Unconditionally append this to base_cmake_args instead once |
- # compiler-rt can build with clang-cl on Windows (http://llvm.org/PR23698) |
- cc_args = base_cmake_args if sys.platform != 'win32' else cmake_args |
- if cc is not None: cc_args.append('-DCMAKE_C_COMPILER=' + cc) |
- if cxx is not None: cc_args.append('-DCMAKE_CXX_COMPILER=' + cxx) |
- if not os.path.exists(LLVM_BUILD_DIR): |
- os.makedirs(LLVM_BUILD_DIR) |
+ EnsureDirExists(LLVM_BUILD_DIR) |
os.chdir(LLVM_BUILD_DIR) |
+ RmCmakeCache('.') |
RunCommand(['cmake'] + cmake_args + [LLVM_DIR], |
msvc_arch='x64', env=deployment_env) |
+ |
+ if args.gcc_toolchain: |
+ # Copy in the right stdlibc++.so.6 so clang can start. |
+ if not os.path.exists(os.path.join(LLVM_BUILD_DIR, 'lib')): |
+ os.mkdir(os.path.join(LLVM_BUILD_DIR, 'lib')) |
+ libstdcpp = subprocess.check_output( |
+ [cxx] + cxxflags + ['-print-file-name=libstdc++.so.6']).rstrip() |
+ CopyFile(libstdcpp, os.path.join(LLVM_BUILD_DIR, 'lib')) |
+ |
RunCommand(['ninja'], msvc_arch='x64') |
if args.tools: |
@@ -587,17 +661,21 @@ def UpdateClang(args): |
RunCommand(['ninja', 'cr-install'], msvc_arch='x64') |
if sys.platform == 'darwin': |
- CopyFile(os.path.join(LLVM_BUILD_DIR, 'libc++.1.dylib'), |
- os.path.join(LLVM_BUILD_DIR, 'bin')) |
# See http://crbug.com/256342 |
RunCommand(['strip', '-x', os.path.join(LLVM_BUILD_DIR, 'bin', 'clang')]) |
elif sys.platform.startswith('linux'): |
RunCommand(['strip', os.path.join(LLVM_BUILD_DIR, 'bin', 'clang')]) |
- # Do an x86 build of compiler-rt to get the 32-bit ASan run-time. |
+ VeryifyVersionOfBuiltClangMatchesVERSION() |
+ |
+ # Do an out-of-tree build of compiler-rt. |
+ # On Windows, this is used to get the 32-bit ASan run-time. |
# TODO(hans): Remove once the regular build above produces this. |
- if not os.path.exists(COMPILER_RT_BUILD_DIR): |
- os.makedirs(COMPILER_RT_BUILD_DIR) |
+ # On Mac and Linux, this is used to get the regular 64-bit run-time. |
+ # Do a clobbered build due to cmake changes. |
+ if os.path.isdir(COMPILER_RT_BUILD_DIR): |
+ RmTree(COMPILER_RT_BUILD_DIR) |
+ os.makedirs(COMPILER_RT_BUILD_DIR) |
os.chdir(COMPILER_RT_BUILD_DIR) |
# TODO(thakis): Add this once compiler-rt can build with clang-cl (see |
# above). |
@@ -608,13 +686,21 @@ def UpdateClang(args): |
compiler_rt_args = base_cmake_args + [ |
'-DCMAKE_C_FLAGS=' + ' '.join(cflags), |
'-DCMAKE_CXX_FLAGS=' + ' '.join(cxxflags)] |
+ if sys.platform == 'darwin': |
+ compiler_rt_args += ['-DCOMPILER_RT_ENABLE_IOS=ON'] |
if sys.platform != 'win32': |
compiler_rt_args += ['-DLLVM_CONFIG_PATH=' + |
- os.path.join(LLVM_BUILD_DIR, 'bin', 'llvm-config')] |
- RunCommand(['cmake'] + compiler_rt_args + [LLVM_DIR], |
- msvc_arch='x86', env=deployment_env) |
+ os.path.join(LLVM_BUILD_DIR, 'bin', 'llvm-config'), |
+ '-DSANITIZER_MIN_OSX_VERSION="10.7"'] |
+ # compiler-rt is part of the llvm checkout on Windows but a stand-alone |
+ # directory elsewhere, see the TODO above COMPILER_RT_DIR. |
+ RmCmakeCache('.') |
+ RunCommand(['cmake'] + compiler_rt_args + |
+ [LLVM_DIR if sys.platform == 'win32' else COMPILER_RT_DIR], |
+ msvc_arch='x86', env=deployment_env) |
RunCommand(['ninja', 'compiler-rt'], msvc_arch='x86') |
+ # Copy select output to the main tree. |
# TODO(hans): Make this (and the .gypi and .isolate files) version number |
# independent. |
if sys.platform == 'win32': |
@@ -624,17 +710,35 @@ def UpdateClang(args): |
else: |
assert sys.platform.startswith('linux') |
platform = 'linux' |
- asan_rt_lib_src_dir = os.path.join(COMPILER_RT_BUILD_DIR, 'lib', 'clang', |
- VERSION, 'lib', platform) |
+ asan_rt_lib_src_dir = os.path.join(COMPILER_RT_BUILD_DIR, 'lib', platform) |
+ if sys.platform == 'win32': |
+ # TODO(thakis): This too is due to compiler-rt being part of the checkout |
+ # on Windows, see TODO above COMPILER_RT_DIR. |
+ asan_rt_lib_src_dir = os.path.join(COMPILER_RT_BUILD_DIR, 'lib', 'clang', |
+ VERSION, 'lib', platform) |
asan_rt_lib_dst_dir = os.path.join(LLVM_BUILD_DIR, 'lib', 'clang', |
VERSION, 'lib', platform) |
- CopyDirectoryContents(asan_rt_lib_src_dir, asan_rt_lib_dst_dir, |
- r'^.*-i386\.lib$') |
- CopyDirectoryContents(asan_rt_lib_src_dir, asan_rt_lib_dst_dir, |
- r'^.*-i386\.dll$') |
+ # Blacklists: |
+ CopyDirectoryContents(os.path.join(asan_rt_lib_src_dir, '..', '..'), |
+ os.path.join(asan_rt_lib_dst_dir, '..', '..'), |
+ r'^.*blacklist\.txt$') |
+ # Headers: |
+ if sys.platform != 'win32': |
+ CopyDirectoryContents( |
+ os.path.join(COMPILER_RT_BUILD_DIR, 'include/sanitizer'), |
+ os.path.join(LLVM_BUILD_DIR, 'lib/clang', VERSION, 'include/sanitizer')) |
+ # Static and dynamic libraries: |
+ CopyDirectoryContents(asan_rt_lib_src_dir, asan_rt_lib_dst_dir) |
+ if sys.platform == 'darwin': |
+ for dylib in glob.glob(os.path.join(asan_rt_lib_dst_dir, '*.dylib')): |
+ # Fix LC_ID_DYLIB for the ASan dynamic libraries to be relative to |
+ # @executable_path. |
+ # TODO(glider): this is transitional. We'll need to fix the dylib |
+ # name either in our build system, or in Clang. See also |
+ # http://crbug.com/344836. |
+ subprocess.call(['install_name_tool', '-id', |
+ '@executable_path/' + os.path.basename(dylib), dylib]) |
- CopyFile(os.path.join(asan_rt_lib_src_dir, '..', '..', 'asan_blacklist.txt'), |
- os.path.join(asan_rt_lib_dst_dir, '..', '..')) |
if sys.platform == 'win32': |
# Make an extra copy of the sanitizer headers, to be put on the include path |
@@ -644,22 +748,69 @@ def UpdateClang(args): |
aux_sanitizer_include_dir = os.path.join(LLVM_BUILD_DIR, 'lib', 'clang', |
VERSION, 'include_sanitizer', |
'sanitizer') |
- if not os.path.exists(aux_sanitizer_include_dir): |
- os.makedirs(aux_sanitizer_include_dir) |
+ EnsureDirExists(aux_sanitizer_include_dir) |
for _, _, files in os.walk(sanitizer_include_dir): |
for f in files: |
CopyFile(os.path.join(sanitizer_include_dir, f), |
aux_sanitizer_include_dir) |
+ if args.with_android: |
+ make_toolchain = os.path.join( |
+ ANDROID_NDK_DIR, 'build', 'tools', 'make_standalone_toolchain.py') |
+ for target_arch in ['aarch64', 'arm', 'i686']: |
+ # Make standalone Android toolchain for target_arch. |
+ toolchain_dir = os.path.join( |
+ LLVM_BUILD_DIR, 'android-toolchain-' + target_arch) |
+ RunCommand([ |
+ make_toolchain, |
+ '--api=' + ('21' if target_arch == 'aarch64' else '19'), |
+ '--force', |
+ '--install-dir=%s' % toolchain_dir, |
+ '--stl=stlport', |
+ '--arch=' + { |
+ 'aarch64': 'arm64', |
+ 'arm': 'arm', |
+ 'i686': 'x86', |
+ }[target_arch]]) |
+ # Android NDK r9d copies a broken unwind.h into the toolchain, see |
+ # http://crbug.com/357890 |
+ for f in glob.glob(os.path.join(toolchain_dir, 'include/c++/*/unwind.h')): |
+ os.remove(f) |
+ |
+ # Build ASan runtime for Android in a separate build tree. |
+ build_dir = os.path.join(LLVM_BUILD_DIR, 'android-' + target_arch) |
+ if not os.path.exists(build_dir): |
+ os.mkdir(os.path.join(build_dir)) |
+ os.chdir(build_dir) |
+ cflags = ['--target=%s-linux-androideabi' % target_arch, |
+ '--sysroot=%s/sysroot' % toolchain_dir, |
+ '-B%s' % toolchain_dir] |
+ android_args = base_cmake_args + [ |
+ '-DCMAKE_C_COMPILER=' + os.path.join(LLVM_BUILD_DIR, 'bin/clang'), |
+ '-DCMAKE_CXX_COMPILER=' + os.path.join(LLVM_BUILD_DIR, 'bin/clang++'), |
+ '-DLLVM_CONFIG_PATH=' + os.path.join(LLVM_BUILD_DIR, 'bin/llvm-config'), |
+ '-DCMAKE_C_FLAGS=' + ' '.join(cflags), |
+ '-DCMAKE_CXX_FLAGS=' + ' '.join(cflags), |
+ '-DANDROID=1'] |
+ RmCmakeCache('.') |
+ RunCommand(['cmake'] + android_args + [COMPILER_RT_DIR]) |
+ RunCommand(['ninja', 'libclang_rt.asan-%s-android.so' % target_arch]) |
+ |
+ # And copy it into the main build tree. |
+ runtime = 'libclang_rt.asan-%s-android.so' % target_arch |
+ for root, _, files in os.walk(build_dir): |
+ if runtime in files: |
+ shutil.copy(os.path.join(root, runtime), asan_rt_lib_dst_dir) |
+ |
# Run tests. |
if args.run_tests or use_head_revision: |
os.chdir(LLVM_BUILD_DIR) |
- RunCommand(GetVSVersion().SetupScript('x64') + |
- ['&&', 'ninja', 'cr-check-all']) |
+ RunCommand(['ninja', 'cr-check-all'], msvc_arch='x64') |
if args.run_tests: |
+ if sys.platform == 'win32': |
+ CopyDiaDllTo(os.path.join(LLVM_BUILD_DIR, 'bin')) |
os.chdir(LLVM_BUILD_DIR) |
- RunCommand(GetVSVersion().SetupScript('x64') + |
- ['&&', 'ninja', 'check-all']) |
+ RunCommand(['ninja', 'check-all'], msvc_arch='x64') |
WriteStampFile(PACKAGE_VERSION) |
print 'Clang update was successful.' |
@@ -667,31 +818,6 @@ def UpdateClang(args): |
def main(): |
- if not sys.platform in ['win32', 'cygwin']: |
- # For non-Windows, fall back to update.sh. |
- # TODO(hans): Make update.py replace update.sh completely. |
- |
- # This script is called by gclient. gclient opens its hooks subprocesses |
- # with (stdout=subprocess.PIPE, stderr=subprocess.STDOUT) and then does |
- # custom output processing that breaks printing '\r' characters for |
- # single-line updating status messages as printed by curl and wget. |
- # Work around this by setting stderr of the update.sh process to stdin (!): |
- # gclient doesn't redirect stdin, and while stdin itself is read-only, a |
- # dup()ed sys.stdin is writable, try |
- # fd2 = os.dup(sys.stdin.fileno()); os.write(fd2, 'hi') |
- # TODO: Fix gclient instead, http://crbug.com/95350 |
- if '--no-stdin-hack' in sys.argv: |
- sys.argv.remove('--no-stdin-hack') |
- stderr = None |
- else: |
- try: |
- stderr = os.fdopen(os.dup(sys.stdin.fileno())) |
- except: |
- stderr = sys.stderr |
- return subprocess.call( |
- [os.path.join(os.path.dirname(__file__), 'update.sh')] + sys.argv[1:], |
- stderr=stderr) |
- |
parser = argparse.ArgumentParser(description='Build Clang.') |
parser.add_argument('--bootstrap', action='store_true', |
help='first build clang with CC, then with itself.') |
@@ -699,6 +825,14 @@ def main(): |
help="run only if the script thinks clang is needed") |
parser.add_argument('--force-local-build', action='store_true', |
help="don't try to download prebuild binaries") |
+ parser.add_argument('--gcc-toolchain', help='set the version for which gcc ' |
+ 'version be used for building; --gcc-toolchain=/opt/foo ' |
+ 'picks /opt/foo/bin/gcc') |
+ parser.add_argument('--lto-gold-plugin', action='store_true', |
+ help='build LLVM Gold plugin with LTO') |
+ parser.add_argument('--llvm-force-head-revision', action='store_true', |
+ help=('use the revision in the repo when printing ' |
+ 'the revision')) |
parser.add_argument('--print-revision', action='store_true', |
help='print current clang revision and exit.') |
parser.add_argument('--print-clang-version', action='store_true', |
@@ -708,16 +842,19 @@ def main(): |
parser.add_argument('--tools', nargs='*', |
help='select which chrome tools to build', |
default=['plugins', 'blink_gc_plugin']) |
- parser.add_argument('--without-patches', action='store_false', |
- help="don't apply patches (default)", dest='with_patches', |
- default=True) |
- |
- # For now, these flags are only used for the non-Windows flow, but argparser |
- # gets mad if it sees a flag it doesn't recognize. |
- parser.add_argument('--no-stdin-hack', action='store_true') |
- |
+ parser.add_argument('--without-android', action='store_false', |
+ help='don\'t build Android ASan runtime (linux only)', |
+ dest='with_android', |
+ default=sys.platform.startswith('linux')) |
args = parser.parse_args() |
+ if args.lto_gold_plugin and not args.bootstrap: |
+ print '--lto-gold-plugin requires --bootstrap' |
+ return 1 |
+ if args.lto_gold_plugin and not sys.platform.startswith('linux'): |
+ print '--lto-gold-plugin is only effective on Linux. Ignoring the option.' |
+ args.lto_gold_plugin = False |
+ |
if args.if_needed: |
is_clang_required = False |
# clang is always used on Mac and Linux. |
@@ -727,7 +864,7 @@ def main(): |
if re.search(r'\b(clang|asan|lsan|msan|tsan)=1', |
os.environ.get('GYP_DEFINES', '')): |
is_clang_required = True |
- # clang previously downloaded, keep it up-to-date. |
+ # clang previously downloaded, keep it up to date. |
# If you don't want this, delete third_party/llvm-build on your machine. |
if os.path.isdir(LLVM_BUILD_DIR): |
is_clang_required = True |
@@ -737,9 +874,9 @@ def main(): |
print 'Skipping Clang update (make_clang_dir= was set in GYP_DEFINES).' |
return 0 |
- global LLVM_WIN_REVISION, PACKAGE_VERSION |
+ global CLANG_REVISION, PACKAGE_VERSION |
if args.print_revision: |
- if use_head_revision: |
+ if use_head_revision or args.llvm_force_head_revision: |
print GetSvnRevision(LLVM_DIR) |
else: |
print PACKAGE_VERSION |
@@ -757,12 +894,13 @@ def main(): |
if use_head_revision: |
# Use a real revision number rather than HEAD to make sure that the stamp |
# file logic works. |
- LLVM_WIN_REVISION = GetSvnRevision(LLVM_REPO_URL) |
- PACKAGE_VERSION = LLVM_WIN_REVISION + '-0' |
+ CLANG_REVISION = GetSvnRevision(LLVM_REPO_URL) |
+ PACKAGE_VERSION = CLANG_REVISION + '-0' |
args.force_local_build = True |
- # Skip local patches when using HEAD: they probably don't apply anymore. |
- args.with_patches = False |
+ if 'OS=android' not in os.environ.get('GYP_DEFINES', ''): |
+ # Only build the Android ASan rt on ToT bots when targetting Android. |
+ args.with_android = False |
return UpdateClang(args) |