| Index: tools/clang/scripts/update.py
|
| diff --git a/tools/clang/scripts/update.py b/tools/clang/scripts/update.py
|
| index 02488a6c3306a17303831b7a96569dfd26ed7cb2..38c0e779d212ffb588f0d2d54d535e409ad03f41 100755
|
| --- a/tools/clang/scripts/update.py
|
| +++ b/tools/clang/scripts/update.py
|
| @@ -7,32 +7,46 @@
|
| update.sh. This script should replace update.sh on all platforms eventually."""
|
|
|
| import argparse
|
| +import contextlib
|
| +import cStringIO
|
| +import glob
|
| import os
|
| +import pipes
|
| import re
|
| import shutil
|
| import subprocess
|
| import stat
|
| import sys
|
| +import tarfile
|
| import time
|
| +import urllib2
|
| +import zipfile
|
|
|
| # Do NOT CHANGE this if you don't know what you're doing -- see
|
| # https://code.google.com/p/chromium/wiki/UpdatingClang
|
| # Reverting problematic clang rolls is safe, though.
|
| # Note: this revision is only used for Windows. Other platforms use update.sh.
|
| -LLVM_WIN_REVISION = 'HEAD'
|
| +# TODO(thakis): Use the same revision on Windows and non-Windows.
|
| +# TODO(thakis): Remove update.sh, use update.py everywhere.
|
| +LLVM_WIN_REVISION = '247874'
|
|
|
| -# ASan on Windows is useful enough to use it even while the clang/win is still
|
| -# in bringup. Use a pinned revision to make it slightly more stable.
|
| -use_head_revision = ('LLVM_FORCE_HEAD_REVISION' in os.environ or
|
| - not re.search(r'\b(asan)=1', os.environ.get('GYP_DEFINES', '')))
|
| +use_head_revision = 'LLVM_FORCE_HEAD_REVISION' in os.environ
|
| +if use_head_revision:
|
| + LLVM_WIN_REVISION = 'HEAD'
|
|
|
| -if not use_head_revision:
|
| - LLVM_WIN_REVISION = '235968'
|
| +# This is incremented when pushing a new build of Clang at the same revision.
|
| +CLANG_SUB_REVISION=1
|
| +
|
| +PACKAGE_VERSION = "%s-%s" % (LLVM_WIN_REVISION, CLANG_SUB_REVISION)
|
|
|
| # Path constants. (All of these should be absolute paths.)
|
| THIS_DIR = os.path.abspath(os.path.dirname(__file__))
|
| CHROMIUM_DIR = os.path.abspath(os.path.join(THIS_DIR, '..', '..', '..'))
|
| -LLVM_DIR = os.path.join(CHROMIUM_DIR, 'third_party', 'llvm')
|
| +THIRD_PARTY_DIR = os.path.join(CHROMIUM_DIR, 'third_party')
|
| +LLVM_DIR = os.path.join(THIRD_PARTY_DIR, 'llvm')
|
| +LLVM_BOOTSTRAP_DIR = os.path.join(THIRD_PARTY_DIR, 'llvm-bootstrap')
|
| +LLVM_BOOTSTRAP_INSTALL_DIR = os.path.join(THIRD_PARTY_DIR,
|
| + 'llvm-bootstrap-install')
|
| CHROME_TOOLS_SHIM_DIR = os.path.join(LLVM_DIR, 'tools', 'chrometools')
|
| LLVM_BUILD_DIR = os.path.join(CHROMIUM_DIR, 'third_party', 'llvm-build',
|
| 'Release+Asserts')
|
| @@ -40,44 +54,67 @@ COMPILER_RT_BUILD_DIR = os.path.join(LLVM_BUILD_DIR, '32bit-compiler-rt')
|
| CLANG_DIR = os.path.join(LLVM_DIR, 'tools', 'clang')
|
| LLD_DIR = os.path.join(LLVM_DIR, 'tools', 'lld')
|
| COMPILER_RT_DIR = os.path.join(LLVM_DIR, 'projects', 'compiler-rt')
|
| -STAMP_FILE = os.path.join(LLVM_BUILD_DIR, 'cr_build_revision')
|
| -VERSION = '3.7.0'
|
| +LIBCXX_DIR = os.path.join(LLVM_DIR, 'projects', 'libcxx')
|
| +LIBCXXABI_DIR = os.path.join(LLVM_DIR, 'projects', 'libcxxabi')
|
| +LLVM_BUILD_TOOLS_DIR = os.path.abspath(
|
| + os.path.join(LLVM_DIR, '..', 'llvm-build-tools'))
|
| +STAMP_FILE = os.path.join(LLVM_DIR, '..', 'llvm-build', 'cr_build_revision')
|
| +BINUTILS_DIR = os.path.join(THIRD_PARTY_DIR, 'binutils')
|
| +VERSION = '3.8.0'
|
| +
|
| +# URL for pre-built binaries.
|
| +CDS_URL = 'https://commondatastorage.googleapis.com/chromium-browser-clang'
|
|
|
| LLVM_REPO_URL='https://llvm.org/svn/llvm-project'
|
| if 'LLVM_REPO_URL' in os.environ:
|
| LLVM_REPO_URL = os.environ['LLVM_REPO_URL']
|
|
|
|
|
| +def DownloadUrl(url, output_file):
|
| + """Download url into output_file."""
|
| + CHUNK_SIZE = 4096
|
| + TOTAL_DOTS = 10
|
| + sys.stdout.write('Downloading %s ' % url)
|
| + sys.stdout.flush()
|
| + response = urllib2.urlopen(url)
|
| + total_size = int(response.info().getheader('Content-Length').strip())
|
| + bytes_done = 0
|
| + dots_printed = 0
|
| + while True:
|
| + chunk = response.read(CHUNK_SIZE)
|
| + if not chunk:
|
| + break
|
| + output_file.write(chunk)
|
| + bytes_done += len(chunk)
|
| + num_dots = TOTAL_DOTS * bytes_done / total_size
|
| + sys.stdout.write('.' * (num_dots - dots_printed))
|
| + sys.stdout.flush()
|
| + dots_printed = num_dots
|
| + print ' Done.'
|
| +
|
| +
|
| def ReadStampFile():
|
| """Return the contents of the stamp file, or '' if it doesn't exist."""
|
| try:
|
| with open(STAMP_FILE, 'r') as f:
|
| - return f.read();
|
| + return f.read()
|
| except IOError:
|
| return ''
|
|
|
|
|
| def WriteStampFile(s):
|
| """Write s to the stamp file."""
|
| - if not os.path.exists(LLVM_BUILD_DIR):
|
| - os.makedirs(LLVM_BUILD_DIR)
|
| + if not os.path.exists(os.path.dirname(STAMP_FILE)):
|
| + os.makedirs(os.path.dirname(STAMP_FILE))
|
| with open(STAMP_FILE, 'w') as f:
|
| f.write(s)
|
|
|
|
|
| -def PrintRevision():
|
| - """Print the current Clang revision."""
|
| - # gyp runs update.py --print-revision even when clang isn't used.
|
| - # It won't use the value, but we must not error.
|
| - if not os.path.exists(LLVM_DIR):
|
| - print "0"
|
| - return
|
| -
|
| - # TODO(hans): This needs an update when we move to prebuilt Clang binaries.
|
| - svn_info = subprocess.check_output(['svn', 'info', LLVM_DIR], shell=True)
|
| +def GetSvnRevision(svn_repo):
|
| + """Returns current revision of the svn repo at svn_repo."""
|
| + svn_info = subprocess.check_output('svn info ' + svn_repo, shell=True)
|
| m = re.search(r'Revision: (\d+)', svn_info)
|
| - assert m
|
| - print m.group(1)
|
| + return m.group(1)
|
|
|
|
|
| def RmTree(dir):
|
| @@ -92,12 +129,31 @@ def RmTree(dir):
|
| shutil.rmtree(dir, onerror=ChmodAndRetry)
|
|
|
|
|
| -def RunCommand(command, fail_hard=True):
|
| +def RunCommand(command, msvc_arch=None, env=None, fail_hard=True):
|
| """Run command and return success (True) or failure; or if fail_hard is
|
| - True, exit on failure."""
|
| -
|
| - print 'Running %s' % (str(command))
|
| - if subprocess.call(command, shell=True) == 0:
|
| + True, exit on failure. If msvc_arch is set, runs the command in a
|
| + shell with the msvc tools for that architecture."""
|
| +
|
| + if msvc_arch and sys.platform == 'win32':
|
| + command = GetVSVersion().SetupScript(msvc_arch) + ['&&'] + command
|
| +
|
| + # https://docs.python.org/2/library/subprocess.html:
|
| + # "On Unix with shell=True [...] if args is a sequence, the first item
|
| + # specifies the command string, and any additional items will be treated as
|
| + # additional arguments to the shell itself. That is to say, Popen does the
|
| + # equivalent of:
|
| + # Popen(['/bin/sh', '-c', args[0], args[1], ...])"
|
| + #
|
| + # We want to pass additional arguments to command[0], not to the shell,
|
| + # so manually join everything into a single string.
|
| + # Annoyingly, for "svn co url c:\path", pipes.quote() thinks that it should
|
| + # quote c:\path but svn can't handle quoted paths on Windows. Since on
|
| + # Windows follow-on args are passed to args[0] instead of the shell, don't
|
| + # do the single-string transformation there.
|
| + if sys.platform != 'win32':
|
| + command = ' '.join([pipes.quote(c) for c in command])
|
| + print 'Running', command
|
| + if subprocess.call(command, env=env, shell=True) == 0:
|
| return True
|
| print 'Failed.'
|
| if fail_hard:
|
| @@ -139,7 +195,58 @@ def Checkout(name, url, dir):
|
| RunCommand(command)
|
|
|
|
|
| +def RevertPreviouslyPatchedFiles():
|
| + print 'Reverting previously patched files'
|
| + files = [
|
| + '%(clang)s/test/Index/crash-recovery-modules.m',
|
| + '%(clang)s/unittests/libclang/LibclangTest.cpp',
|
| + '%(compiler_rt)s/lib/asan/asan_rtl.cc',
|
| + '%(compiler_rt)s/test/asan/TestCases/Linux/new_array_cookie_test.cc',
|
| + '%(llvm)s/test/DebugInfo/gmlt.ll',
|
| + '%(llvm)s/lib/CodeGen/SpillPlacement.cpp',
|
| + '%(llvm)s/lib/CodeGen/SpillPlacement.h',
|
| + '%(llvm)s/lib/Transforms/Instrumentation/MemorySanitizer.cpp',
|
| + '%(clang)s/test/Driver/env.c',
|
| + '%(clang)s/lib/Frontend/InitPreprocessor.cpp',
|
| + '%(clang)s/test/Frontend/exceptions.c',
|
| + '%(clang)s/test/Preprocessor/predefined-exceptions.m',
|
| + '%(llvm)s/test/Bindings/Go/go.test',
|
| + '%(clang)s/lib/Parse/ParseExpr.cpp',
|
| + '%(clang)s/lib/Parse/ParseTemplate.cpp',
|
| + '%(clang)s/lib/Sema/SemaDeclCXX.cpp',
|
| + '%(clang)s/lib/Sema/SemaExprCXX.cpp',
|
| + '%(clang)s/test/SemaCXX/default2.cpp',
|
| + '%(clang)s/test/SemaCXX/typo-correction-delayed.cpp',
|
| + '%(compiler_rt)s/lib/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cc',
|
| + '%(compiler_rt)s/test/tsan/signal_segv_handler.cc',
|
| + '%(compiler_rt)s/lib/sanitizer_common/sanitizer_coverage_libcdep.cc',
|
| + '%(compiler_rt)s/cmake/config-ix.cmake',
|
| + '%(compiler_rt)s/CMakeLists.txt',
|
| + '%(compiler_rt)s/lib/ubsan/ubsan_platform.h',
|
| + ]
|
| + for f in files:
|
| + f = f % {
|
| + 'clang': CLANG_DIR,
|
| + 'compiler_rt': COMPILER_RT_DIR,
|
| + 'llvm': LLVM_DIR,
|
| + }
|
| + if os.path.exists(f):
|
| + os.remove(f) # For unversioned files.
|
| + RunCommand(['svn', 'revert', f])
|
| +
|
| +
|
| +def ApplyLocalPatches():
|
| + # There's no patch program on Windows by default. We don't need patches on
|
| + # Windows yet, and maybe this not working on Windows will motivate us to
|
| + # remove patches over time.
|
| + assert sys.platform != 'win32'
|
| +
|
| + # No patches.
|
| +
|
| +
|
| def DeleteChromeToolsShim():
|
| + OLD_SHIM_DIR = os.path.join(LLVM_DIR, 'tools', 'zzz-chrometools')
|
| + shutil.rmtree(OLD_SHIM_DIR, ignore_errors=True)
|
| shutil.rmtree(CHROME_TOOLS_SHIM_DIR, ignore_errors=True)
|
|
|
|
|
| @@ -161,27 +268,36 @@ def CreateChromeToolsShim():
|
| f.write('# two arg version to specify where build artifacts go. CMake\n')
|
| f.write('# disallows reuse of the same binary dir for multiple source\n')
|
| f.write('# dirs, so the build artifacts need to go into a subdirectory.\n')
|
| - f.write('add_subdirectory(${CHROMIUM_TOOLS_SRC} ' +
|
| - '${CMAKE_CURRENT_BINARY_DIR}/a)\n')
|
| + f.write('# dirs, so the build artifacts need to go into a subdirectory.\n')
|
| + f.write('if (CHROMIUM_TOOLS_SRC)\n')
|
| + f.write(' add_subdirectory(${CHROMIUM_TOOLS_SRC} ' +
|
| + '${CMAKE_CURRENT_BINARY_DIR}/a)\n')
|
| + f.write('endif (CHROMIUM_TOOLS_SRC)\n')
|
|
|
|
|
| def AddCMakeToPath():
|
| - """Look for CMake and add it to PATH if it's not there already."""
|
| - try:
|
| - # First check if cmake is already on PATH.
|
| - subprocess.call(['cmake', '--version'])
|
| - return
|
| - except OSError as e:
|
| - if e.errno != os.errno.ENOENT:
|
| - raise
|
| -
|
| - cmake_dir = 'C:\\Program Files (x86)\\CMake\\bin'
|
| - if os.path.isdir(cmake_dir):
|
| - os.environ['PATH'] = os.environ.get('PATH', '') + os.pathsep + cmake_dir
|
| - return
|
| - print 'Failed to find CMake!'
|
| - sys.exit(1)
|
| -
|
| + """Download CMake and add it to PATH."""
|
| + if sys.platform == 'win32':
|
| + zip_name = 'cmake-3.2.2-win32-x86.zip'
|
| + cmake_dir = os.path.join(LLVM_BUILD_TOOLS_DIR,
|
| + 'cmake-3.2.2-win32-x86', 'bin')
|
| + else:
|
| + suffix = 'Darwin' if sys.platform == 'darwin' else 'Linux'
|
| + zip_name = 'cmake310_%s.tgz' % suffix
|
| + cmake_dir = os.path.join(LLVM_BUILD_TOOLS_DIR, 'cmake310', 'bin')
|
| + if not os.path.exists(cmake_dir):
|
| + if not os.path.exists(LLVM_BUILD_TOOLS_DIR):
|
| + os.makedirs(LLVM_BUILD_TOOLS_DIR)
|
| + # The cmake archive is smaller than 20 MB, small enough to keep in memory:
|
| + with contextlib.closing(cStringIO.StringIO()) as f:
|
| + DownloadUrl(CDS_URL + '/tools/' + zip_name, f)
|
| + f.seek(0)
|
| + if zip_name.endswith('.zip'):
|
| + zipfile.ZipFile(f).extractall(path=LLVM_BUILD_TOOLS_DIR)
|
| + else:
|
| + tarfile.open(mode='r:gz', fileobj=f).extractall(path=
|
| + LLVM_BUILD_TOOLS_DIR)
|
| + os.environ['PATH'] = cmake_dir + os.pathsep + os.environ.get('PATH', '')
|
|
|
| vs_version = None
|
| def GetVSVersion():
|
| @@ -203,76 +319,292 @@ def GetVSVersion():
|
| return vs_version
|
|
|
|
|
| -def SubversionCmakeArg():
|
| - # Since cmake's find_program can only find .exe and .com,
|
| - # svn.bat in depot_tools will be ignored.
|
| - default_pathext = ('.com', '.exe', '.bat', '.cmd')
|
| - for path in os.environ.get('PATH', '').split(os.pathsep):
|
| - for ext in default_pathext:
|
| - candidate = os.path.join(path, 'svn' + ext)
|
| - if os.path.isfile(candidate):
|
| - return '-DSubversion_SVN_EXECUTABLE=%s' % candidate
|
| - return ''
|
| -
|
| -
|
| def UpdateClang(args):
|
| - print 'Updating Clang to %s...' % (LLVM_WIN_REVISION)
|
| - if LLVM_WIN_REVISION != 'HEAD' and ReadStampFile() == LLVM_WIN_REVISION:
|
| + print 'Updating Clang to %s...' % PACKAGE_VERSION
|
| + if ReadStampFile() == PACKAGE_VERSION:
|
| print 'Already up to date.'
|
| return 0
|
|
|
| - AddCMakeToPath()
|
| # Reset the stamp file in case the build is unsuccessful.
|
| WriteStampFile('')
|
|
|
| - DeleteChromeToolsShim();
|
| + if not args.force_local_build:
|
| + cds_file = "clang-%s.tgz" % PACKAGE_VERSION
|
| + cds_full_url = CDS_URL + '/Win/' + cds_file
|
| +
|
| + # Check if there's a prebuilt binary and if so just fetch that. That's
|
| + # faster, and goma relies on having matching binary hashes on client and
|
| + # server too.
|
| + print 'Trying to download prebuilt clang'
|
| +
|
| + # clang packages are smaller than 50 MB, small enough to keep in memory.
|
| + with contextlib.closing(cStringIO.StringIO()) as f:
|
| + try:
|
| + DownloadUrl(cds_full_url, f)
|
| + f.seek(0)
|
| + # TODO(thakis): Delete LLVM_BUILD_DIR before extracting.
|
| + tarfile.open(mode='r:gz', fileobj=f).extractall(path=LLVM_BUILD_DIR)
|
| + print 'clang %s unpacked' % PACKAGE_VERSION
|
| + # Download the gold plugin if requested to by an environment variable.
|
| + # This is used by the CFI ClusterFuzz bot.
|
| + if 'LLVM_DOWNLOAD_GOLD_PLUGIN' in os.environ:
|
| + RunCommand(['python', CHROMIUM_DIR+'/build/download_gold_plugin.py'])
|
| + WriteStampFile(PACKAGE_VERSION)
|
| + return 0
|
| + except urllib2.HTTPError:
|
| + print 'Did not find prebuilt clang %s, building locally' % cds_file
|
| +
|
| + AddCMakeToPath()
|
| +
|
| + RevertPreviouslyPatchedFiles()
|
| + DeleteChromeToolsShim()
|
| +
|
| Checkout('LLVM', LLVM_REPO_URL + '/llvm/trunk', LLVM_DIR)
|
| Checkout('Clang', LLVM_REPO_URL + '/cfe/trunk', CLANG_DIR)
|
| - Checkout('LLD', LLVM_REPO_URL + '/lld/trunk', LLD_DIR)
|
| + if sys.platform == 'win32':
|
| + Checkout('LLD', LLVM_REPO_URL + '/lld/trunk', LLD_DIR)
|
| Checkout('compiler-rt', LLVM_REPO_URL + '/compiler-rt/trunk', COMPILER_RT_DIR)
|
| - CreateChromeToolsShim();
|
| + if sys.platform == 'darwin':
|
| + # clang needs a libc++ checkout, else -stdlib=libc++ won't find includes
|
| + # (i.e. this is needed for bootstrap builds).
|
| + Checkout('libcxx', LLVM_REPO_URL + '/libcxx/trunk', LIBCXX_DIR)
|
| + # While we're bundling our own libc++ on OS X, we need to compile libc++abi
|
| + # into it too (since OS X 10.6 doesn't have libc++abi.dylib either).
|
| + Checkout('libcxxabi', LLVM_REPO_URL + '/libcxxabi/trunk', LIBCXXABI_DIR)
|
| +
|
| + if args.with_patches and sys.platform != 'win32':
|
| + ApplyLocalPatches()
|
| +
|
| + cc, cxx = None, None
|
| + libstdcpp = None
|
| + if args.gcc_toolchain: # This option is only used on Linux.
|
| + # Use the specified gcc installation for building.
|
| + cc = os.path.join(args.gcc_toolchain, 'bin', 'gcc')
|
| + cxx = os.path.join(args.gcc_toolchain, 'bin', 'g++')
|
| +
|
| + if not os.access(cc, os.X_OK):
|
| + print 'Invalid --gcc-toolchain: "%s"' % args.gcc_toolchain
|
| + print '"%s" does not appear to be valid.' % cc
|
| + return 1
|
| +
|
| + # Set LD_LIBRARY_PATH to make auxiliary targets (tablegen, bootstrap
|
| + # compiler, etc.) find the .so.
|
| + libstdcpp = subprocess.check_output(
|
| + [cxx, '-print-file-name=libstdc++.so.6']).rstrip()
|
| + os.environ['LD_LIBRARY_PATH'] = os.path.dirname(libstdcpp)
|
| +
|
| + cflags = cxxflags = ldflags = []
|
| +
|
| + # LLVM uses C++11 starting in llvm 3.5. On Linux, this means libstdc++4.7+ is
|
| + # needed, on OS X it requires libc++. clang only automatically links to libc++
|
| + # when targeting OS X 10.9+, so add stdlib=libc++ explicitly so clang can run
|
| + # on OS X versions as old as 10.7.
|
| + # TODO(thakis): Some bots are still on 10.6 (nacl...), so for now bundle
|
| + # libc++.dylib. Remove this once all bots are on 10.7+, then use
|
| + # -DLLVM_ENABLE_LIBCXX=ON and change deployment_target to 10.7.
|
| + deployment_target = ''
|
| +
|
| + if sys.platform == 'darwin':
|
| + # When building on 10.9, /usr/include usually doesn't exist, and while
|
| + # Xcode's clang automatically sets a sysroot, self-built clangs don't.
|
| + cflags = ['-isysroot', subprocess.check_output(
|
| + ['xcrun', '--show-sdk-path']).rstrip()]
|
| + cxxflags = ['-stdlib=libc++', '-nostdinc++',
|
| + '-I' + os.path.join(LIBCXX_DIR, 'include')] + cflags
|
| + if args.bootstrap:
|
| + deployment_target = '10.6'
|
| +
|
| + base_cmake_args = ['-GNinja',
|
| + '-DCMAKE_BUILD_TYPE=Release',
|
| + '-DLLVM_ENABLE_ASSERTIONS=ON',
|
| + '-DLLVM_ENABLE_THREADS=OFF',
|
| + ]
|
| +
|
| + if args.bootstrap:
|
| + print 'Building bootstrap compiler'
|
| + if not os.path.exists(LLVM_BOOTSTRAP_DIR):
|
| + os.makedirs(LLVM_BOOTSTRAP_DIR)
|
| + os.chdir(LLVM_BOOTSTRAP_DIR)
|
| + bootstrap_args = base_cmake_args + [
|
| + '-DLLVM_TARGETS_TO_BUILD=host',
|
| + '-DCMAKE_INSTALL_PREFIX=' + LLVM_BOOTSTRAP_INSTALL_DIR,
|
| + '-DCMAKE_C_FLAGS=' + ' '.join(cflags),
|
| + '-DCMAKE_CXX_FLAGS=' + ' '.join(cxxflags),
|
| + ]
|
| + if cc is not None: bootstrap_args.append('-DCMAKE_C_COMPILER=' + cc)
|
| + if cxx is not None: bootstrap_args.append('-DCMAKE_CXX_COMPILER=' + cxx)
|
| + RunCommand(['cmake'] + bootstrap_args + [LLVM_DIR], msvc_arch='x64')
|
| + RunCommand(['ninja'], msvc_arch='x64')
|
| + if args.run_tests:
|
| + RunCommand(['ninja', 'check-all'], msvc_arch='x64')
|
| + RunCommand(['ninja', 'install'], msvc_arch='x64')
|
| + if args.gcc_toolchain:
|
| + # Copy that gcc's stdlibc++.so.6 to the build dir, so the bootstrap
|
| + # compiler can start.
|
| + CopyFile(libstdcpp, os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'lib'))
|
| +
|
| + if sys.platform == 'win32':
|
| + cc = os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'bin', 'clang-cl.exe')
|
| + cxx = os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'bin', 'clang-cl.exe')
|
| + # CMake has a hard time with backslashes in compiler paths:
|
| + # https://stackoverflow.com/questions/13050827
|
| + cc = cc.replace('\\', '/')
|
| + cxx = cxx.replace('\\', '/')
|
| + else:
|
| + cc = os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'bin', 'clang')
|
| + cxx = os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'bin', 'clang++')
|
| +
|
| + if args.gcc_toolchain:
|
| + # Tell the bootstrap compiler to use a specific gcc prefix to search
|
| + # for standard library headers and shared object files.
|
| + cflags = ['--gcc-toolchain=' + args.gcc_toolchain]
|
| + cxxflags = ['--gcc-toolchain=' + args.gcc_toolchain]
|
| + print 'Building final compiler'
|
| +
|
| + if sys.platform == 'darwin':
|
| + # Build libc++.dylib while some bots are still on OS X 10.6.
|
| + libcxxbuild = os.path.join(LLVM_BUILD_DIR, 'libcxxbuild')
|
| + if os.path.isdir(libcxxbuild):
|
| + RmTree(libcxxbuild)
|
| + libcxxflags = ['-O3', '-std=c++11', '-fstrict-aliasing']
|
| +
|
| + # libcxx and libcxxabi both have a file stdexcept.cpp, so put their .o files
|
| + # into different subdirectories.
|
| + os.makedirs(os.path.join(libcxxbuild, 'libcxx'))
|
| + os.chdir(os.path.join(libcxxbuild, 'libcxx'))
|
| + RunCommand(['c++', '-c'] + cxxflags + libcxxflags +
|
| + glob.glob(os.path.join(LIBCXX_DIR, 'src', '*.cpp')))
|
| +
|
| + os.makedirs(os.path.join(libcxxbuild, 'libcxxabi'))
|
| + os.chdir(os.path.join(libcxxbuild, 'libcxxabi'))
|
| + RunCommand(['c++', '-c'] + cxxflags + libcxxflags +
|
| + glob.glob(os.path.join(LIBCXXABI_DIR, 'src', '*.cpp')) +
|
| + ['-I' + os.path.join(LIBCXXABI_DIR, 'include')])
|
| +
|
| + os.chdir(libcxxbuild)
|
| + libdir = os.path.join(LIBCXX_DIR, 'lib')
|
| + RunCommand(['cc'] + glob.glob('libcxx/*.o') + glob.glob('libcxxabi/*.o') +
|
| + ['-o', 'libc++.1.dylib', '-dynamiclib', '-nodefaultlibs',
|
| + '-current_version', '1', '-compatibility_version', '1', '-lSystem',
|
| + '-install_name', '@executable_path/libc++.dylib',
|
| + '-Wl,-unexported_symbols_list,' + libdir + '/libc++unexp.exp',
|
| + '-Wl,-force_symbols_not_weak_list,' + libdir + '/notweak.exp',
|
| + '-Wl,-force_symbols_weak_list,' + libdir + '/weak.exp'])
|
| + if os.path.exists('libc++.dylib'):
|
| + os.remove('libc++.dylib')
|
| + os.symlink('libc++.1.dylib', 'libc++.dylib')
|
| + ldflags += ['-stdlib=libc++', '-L' + libcxxbuild]
|
| +
|
| + if args.bootstrap:
|
| + # Now that the libc++ headers have been installed and libc++.dylib is
|
| + # built, delete the libc++ checkout again so that it's not part of the
|
| + # main build below -- the libc++(abi) tests don't pass on OS X in
|
| + # bootstrap builds (http://llvm.org/PR24068)
|
| + RmTree(LIBCXX_DIR)
|
| + RmTree(LIBCXXABI_DIR)
|
| + cxxflags = ['-stdlib=libc++', '-nostdinc++',
|
| + '-I' + os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR,
|
| + 'include/c++/v1')
|
| + ] + cflags
|
| +
|
| + # Build clang.
|
| + binutils_incdir = ''
|
| + if sys.platform.startswith('linux'):
|
| + binutils_incdir = os.path.join(BINUTILS_DIR, 'Linux_x64/Release/include')
|
| +
|
| + # If building at head, define a macro that plugins can use for #ifdefing
|
| + # out code that builds at head, but not at LLVM_WIN_REVISION or vice versa.
|
| + if use_head_revision:
|
| + cflags += ['-DLLVM_FORCE_HEAD_REVISION']
|
| + cxxflags += ['-DLLVM_FORCE_HEAD_REVISION']
|
| +
|
| + CreateChromeToolsShim()
|
| +
|
| + deployment_env = None
|
| + if deployment_target:
|
| + deployment_env = os.environ.copy()
|
| + deployment_env['MACOSX_DEPLOYMENT_TARGET'] = deployment_target
|
| +
|
| + cmake_args = base_cmake_args + [
|
| + '-DLLVM_BINUTILS_INCDIR=' + binutils_incdir,
|
| + '-DLLVM_EXPERIMENTAL_TARGETS_TO_BUILD=WebAssembly',
|
| + '-DCMAKE_C_FLAGS=' + ' '.join(cflags),
|
| + '-DCMAKE_CXX_FLAGS=' + ' '.join(cxxflags),
|
| + '-DCMAKE_EXE_LINKER_FLAGS=' + ' '.join(ldflags),
|
| + '-DCMAKE_SHARED_LINKER_FLAGS=' + ' '.join(ldflags),
|
| + '-DCMAKE_MODULE_LINKER_FLAGS=' + ' '.join(ldflags),
|
| + '-DCMAKE_INSTALL_PREFIX=' + LLVM_BUILD_DIR,
|
| + '-DCHROMIUM_TOOLS_SRC=%s' % os.path.join(CHROMIUM_DIR, 'tools', 'clang'),
|
| + '-DCHROMIUM_TOOLS=%s' % ';'.join(args.tools)]
|
| + # TODO(thakis): Unconditionally append this to base_cmake_args instead once
|
| + # compiler-rt can build with clang-cl on Windows (http://llvm.org/PR23698)
|
| + cc_args = base_cmake_args if sys.platform != 'win32' else cmake_args
|
| + if cc is not None: cc_args.append('-DCMAKE_C_COMPILER=' + cc)
|
| + if cxx is not None: cc_args.append('-DCMAKE_CXX_COMPILER=' + cxx)
|
|
|
| if not os.path.exists(LLVM_BUILD_DIR):
|
| os.makedirs(LLVM_BUILD_DIR)
|
| os.chdir(LLVM_BUILD_DIR)
|
| -
|
| - # If building at head, define a macro that plugins can use for #ifdefing
|
| - # out code that builds at head, but not at CLANG_REVISION or vice versa.
|
| - cflags = cxxflags = ''
|
| -
|
| - # TODO(thakis): Set this only conditionally if use_head_revision once posix
|
| - # and win clang are in sync. At the moment, the plugins only build at clang
|
| - # head on posix, but they build at both head and the pinned win version :-/
|
| - cflags += ' -DLLVM_FORCE_HEAD_REVISION'
|
| - cxxflags += ' -DLLVM_FORCE_HEAD_REVISION'
|
| -
|
| - cmake_args = ['-GNinja', '-DCMAKE_BUILD_TYPE=Release',
|
| - '-DLLVM_ENABLE_ASSERTIONS=ON', SubversionCmakeArg(),
|
| - '-DCMAKE_C_FLAGS=' + cflags,
|
| - '-DCMAKE_CXX_FLAGS=' + cxxflags,
|
| - '-DCHROMIUM_TOOLS_SRC=%s' % os.path.join(
|
| - CHROMIUM_DIR, 'tools', 'clang'),
|
| - '-DCHROMIUM_TOOLS=%s' % ';'.join(args.tools)]
|
| -
|
| - RunCommand(GetVSVersion().SetupScript('x64') +
|
| - ['&&', 'cmake'] + cmake_args + [LLVM_DIR])
|
| - RunCommand(GetVSVersion().SetupScript('x64') + ['&&', 'ninja', 'all'])
|
| + RunCommand(['cmake'] + cmake_args + [LLVM_DIR],
|
| + msvc_arch='x64', env=deployment_env)
|
| +
|
| + if args.gcc_toolchain:
|
| + # Copy in the right stdlibc++.so.6 so clang can start.
|
| + if not os.path.exists(os.path.join(LLVM_BUILD_DIR, 'lib')):
|
| + os.mkdir(os.path.join(LLVM_BUILD_DIR, 'lib'))
|
| + libstdcpp = subprocess.check_output(
|
| + [cxx] + cxxflags + ['-print-file-name=libstdc++.so.6']).rstrip()
|
| + CopyFile(libstdcpp, os.path.join(LLVM_BUILD_DIR, 'lib'))
|
| +
|
| + RunCommand(['ninja'], msvc_arch='x64')
|
| +
|
| + if args.tools:
|
| + # If any Chromium tools were built, install those now.
|
| + RunCommand(['ninja', 'cr-install'], msvc_arch='x64')
|
| +
|
| + if sys.platform == 'darwin':
|
| + CopyFile(os.path.join(LLVM_BUILD_DIR, 'libc++.1.dylib'),
|
| + os.path.join(LLVM_BUILD_DIR, 'bin'))
|
| + # See http://crbug.com/256342
|
| + RunCommand(['strip', '-x', os.path.join(LLVM_BUILD_DIR, 'bin', 'clang')])
|
| + elif sys.platform.startswith('linux'):
|
| + RunCommand(['strip', os.path.join(LLVM_BUILD_DIR, 'bin', 'clang')])
|
|
|
| # Do an x86 build of compiler-rt to get the 32-bit ASan run-time.
|
| # TODO(hans): Remove once the regular build above produces this.
|
| if not os.path.exists(COMPILER_RT_BUILD_DIR):
|
| os.makedirs(COMPILER_RT_BUILD_DIR)
|
| os.chdir(COMPILER_RT_BUILD_DIR)
|
| - RunCommand(GetVSVersion().SetupScript('x86') +
|
| - ['&&', 'cmake'] + cmake_args + [LLVM_DIR])
|
| - RunCommand(GetVSVersion().SetupScript('x86') + ['&&', 'ninja', 'compiler-rt'])
|
| + # TODO(thakis): Add this once compiler-rt can build with clang-cl (see
|
| + # above).
|
| + #if args.bootstrap and sys.platform == 'win32':
|
| + # The bootstrap compiler produces 64-bit binaries by default.
|
| + #cflags += ['-m32']
|
| + #cxxflags += ['-m32']
|
| + compiler_rt_args = base_cmake_args + [
|
| + '-DCMAKE_C_FLAGS=' + ' '.join(cflags),
|
| + '-DCMAKE_CXX_FLAGS=' + ' '.join(cxxflags)]
|
| + if sys.platform != 'win32':
|
| + compiler_rt_args += ['-DLLVM_CONFIG_PATH=' +
|
| + os.path.join(LLVM_BUILD_DIR, 'bin', 'llvm-config'),
|
| + '-DSANITIZER_MIN_OSX_VERSION="10.7"']
|
| + RunCommand(['cmake'] + compiler_rt_args + [LLVM_DIR],
|
| + msvc_arch='x86', env=deployment_env)
|
| + RunCommand(['ninja', 'compiler-rt'], msvc_arch='x86')
|
|
|
| # TODO(hans): Make this (and the .gypi and .isolate files) version number
|
| # independent.
|
| + if sys.platform == 'win32':
|
| + platform = 'windows'
|
| + elif sys.platform == 'darwin':
|
| + platform = 'darwin'
|
| + else:
|
| + assert sys.platform.startswith('linux')
|
| + platform = 'linux'
|
| asan_rt_lib_src_dir = os.path.join(COMPILER_RT_BUILD_DIR, 'lib', 'clang',
|
| - VERSION, 'lib', 'windows')
|
| + VERSION, 'lib', platform)
|
| asan_rt_lib_dst_dir = os.path.join(LLVM_BUILD_DIR, 'lib', 'clang',
|
| - VERSION, 'lib', 'windows')
|
| + VERSION, 'lib', platform)
|
| CopyDirectoryContents(asan_rt_lib_src_dir, asan_rt_lib_dst_dir,
|
| r'^.*-i386\.lib$')
|
| CopyDirectoryContents(asan_rt_lib_src_dir, asan_rt_lib_dst_dir,
|
| @@ -281,26 +613,32 @@ def UpdateClang(args):
|
| CopyFile(os.path.join(asan_rt_lib_src_dir, '..', '..', 'asan_blacklist.txt'),
|
| os.path.join(asan_rt_lib_dst_dir, '..', '..'))
|
|
|
| - # Make an extra copy of the sanitizer headers, to be put on the include path
|
| - # of the fallback compiler.
|
| - sanitizer_include_dir = os.path.join(LLVM_BUILD_DIR, 'lib', 'clang', VERSION,
|
| - 'include', 'sanitizer')
|
| - aux_sanitizer_include_dir = os.path.join(LLVM_BUILD_DIR, 'lib', 'clang',
|
| - VERSION, 'include_sanitizer',
|
| - 'sanitizer')
|
| - if not os.path.exists(aux_sanitizer_include_dir):
|
| - os.makedirs(aux_sanitizer_include_dir)
|
| - for _, _, files in os.walk(sanitizer_include_dir):
|
| - for f in files:
|
| - CopyFile(os.path.join(sanitizer_include_dir, f),
|
| - aux_sanitizer_include_dir)
|
| -
|
| - if args.run_tests:
|
| + if sys.platform == 'win32':
|
| + # Make an extra copy of the sanitizer headers, to be put on the include path
|
| + # of the fallback compiler.
|
| + sanitizer_include_dir = os.path.join(LLVM_BUILD_DIR, 'lib', 'clang',
|
| + VERSION, 'include', 'sanitizer')
|
| + aux_sanitizer_include_dir = os.path.join(LLVM_BUILD_DIR, 'lib', 'clang',
|
| + VERSION, 'include_sanitizer',
|
| + 'sanitizer')
|
| + if not os.path.exists(aux_sanitizer_include_dir):
|
| + os.makedirs(aux_sanitizer_include_dir)
|
| + for _, _, files in os.walk(sanitizer_include_dir):
|
| + for f in files:
|
| + CopyFile(os.path.join(sanitizer_include_dir, f),
|
| + aux_sanitizer_include_dir)
|
| +
|
| + # Run tests.
|
| + if args.run_tests or use_head_revision:
|
| os.chdir(LLVM_BUILD_DIR)
|
| RunCommand(GetVSVersion().SetupScript('x64') +
|
| ['&&', 'ninja', 'cr-check-all'])
|
| + if args.run_tests:
|
| + os.chdir(LLVM_BUILD_DIR)
|
| + RunCommand(GetVSVersion().SetupScript('x64') +
|
| + ['&&', 'ninja', 'check-all'])
|
|
|
| - WriteStampFile(LLVM_WIN_REVISION)
|
| + WriteStampFile(PACKAGE_VERSION)
|
| print 'Clang update was successful.'
|
| return 0
|
|
|
| @@ -319,36 +657,92 @@ def main():
|
| # dup()ed sys.stdin is writable, try
|
| # fd2 = os.dup(sys.stdin.fileno()); os.write(fd2, 'hi')
|
| # TODO: Fix gclient instead, http://crbug.com/95350
|
| - try:
|
| - stderr = os.fdopen(os.dup(sys.stdin.fileno()))
|
| - except:
|
| - stderr = sys.stderr
|
| + if '--no-stdin-hack' in sys.argv:
|
| + sys.argv.remove('--no-stdin-hack')
|
| + stderr = None
|
| + else:
|
| + try:
|
| + stderr = os.fdopen(os.dup(sys.stdin.fileno()))
|
| + except:
|
| + stderr = sys.stderr
|
| return subprocess.call(
|
| [os.path.join(os.path.dirname(__file__), 'update.sh')] + sys.argv[1:],
|
| stderr=stderr)
|
|
|
| parser = argparse.ArgumentParser(description='Build Clang.')
|
| + parser.add_argument('--bootstrap', action='store_true',
|
| + help='first build clang with CC, then with itself.')
|
| + parser.add_argument('--if-needed', action='store_true',
|
| + help="run only if the script thinks clang is needed")
|
| + parser.add_argument('--force-local-build', action='store_true',
|
| + help="don't try to download prebuild binaries")
|
| + parser.add_argument('--gcc-toolchain', help='set the version for which gcc '
|
| + 'version be used for building; --gcc-toolchain=/opt/foo '
|
| + 'picks /opt/foo/bin/gcc')
|
| + parser.add_argument('--print-revision', action='store_true',
|
| + help='print current clang revision and exit.')
|
| + parser.add_argument('--print-clang-version', action='store_true',
|
| + help='print current clang version (e.g. x.y.z) and exit.')
|
| + parser.add_argument('--run-tests', action='store_true',
|
| + help='run tests after building; only for local builds')
|
| parser.add_argument('--tools', nargs='*',
|
| + help='select which chrome tools to build',
|
| default=['plugins', 'blink_gc_plugin'])
|
| - # For now, this flag is only used for the non-Windows flow, but argparser gets
|
| - # mad if it sees a flag it doesn't recognize.
|
| - parser.add_argument('--if-needed', action='store_true')
|
| - parser.add_argument('--print-revision', action='store_true')
|
| - parser.add_argument('--run-tests', action='store_true')
|
| + parser.add_argument('--without-patches', action='store_false',
|
| + help="don't apply patches (default)", dest='with_patches',
|
| + default=True)
|
| +
|
| + # For now, these flags are only used for the non-Windows flow, but argparser
|
| + # gets mad if it sees a flag it doesn't recognize.
|
| + parser.add_argument('--no-stdin-hack', action='store_true')
|
|
|
| args = parser.parse_args()
|
|
|
| + if args.if_needed:
|
| + is_clang_required = False
|
| + # clang is always used on Mac and Linux.
|
| + if sys.platform == 'darwin' or sys.platform.startswith('linux'):
|
| + is_clang_required = True
|
| + # clang requested via $GYP_DEFINES.
|
| + if re.search(r'\b(clang|asan|lsan|msan|tsan)=1',
|
| + os.environ.get('GYP_DEFINES', '')):
|
| + is_clang_required = True
|
| + # clang previously downloaded, keep it up-to-date.
|
| + # If you don't want this, delete third_party/llvm-build on your machine.
|
| + if os.path.isdir(LLVM_BUILD_DIR):
|
| + is_clang_required = True
|
| + if not is_clang_required:
|
| + return 0
|
| + if re.search(r'\b(make_clang_dir)=', os.environ.get('GYP_DEFINES', '')):
|
| + print 'Skipping Clang update (make_clang_dir= was set in GYP_DEFINES).'
|
| + return 0
|
| +
|
| + global LLVM_WIN_REVISION, PACKAGE_VERSION
|
| if args.print_revision:
|
| - PrintRevision()
|
| + if use_head_revision:
|
| + print GetSvnRevision(LLVM_DIR)
|
| + else:
|
| + print PACKAGE_VERSION
|
| return 0
|
|
|
| - if not re.search(r'\b(clang|asan)=1', os.environ.get('GYP_DEFINES', '')):
|
| - print 'Skipping Clang update (clang=1 was not set in GYP_DEFINES).'
|
| + if args.print_clang_version:
|
| + sys.stdout.write(VERSION)
|
| return 0
|
|
|
| - if re.search(r'\b(make_clang_dir)=', os.environ.get('GYP_DEFINES', '')):
|
| - print 'Skipping Clang update (make_clang_dir= was set in GYP_DEFINES).'
|
| - return 0
|
| + # Don't buffer stdout, so that print statements are immediately flushed.
|
| + # Do this only after --print-revision has been handled, else we'll get
|
| + # an error message when this script is run from gn for some reason.
|
| + sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
|
| +
|
| + if use_head_revision:
|
| + # Use a real revision number rather than HEAD to make sure that the stamp
|
| + # file logic works.
|
| + LLVM_WIN_REVISION = GetSvnRevision(LLVM_REPO_URL)
|
| + PACKAGE_VERSION = LLVM_WIN_REVISION + '-0'
|
| +
|
| + args.force_local_build = True
|
| + # Skip local patches when using HEAD: they probably don't apply anymore.
|
| + args.with_patches = False
|
|
|
| return UpdateClang(args)
|
|
|
|
|
Chromium Code Reviews
