OLD | NEW |
(Empty) | |
| 1 #!/usr/bin/env python |
| 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. |
| 3 # Use of this source code is governed by a BSD-style license that can be |
| 4 # found in the LICENSE file. |
| 5 |
| 6 """Windows can't run .sh files, so this is a Python implementation of |
| 7 update.sh. This script should replace update.sh on all platforms eventually.""" |
| 8 |
| 9 import argparse |
| 10 import contextlib |
| 11 import cStringIO |
| 12 import glob |
| 13 import os |
| 14 import pipes |
| 15 import re |
| 16 import shutil |
| 17 import subprocess |
| 18 import stat |
| 19 import sys |
| 20 import tarfile |
| 21 import time |
| 22 import urllib2 |
| 23 import zipfile |
| 24 |
| 25 # Do NOT CHANGE this if you don't know what you're doing -- see |
| 26 # https://code.google.com/p/chromium/wiki/UpdatingClang |
| 27 # Reverting problematic clang rolls is safe, though. |
| 28 # Note: this revision is only used for Windows. Other platforms use update.sh. |
| 29 # TODO(thakis): Use the same revision on Windows and non-Windows. |
| 30 # TODO(thakis): Remove update.sh, use update.py everywhere. |
| 31 LLVM_WIN_REVISION = '246985' |
| 32 |
| 33 use_head_revision = 'LLVM_FORCE_HEAD_REVISION' in os.environ |
| 34 if use_head_revision: |
| 35 LLVM_WIN_REVISION = 'HEAD' |
| 36 |
| 37 # This is incremented when pushing a new build of Clang at the same revision. |
| 38 CLANG_SUB_REVISION=1 |
| 39 |
| 40 PACKAGE_VERSION = "%s-%s" % (LLVM_WIN_REVISION, CLANG_SUB_REVISION) |
| 41 |
| 42 # Path constants. (All of these should be absolute paths.) |
| 43 THIS_DIR = os.path.abspath(os.path.dirname(__file__)) |
| 44 CHROMIUM_DIR = os.path.abspath(os.path.join(THIS_DIR, '..', '..', '..')) |
| 45 THIRD_PARTY_DIR = os.path.join(CHROMIUM_DIR, 'third_party') |
| 46 LLVM_DIR = os.path.join(THIRD_PARTY_DIR, 'llvm') |
| 47 LLVM_BOOTSTRAP_DIR = os.path.join(THIRD_PARTY_DIR, 'llvm-bootstrap') |
| 48 LLVM_BOOTSTRAP_INSTALL_DIR = os.path.join(THIRD_PARTY_DIR, |
| 49 'llvm-bootstrap-install') |
| 50 CHROME_TOOLS_SHIM_DIR = os.path.join(LLVM_DIR, 'tools', 'chrometools') |
| 51 LLVM_BUILD_DIR = os.path.join(CHROMIUM_DIR, 'third_party', 'llvm-build', |
| 52 'Release+Asserts') |
| 53 COMPILER_RT_BUILD_DIR = os.path.join(LLVM_BUILD_DIR, '32bit-compiler-rt') |
| 54 CLANG_DIR = os.path.join(LLVM_DIR, 'tools', 'clang') |
| 55 LLD_DIR = os.path.join(LLVM_DIR, 'tools', 'lld') |
| 56 COMPILER_RT_DIR = os.path.join(LLVM_DIR, 'projects', 'compiler-rt') |
| 57 LIBCXX_DIR = os.path.join(LLVM_DIR, 'projects', 'libcxx') |
| 58 LIBCXXABI_DIR = os.path.join(LLVM_DIR, 'projects', 'libcxxabi') |
| 59 LLVM_BUILD_TOOLS_DIR = os.path.abspath( |
| 60 os.path.join(LLVM_DIR, '..', 'llvm-build-tools')) |
| 61 STAMP_FILE = os.path.join(LLVM_DIR, '..', 'llvm-build', 'cr_build_revision') |
| 62 BINUTILS_DIR = os.path.join(THIRD_PARTY_DIR, 'binutils') |
| 63 VERSION = '3.8.0' |
| 64 |
| 65 # URL for pre-built binaries. |
| 66 CDS_URL = 'https://commondatastorage.googleapis.com/chromium-browser-clang' |
| 67 |
| 68 LLVM_REPO_URL='https://llvm.org/svn/llvm-project' |
| 69 if 'LLVM_REPO_URL' in os.environ: |
| 70 LLVM_REPO_URL = os.environ['LLVM_REPO_URL'] |
| 71 |
| 72 |
| 73 def DownloadUrl(url, output_file): |
| 74 """Download url into output_file.""" |
| 75 CHUNK_SIZE = 4096 |
| 76 TOTAL_DOTS = 10 |
| 77 sys.stdout.write('Downloading %s ' % url) |
| 78 sys.stdout.flush() |
| 79 response = urllib2.urlopen(url) |
| 80 total_size = int(response.info().getheader('Content-Length').strip()) |
| 81 bytes_done = 0 |
| 82 dots_printed = 0 |
| 83 while True: |
| 84 chunk = response.read(CHUNK_SIZE) |
| 85 if not chunk: |
| 86 break |
| 87 output_file.write(chunk) |
| 88 bytes_done += len(chunk) |
| 89 num_dots = TOTAL_DOTS * bytes_done / total_size |
| 90 sys.stdout.write('.' * (num_dots - dots_printed)) |
| 91 sys.stdout.flush() |
| 92 dots_printed = num_dots |
| 93 print ' Done.' |
| 94 |
| 95 |
| 96 def ReadStampFile(): |
| 97 """Return the contents of the stamp file, or '' if it doesn't exist.""" |
| 98 try: |
| 99 with open(STAMP_FILE, 'r') as f: |
| 100 return f.read() |
| 101 except IOError: |
| 102 return '' |
| 103 |
| 104 |
| 105 def WriteStampFile(s): |
| 106 """Write s to the stamp file.""" |
| 107 if not os.path.exists(os.path.dirname(STAMP_FILE)): |
| 108 os.makedirs(os.path.dirname(STAMP_FILE)) |
| 109 with open(STAMP_FILE, 'w') as f: |
| 110 f.write(s) |
| 111 |
| 112 |
| 113 def GetSvnRevision(svn_repo): |
| 114 """Returns current revision of the svn repo at svn_repo.""" |
| 115 svn_info = subprocess.check_output('svn info ' + svn_repo, shell=True) |
| 116 m = re.search(r'Revision: (\d+)', svn_info) |
| 117 return m.group(1) |
| 118 |
| 119 |
| 120 def RmTree(dir): |
| 121 """Delete dir.""" |
| 122 def ChmodAndRetry(func, path, _): |
| 123 # Subversion can leave read-only files around. |
| 124 if not os.access(path, os.W_OK): |
| 125 os.chmod(path, stat.S_IWUSR) |
| 126 return func(path) |
| 127 raise |
| 128 |
| 129 shutil.rmtree(dir, onerror=ChmodAndRetry) |
| 130 |
| 131 |
| 132 def RunCommand(command, msvc_arch=None, env=None, fail_hard=True): |
| 133 """Run command and return success (True) or failure; or if fail_hard is |
| 134 True, exit on failure. If msvc_arch is set, runs the command in a |
| 135 shell with the msvc tools for that architecture.""" |
| 136 |
| 137 if msvc_arch and sys.platform == 'win32': |
| 138 command = GetVSVersion().SetupScript(msvc_arch) + ['&&'] + command |
| 139 |
| 140 # https://docs.python.org/2/library/subprocess.html: |
| 141 # "On Unix with shell=True [...] if args is a sequence, the first item |
| 142 # specifies the command string, and any additional items will be treated as |
| 143 # additional arguments to the shell itself. That is to say, Popen does the |
| 144 # equivalent of: |
| 145 # Popen(['/bin/sh', '-c', args[0], args[1], ...])" |
| 146 # |
| 147 # We want to pass additional arguments to command[0], not to the shell, |
| 148 # so manually join everything into a single string. |
| 149 # Annoyingly, for "svn co url c:\path", pipes.quote() thinks that it should |
| 150 # quote c:\path but svn can't handle quoted paths on Windows. Since on |
| 151 # Windows follow-on args are passed to args[0] instead of the shell, don't |
| 152 # do the single-string transformation there. |
| 153 if sys.platform != 'win32': |
| 154 command = ' '.join([pipes.quote(c) for c in command]) |
| 155 print 'Running', command |
| 156 if subprocess.call(command, env=env, shell=True) == 0: |
| 157 return True |
| 158 print 'Failed.' |
| 159 if fail_hard: |
| 160 sys.exit(1) |
| 161 return False |
| 162 |
| 163 |
| 164 def CopyFile(src, dst): |
| 165 """Copy a file from src to dst.""" |
| 166 shutil.copy(src, dst) |
| 167 print "Copying %s to %s" % (src, dst) |
| 168 |
| 169 |
| 170 def CopyDirectoryContents(src, dst, filename_filter=None): |
| 171 """Copy the files from directory src to dst |
| 172 with an optional filename filter.""" |
| 173 if not os.path.exists(dst): |
| 174 os.makedirs(dst) |
| 175 for root, _, files in os.walk(src): |
| 176 for f in files: |
| 177 if filename_filter and not re.match(filename_filter, f): |
| 178 continue |
| 179 CopyFile(os.path.join(root, f), dst) |
| 180 |
| 181 |
| 182 def Checkout(name, url, dir): |
| 183 """Checkout the SVN module at url into dir. Use name for the log message.""" |
| 184 print "Checking out %s r%s into '%s'" % (name, LLVM_WIN_REVISION, dir) |
| 185 |
| 186 command = ['svn', 'checkout', '--force', url + '@' + LLVM_WIN_REVISION, dir] |
| 187 if RunCommand(command, fail_hard=False): |
| 188 return |
| 189 |
| 190 if os.path.isdir(dir): |
| 191 print "Removing %s." % (dir) |
| 192 RmTree(dir) |
| 193 |
| 194 print "Retrying." |
| 195 RunCommand(command) |
| 196 |
| 197 |
| 198 def RevertPreviouslyPatchedFiles(): |
| 199 print 'Reverting previously patched files' |
| 200 files = [ |
| 201 '%(clang)s/test/Index/crash-recovery-modules.m', |
| 202 '%(clang)s/unittests/libclang/LibclangTest.cpp', |
| 203 '%(compiler_rt)s/lib/asan/asan_rtl.cc', |
| 204 '%(compiler_rt)s/test/asan/TestCases/Linux/new_array_cookie_test.cc', |
| 205 '%(llvm)s/test/DebugInfo/gmlt.ll', |
| 206 '%(llvm)s/lib/CodeGen/SpillPlacement.cpp', |
| 207 '%(llvm)s/lib/CodeGen/SpillPlacement.h', |
| 208 '%(llvm)s/lib/Transforms/Instrumentation/MemorySanitizer.cpp', |
| 209 '%(clang)s/test/Driver/env.c', |
| 210 '%(clang)s/lib/Frontend/InitPreprocessor.cpp', |
| 211 '%(clang)s/test/Frontend/exceptions.c', |
| 212 '%(clang)s/test/Preprocessor/predefined-exceptions.m', |
| 213 '%(llvm)s/test/Bindings/Go/go.test', |
| 214 '%(clang)s/lib/Parse/ParseExpr.cpp', |
| 215 '%(clang)s/lib/Parse/ParseTemplate.cpp', |
| 216 '%(clang)s/lib/Sema/SemaDeclCXX.cpp', |
| 217 '%(clang)s/lib/Sema/SemaExprCXX.cpp', |
| 218 '%(clang)s/test/SemaCXX/default2.cpp', |
| 219 '%(clang)s/test/SemaCXX/typo-correction-delayed.cpp', |
| 220 '%(compiler_rt)s/lib/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.c
c', |
| 221 '%(compiler_rt)s/test/tsan/signal_segv_handler.cc', |
| 222 '%(compiler_rt)s/lib/sanitizer_common/sanitizer_coverage_libcdep.cc', |
| 223 '%(compiler_rt)s/cmake/config-ix.cmake', |
| 224 '%(compiler_rt)s/CMakeLists.txt', |
| 225 '%(compiler_rt)s/lib/ubsan/ubsan_platform.h', |
| 226 ] |
| 227 for f in files: |
| 228 f = f % { |
| 229 'clang': CLANG_DIR, |
| 230 'compiler_rt': COMPILER_RT_DIR, |
| 231 'llvm': LLVM_DIR, |
| 232 } |
| 233 if os.path.exists(f): |
| 234 os.remove(f) # For unversioned files. |
| 235 RunCommand(['svn', 'revert', f]) |
| 236 |
| 237 |
| 238 def ApplyLocalPatches(): |
| 239 # There's no patch program on Windows by default. We don't need patches on |
| 240 # Windows yet, and maybe this not working on Windows will motivate us to |
| 241 # remove patches over time. |
| 242 assert sys.platform != 'win32' |
| 243 |
| 244 # No patches. |
| 245 |
| 246 |
| 247 def DeleteChromeToolsShim(): |
| 248 OLD_SHIM_DIR = os.path.join(LLVM_DIR, 'tools', 'zzz-chrometools') |
| 249 shutil.rmtree(OLD_SHIM_DIR, ignore_errors=True) |
| 250 shutil.rmtree(CHROME_TOOLS_SHIM_DIR, ignore_errors=True) |
| 251 |
| 252 |
| 253 def CreateChromeToolsShim(): |
| 254 """Hooks the Chrome tools into the LLVM build. |
| 255 |
| 256 Several Chrome tools have dependencies on LLVM/Clang libraries. The LLVM build |
| 257 detects implicit tools in the tools subdirectory, so this helper install a |
| 258 shim CMakeLists.txt that forwards to the real directory for the Chrome tools. |
| 259 |
| 260 Note that the shim directory name intentionally has no - or _. The implicit |
| 261 tool detection logic munges them in a weird way.""" |
| 262 assert not any(i in os.path.basename(CHROME_TOOLS_SHIM_DIR) for i in '-_') |
| 263 os.mkdir(CHROME_TOOLS_SHIM_DIR) |
| 264 with file(os.path.join(CHROME_TOOLS_SHIM_DIR, 'CMakeLists.txt'), 'w') as f: |
| 265 f.write('# Automatically generated by tools/clang/scripts/update.py. ' + |
| 266 'Do not edit.\n') |
| 267 f.write('# Since tools/clang is located in another directory, use the \n') |
| 268 f.write('# two arg version to specify where build artifacts go. CMake\n') |
| 269 f.write('# disallows reuse of the same binary dir for multiple source\n') |
| 270 f.write('# dirs, so the build artifacts need to go into a subdirectory.\n') |
| 271 f.write('# dirs, so the build artifacts need to go into a subdirectory.\n') |
| 272 f.write('if (CHROMIUM_TOOLS_SRC)\n') |
| 273 f.write(' add_subdirectory(${CHROMIUM_TOOLS_SRC} ' + |
| 274 '${CMAKE_CURRENT_BINARY_DIR}/a)\n') |
| 275 f.write('endif (CHROMIUM_TOOLS_SRC)\n') |
| 276 |
| 277 |
| 278 def AddCMakeToPath(): |
| 279 """Download CMake and add it to PATH.""" |
| 280 if sys.platform == 'win32': |
| 281 zip_name = 'cmake-3.2.2-win32-x86.zip' |
| 282 cmake_dir = os.path.join(LLVM_BUILD_TOOLS_DIR, |
| 283 'cmake-3.2.2-win32-x86', 'bin') |
| 284 else: |
| 285 suffix = 'Darwin' if sys.platform == 'darwin' else 'Linux' |
| 286 zip_name = 'cmake310_%s.tgz' % suffix |
| 287 cmake_dir = os.path.join(LLVM_BUILD_TOOLS_DIR, 'cmake310', 'bin') |
| 288 if not os.path.exists(cmake_dir): |
| 289 if not os.path.exists(LLVM_BUILD_TOOLS_DIR): |
| 290 os.makedirs(LLVM_BUILD_TOOLS_DIR) |
| 291 # The cmake archive is smaller than 20 MB, small enough to keep in memory: |
| 292 with contextlib.closing(cStringIO.StringIO()) as f: |
| 293 DownloadUrl(CDS_URL + '/tools/' + zip_name, f) |
| 294 f.seek(0) |
| 295 if zip_name.endswith('.zip'): |
| 296 zipfile.ZipFile(f).extractall(path=LLVM_BUILD_TOOLS_DIR) |
| 297 else: |
| 298 tarfile.open(mode='r:gz', fileobj=f).extractall(path= |
| 299 LLVM_BUILD_TOOLS_DIR) |
| 300 os.environ['PATH'] = cmake_dir + os.pathsep + os.environ.get('PATH', '') |
| 301 |
| 302 vs_version = None |
| 303 def GetVSVersion(): |
| 304 global vs_version |
| 305 if vs_version: |
| 306 return vs_version |
| 307 |
| 308 # Try using the toolchain in depot_tools. |
| 309 # This sets environment variables used by SelectVisualStudioVersion below. |
| 310 sys.path.append(os.path.join(CHROMIUM_DIR, 'build')) |
| 311 import vs_toolchain |
| 312 vs_toolchain.SetEnvironmentAndGetRuntimeDllDirs() |
| 313 |
| 314 # Use gyp to find the MSVS installation, either in depot_tools as per above, |
| 315 # or a system-wide installation otherwise. |
| 316 sys.path.append(os.path.join(CHROMIUM_DIR, 'tools', 'gyp', 'pylib')) |
| 317 import gyp.MSVSVersion |
| 318 vs_version = gyp.MSVSVersion.SelectVisualStudioVersion('2013') |
| 319 return vs_version |
| 320 |
| 321 |
| 322 def UpdateClang(args): |
| 323 print 'Updating Clang to %s...' % PACKAGE_VERSION |
| 324 if ReadStampFile() == PACKAGE_VERSION: |
| 325 print 'Already up to date.' |
| 326 return 0 |
| 327 |
| 328 # Reset the stamp file in case the build is unsuccessful. |
| 329 WriteStampFile('') |
| 330 |
| 331 if not args.force_local_build: |
| 332 cds_file = "clang-%s.tgz" % PACKAGE_VERSION |
| 333 cds_full_url = CDS_URL + '/Win/' + cds_file |
| 334 |
| 335 # Check if there's a prebuilt binary and if so just fetch that. That's |
| 336 # faster, and goma relies on having matching binary hashes on client and |
| 337 # server too. |
| 338 print 'Trying to download prebuilt clang' |
| 339 |
| 340 # clang packages are smaller than 50 MB, small enough to keep in memory. |
| 341 with contextlib.closing(cStringIO.StringIO()) as f: |
| 342 try: |
| 343 DownloadUrl(cds_full_url, f) |
| 344 f.seek(0) |
| 345 # TODO(thakis): Delete LLVM_BUILD_DIR before extracting. |
| 346 tarfile.open(mode='r:gz', fileobj=f).extractall(path=LLVM_BUILD_DIR) |
| 347 print 'clang %s unpacked' % PACKAGE_VERSION |
| 348 # Download the gold plugin if requested to by an environment variable. |
| 349 # This is used by the CFI ClusterFuzz bot. |
| 350 if 'LLVM_DOWNLOAD_GOLD_PLUGIN' in os.environ: |
| 351 RunCommand(['python', CHROMIUM_DIR+'/build/download_gold_plugin.py']) |
| 352 WriteStampFile(PACKAGE_VERSION) |
| 353 return 0 |
| 354 except urllib2.HTTPError: |
| 355 print 'Did not find prebuilt clang %s, building locally' % cds_file |
| 356 |
| 357 AddCMakeToPath() |
| 358 |
| 359 RevertPreviouslyPatchedFiles() |
| 360 DeleteChromeToolsShim() |
| 361 |
| 362 Checkout('LLVM', LLVM_REPO_URL + '/llvm/trunk', LLVM_DIR) |
| 363 Checkout('Clang', LLVM_REPO_URL + '/cfe/trunk', CLANG_DIR) |
| 364 if sys.platform == 'win32': |
| 365 Checkout('LLD', LLVM_REPO_URL + '/lld/trunk', LLD_DIR) |
| 366 Checkout('compiler-rt', LLVM_REPO_URL + '/compiler-rt/trunk', COMPILER_RT_DIR) |
| 367 if sys.platform == 'darwin': |
| 368 # clang needs a libc++ checkout, else -stdlib=libc++ won't find includes |
| 369 # (i.e. this is needed for bootstrap builds). |
| 370 Checkout('libcxx', LLVM_REPO_URL + '/libcxx/trunk', LIBCXX_DIR) |
| 371 # While we're bundling our own libc++ on OS X, we need to compile libc++abi |
| 372 # into it too (since OS X 10.6 doesn't have libc++abi.dylib either). |
| 373 Checkout('libcxxabi', LLVM_REPO_URL + '/libcxxabi/trunk', LIBCXXABI_DIR) |
| 374 |
| 375 if args.with_patches and sys.platform != 'win32': |
| 376 ApplyLocalPatches() |
| 377 |
| 378 cc, cxx = None, None |
| 379 libstdcpp = None |
| 380 if args.gcc_toolchain: # This option is only used on Linux. |
| 381 # Use the specified gcc installation for building. |
| 382 cc = os.path.join(args.gcc_toolchain, 'bin', 'gcc') |
| 383 cxx = os.path.join(args.gcc_toolchain, 'bin', 'g++') |
| 384 |
| 385 if not os.access(cc, os.X_OK): |
| 386 print 'Invalid --gcc-toolchain: "%s"' % args.gcc_toolchain |
| 387 print '"%s" does not appear to be valid.' % cc |
| 388 return 1 |
| 389 |
| 390 # Set LD_LIBRARY_PATH to make auxiliary targets (tablegen, bootstrap |
| 391 # compiler, etc.) find the .so. |
| 392 libstdcpp = subprocess.check_output( |
| 393 [cxx, '-print-file-name=libstdc++.so.6']).rstrip() |
| 394 os.environ['LD_LIBRARY_PATH'] = os.path.dirname(libstdcpp) |
| 395 |
| 396 cflags = cxxflags = ldflags = [] |
| 397 |
| 398 # LLVM uses C++11 starting in llvm 3.5. On Linux, this means libstdc++4.7+ is |
| 399 # needed, on OS X it requires libc++. clang only automatically links to libc++ |
| 400 # when targeting OS X 10.9+, so add stdlib=libc++ explicitly so clang can run |
| 401 # on OS X versions as old as 10.7. |
| 402 # TODO(thakis): Some bots are still on 10.6 (nacl...), so for now bundle |
| 403 # libc++.dylib. Remove this once all bots are on 10.7+, then use |
| 404 # -DLLVM_ENABLE_LIBCXX=ON and change deployment_target to 10.7. |
| 405 deployment_target = '' |
| 406 |
| 407 if sys.platform == 'darwin': |
| 408 # When building on 10.9, /usr/include usually doesn't exist, and while |
| 409 # Xcode's clang automatically sets a sysroot, self-built clangs don't. |
| 410 cflags = ['-isysroot', subprocess.check_output( |
| 411 ['xcrun', '--show-sdk-path']).rstrip()] |
| 412 cxxflags = ['-stdlib=libc++', '-nostdinc++', |
| 413 '-I' + os.path.join(LIBCXX_DIR, 'include')] + cflags |
| 414 if args.bootstrap: |
| 415 deployment_target = '10.6' |
| 416 |
| 417 base_cmake_args = ['-GNinja', |
| 418 '-DCMAKE_BUILD_TYPE=Release', |
| 419 '-DLLVM_ENABLE_ASSERTIONS=ON', |
| 420 '-DLLVM_ENABLE_THREADS=OFF', |
| 421 ] |
| 422 |
| 423 if args.bootstrap: |
| 424 print 'Building bootstrap compiler' |
| 425 if not os.path.exists(LLVM_BOOTSTRAP_DIR): |
| 426 os.makedirs(LLVM_BOOTSTRAP_DIR) |
| 427 os.chdir(LLVM_BOOTSTRAP_DIR) |
| 428 bootstrap_args = base_cmake_args + [ |
| 429 '-DLLVM_TARGETS_TO_BUILD=host', |
| 430 '-DCMAKE_INSTALL_PREFIX=' + LLVM_BOOTSTRAP_INSTALL_DIR, |
| 431 '-DCMAKE_C_FLAGS=' + ' '.join(cflags), |
| 432 '-DCMAKE_CXX_FLAGS=' + ' '.join(cxxflags), |
| 433 ] |
| 434 if cc is not None: bootstrap_args.append('-DCMAKE_C_COMPILER=' + cc) |
| 435 if cxx is not None: bootstrap_args.append('-DCMAKE_CXX_COMPILER=' + cxx) |
| 436 RunCommand(['cmake'] + bootstrap_args + [LLVM_DIR], msvc_arch='x64') |
| 437 RunCommand(['ninja'], msvc_arch='x64') |
| 438 if args.run_tests: |
| 439 RunCommand(['ninja', 'check-all'], msvc_arch='x64') |
| 440 RunCommand(['ninja', 'install'], msvc_arch='x64') |
| 441 if args.gcc_toolchain: |
| 442 # Copy that gcc's stdlibc++.so.6 to the build dir, so the bootstrap |
| 443 # compiler can start. |
| 444 CopyFile(libstdcpp, os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'lib')) |
| 445 |
| 446 if sys.platform == 'win32': |
| 447 cc = os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'bin', 'clang-cl.exe') |
| 448 cxx = os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'bin', 'clang-cl.exe') |
| 449 # CMake has a hard time with backslashes in compiler paths: |
| 450 # https://stackoverflow.com/questions/13050827 |
| 451 cc = cc.replace('\\', '/') |
| 452 cxx = cxx.replace('\\', '/') |
| 453 else: |
| 454 cc = os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'bin', 'clang') |
| 455 cxx = os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'bin', 'clang++') |
| 456 |
| 457 if args.gcc_toolchain: |
| 458 # Tell the bootstrap compiler to use a specific gcc prefix to search |
| 459 # for standard library headers and shared object files. |
| 460 cflags = ['--gcc-toolchain=' + args.gcc_toolchain] |
| 461 cxxflags = ['--gcc-toolchain=' + args.gcc_toolchain] |
| 462 print 'Building final compiler' |
| 463 |
| 464 if sys.platform == 'darwin': |
| 465 # Build libc++.dylib while some bots are still on OS X 10.6. |
| 466 libcxxbuild = os.path.join(LLVM_BUILD_DIR, 'libcxxbuild') |
| 467 if os.path.isdir(libcxxbuild): |
| 468 RmTree(libcxxbuild) |
| 469 libcxxflags = ['-O3', '-std=c++11', '-fstrict-aliasing'] |
| 470 |
| 471 # libcxx and libcxxabi both have a file stdexcept.cpp, so put their .o files |
| 472 # into different subdirectories. |
| 473 os.makedirs(os.path.join(libcxxbuild, 'libcxx')) |
| 474 os.chdir(os.path.join(libcxxbuild, 'libcxx')) |
| 475 RunCommand(['c++', '-c'] + cxxflags + libcxxflags + |
| 476 glob.glob(os.path.join(LIBCXX_DIR, 'src', '*.cpp'))) |
| 477 |
| 478 os.makedirs(os.path.join(libcxxbuild, 'libcxxabi')) |
| 479 os.chdir(os.path.join(libcxxbuild, 'libcxxabi')) |
| 480 RunCommand(['c++', '-c'] + cxxflags + libcxxflags + |
| 481 glob.glob(os.path.join(LIBCXXABI_DIR, 'src', '*.cpp')) + |
| 482 ['-I' + os.path.join(LIBCXXABI_DIR, 'include')]) |
| 483 |
| 484 os.chdir(libcxxbuild) |
| 485 libdir = os.path.join(LIBCXX_DIR, 'lib') |
| 486 RunCommand(['cc'] + glob.glob('libcxx/*.o') + glob.glob('libcxxabi/*.o') + |
| 487 ['-o', 'libc++.1.dylib', '-dynamiclib', '-nodefaultlibs', |
| 488 '-current_version', '1', '-compatibility_version', '1', '-lSystem', |
| 489 '-install_name', '@executable_path/libc++.dylib', |
| 490 '-Wl,-unexported_symbols_list,' + libdir + '/libc++unexp.exp', |
| 491 '-Wl,-force_symbols_not_weak_list,' + libdir + '/notweak.exp', |
| 492 '-Wl,-force_symbols_weak_list,' + libdir + '/weak.exp']) |
| 493 if os.path.exists('libc++.dylib'): |
| 494 os.remove('libc++.dylib') |
| 495 os.symlink('libc++.1.dylib', 'libc++.dylib') |
| 496 ldflags += ['-stdlib=libc++', '-L' + libcxxbuild] |
| 497 |
| 498 if args.bootstrap: |
| 499 # Now that the libc++ headers have been installed and libc++.dylib is |
| 500 # built, delete the libc++ checkout again so that it's not part of the |
| 501 # main build below -- the libc++(abi) tests don't pass on OS X in |
| 502 # bootstrap builds (http://llvm.org/PR24068) |
| 503 RmTree(LIBCXX_DIR) |
| 504 RmTree(LIBCXXABI_DIR) |
| 505 cxxflags = ['-stdlib=libc++', '-nostdinc++', |
| 506 '-I' + os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, |
| 507 'include/c++/v1') |
| 508 ] + cflags |
| 509 |
| 510 # Build clang. |
| 511 binutils_incdir = '' |
| 512 if sys.platform.startswith('linux'): |
| 513 binutils_incdir = os.path.join(BINUTILS_DIR, 'Linux_x64/Release/include') |
| 514 |
| 515 # If building at head, define a macro that plugins can use for #ifdefing |
| 516 # out code that builds at head, but not at LLVM_WIN_REVISION or vice versa. |
| 517 if use_head_revision: |
| 518 cflags += ['-DLLVM_FORCE_HEAD_REVISION'] |
| 519 cxxflags += ['-DLLVM_FORCE_HEAD_REVISION'] |
| 520 |
| 521 CreateChromeToolsShim() |
| 522 |
| 523 deployment_env = None |
| 524 if deployment_target: |
| 525 deployment_env = os.environ.copy() |
| 526 deployment_env['MACOSX_DEPLOYMENT_TARGET'] = deployment_target |
| 527 |
| 528 cmake_args = base_cmake_args + [ |
| 529 '-DLLVM_BINUTILS_INCDIR=' + binutils_incdir, |
| 530 '-DLLVM_EXPERIMENTAL_TARGETS_TO_BUILD=WebAssembly', |
| 531 '-DCMAKE_C_FLAGS=' + ' '.join(cflags), |
| 532 '-DCMAKE_CXX_FLAGS=' + ' '.join(cxxflags), |
| 533 '-DCMAKE_EXE_LINKER_FLAGS=' + ' '.join(ldflags), |
| 534 '-DCMAKE_SHARED_LINKER_FLAGS=' + ' '.join(ldflags), |
| 535 '-DCMAKE_MODULE_LINKER_FLAGS=' + ' '.join(ldflags), |
| 536 '-DCMAKE_INSTALL_PREFIX=' + LLVM_BUILD_DIR, |
| 537 '-DCHROMIUM_TOOLS_SRC=%s' % os.path.join(CHROMIUM_DIR, 'tools', 'clang'), |
| 538 '-DCHROMIUM_TOOLS=%s' % ';'.join(args.tools)] |
| 539 # TODO(thakis): Unconditionally append this to base_cmake_args instead once |
| 540 # compiler-rt can build with clang-cl on Windows (http://llvm.org/PR23698) |
| 541 cc_args = base_cmake_args if sys.platform != 'win32' else cmake_args |
| 542 if cc is not None: cc_args.append('-DCMAKE_C_COMPILER=' + cc) |
| 543 if cxx is not None: cc_args.append('-DCMAKE_CXX_COMPILER=' + cxx) |
| 544 |
| 545 if not os.path.exists(LLVM_BUILD_DIR): |
| 546 os.makedirs(LLVM_BUILD_DIR) |
| 547 os.chdir(LLVM_BUILD_DIR) |
| 548 RunCommand(['cmake'] + cmake_args + [LLVM_DIR], |
| 549 msvc_arch='x64', env=deployment_env) |
| 550 |
| 551 if args.gcc_toolchain: |
| 552 # Copy in the right stdlibc++.so.6 so clang can start. |
| 553 if not os.path.exists(os.path.join(LLVM_BUILD_DIR, 'lib')): |
| 554 os.mkdir(os.path.join(LLVM_BUILD_DIR, 'lib')) |
| 555 libstdcpp = subprocess.check_output( |
| 556 [cxx] + cxxflags + ['-print-file-name=libstdc++.so.6']).rstrip() |
| 557 CopyFile(libstdcpp, os.path.join(LLVM_BUILD_DIR, 'lib')) |
| 558 |
| 559 RunCommand(['ninja'], msvc_arch='x64') |
| 560 |
| 561 if args.tools: |
| 562 # If any Chromium tools were built, install those now. |
| 563 RunCommand(['ninja', 'cr-install'], msvc_arch='x64') |
| 564 |
| 565 if sys.platform == 'darwin': |
| 566 CopyFile(os.path.join(LLVM_BUILD_DIR, 'libc++.1.dylib'), |
| 567 os.path.join(LLVM_BUILD_DIR, 'bin')) |
| 568 # See http://crbug.com/256342 |
| 569 RunCommand(['strip', '-x', os.path.join(LLVM_BUILD_DIR, 'bin', 'clang')]) |
| 570 elif sys.platform.startswith('linux'): |
| 571 RunCommand(['strip', os.path.join(LLVM_BUILD_DIR, 'bin', 'clang')]) |
| 572 |
| 573 # Do an x86 build of compiler-rt to get the 32-bit ASan run-time. |
| 574 # TODO(hans): Remove once the regular build above produces this. |
| 575 if not os.path.exists(COMPILER_RT_BUILD_DIR): |
| 576 os.makedirs(COMPILER_RT_BUILD_DIR) |
| 577 os.chdir(COMPILER_RT_BUILD_DIR) |
| 578 # TODO(thakis): Add this once compiler-rt can build with clang-cl (see |
| 579 # above). |
| 580 #if args.bootstrap and sys.platform == 'win32': |
| 581 # The bootstrap compiler produces 64-bit binaries by default. |
| 582 #cflags += ['-m32'] |
| 583 #cxxflags += ['-m32'] |
| 584 compiler_rt_args = base_cmake_args + [ |
| 585 '-DCMAKE_C_FLAGS=' + ' '.join(cflags), |
| 586 '-DCMAKE_CXX_FLAGS=' + ' '.join(cxxflags)] |
| 587 if sys.platform != 'win32': |
| 588 compiler_rt_args += ['-DLLVM_CONFIG_PATH=' + |
| 589 os.path.join(LLVM_BUILD_DIR, 'bin', 'llvm-config'), |
| 590 '-DSANITIZER_MIN_OSX_VERSION="10.7"'] |
| 591 RunCommand(['cmake'] + compiler_rt_args + [LLVM_DIR], |
| 592 msvc_arch='x86', env=deployment_env) |
| 593 RunCommand(['ninja', 'compiler-rt'], msvc_arch='x86') |
| 594 |
| 595 # TODO(hans): Make this (and the .gypi and .isolate files) version number |
| 596 # independent. |
| 597 if sys.platform == 'win32': |
| 598 platform = 'windows' |
| 599 elif sys.platform == 'darwin': |
| 600 platform = 'darwin' |
| 601 else: |
| 602 assert sys.platform.startswith('linux') |
| 603 platform = 'linux' |
| 604 asan_rt_lib_src_dir = os.path.join(COMPILER_RT_BUILD_DIR, 'lib', 'clang', |
| 605 VERSION, 'lib', platform) |
| 606 asan_rt_lib_dst_dir = os.path.join(LLVM_BUILD_DIR, 'lib', 'clang', |
| 607 VERSION, 'lib', platform) |
| 608 CopyDirectoryContents(asan_rt_lib_src_dir, asan_rt_lib_dst_dir, |
| 609 r'^.*-i386\.lib$') |
| 610 CopyDirectoryContents(asan_rt_lib_src_dir, asan_rt_lib_dst_dir, |
| 611 r'^.*-i386\.dll$') |
| 612 |
| 613 CopyFile(os.path.join(asan_rt_lib_src_dir, '..', '..', 'asan_blacklist.txt'), |
| 614 os.path.join(asan_rt_lib_dst_dir, '..', '..')) |
| 615 |
| 616 if sys.platform == 'win32': |
| 617 # Make an extra copy of the sanitizer headers, to be put on the include path |
| 618 # of the fallback compiler. |
| 619 sanitizer_include_dir = os.path.join(LLVM_BUILD_DIR, 'lib', 'clang', |
| 620 VERSION, 'include', 'sanitizer') |
| 621 aux_sanitizer_include_dir = os.path.join(LLVM_BUILD_DIR, 'lib', 'clang', |
| 622 VERSION, 'include_sanitizer', |
| 623 'sanitizer') |
| 624 if not os.path.exists(aux_sanitizer_include_dir): |
| 625 os.makedirs(aux_sanitizer_include_dir) |
| 626 for _, _, files in os.walk(sanitizer_include_dir): |
| 627 for f in files: |
| 628 CopyFile(os.path.join(sanitizer_include_dir, f), |
| 629 aux_sanitizer_include_dir) |
| 630 |
| 631 # Run tests. |
| 632 if args.run_tests or use_head_revision: |
| 633 os.chdir(LLVM_BUILD_DIR) |
| 634 RunCommand(GetVSVersion().SetupScript('x64') + |
| 635 ['&&', 'ninja', 'cr-check-all']) |
| 636 if args.run_tests: |
| 637 os.chdir(LLVM_BUILD_DIR) |
| 638 RunCommand(GetVSVersion().SetupScript('x64') + |
| 639 ['&&', 'ninja', 'check-all']) |
| 640 |
| 641 WriteStampFile(PACKAGE_VERSION) |
| 642 print 'Clang update was successful.' |
| 643 return 0 |
| 644 |
| 645 |
| 646 def main(): |
| 647 if not sys.platform in ['win32', 'cygwin']: |
| 648 # For non-Windows, fall back to update.sh. |
| 649 # TODO(hans): Make update.py replace update.sh completely. |
| 650 |
| 651 # This script is called by gclient. gclient opens its hooks subprocesses |
| 652 # with (stdout=subprocess.PIPE, stderr=subprocess.STDOUT) and then does |
| 653 # custom output processing that breaks printing '\r' characters for |
| 654 # single-line updating status messages as printed by curl and wget. |
| 655 # Work around this by setting stderr of the update.sh process to stdin (!): |
| 656 # gclient doesn't redirect stdin, and while stdin itself is read-only, a |
| 657 # dup()ed sys.stdin is writable, try |
| 658 # fd2 = os.dup(sys.stdin.fileno()); os.write(fd2, 'hi') |
| 659 # TODO: Fix gclient instead, http://crbug.com/95350 |
| 660 if '--no-stdin-hack' in sys.argv: |
| 661 sys.argv.remove('--no-stdin-hack') |
| 662 stderr = None |
| 663 else: |
| 664 try: |
| 665 stderr = os.fdopen(os.dup(sys.stdin.fileno())) |
| 666 except: |
| 667 stderr = sys.stderr |
| 668 return subprocess.call( |
| 669 [os.path.join(os.path.dirname(__file__), 'update.sh')] + sys.argv[1:], |
| 670 stderr=stderr) |
| 671 |
| 672 parser = argparse.ArgumentParser(description='Build Clang.') |
| 673 parser.add_argument('--bootstrap', action='store_true', |
| 674 help='first build clang with CC, then with itself.') |
| 675 parser.add_argument('--if-needed', action='store_true', |
| 676 help="run only if the script thinks clang is needed") |
| 677 parser.add_argument('--force-local-build', action='store_true', |
| 678 help="don't try to download prebuild binaries") |
| 679 parser.add_argument('--gcc-toolchain', help='set the version for which gcc ' |
| 680 'version be used for building; --gcc-toolchain=/opt/foo ' |
| 681 'picks /opt/foo/bin/gcc') |
| 682 parser.add_argument('--print-revision', action='store_true', |
| 683 help='print current clang revision and exit.') |
| 684 parser.add_argument('--print-clang-version', action='store_true', |
| 685 help='print current clang version (e.g. x.y.z) and exit.') |
| 686 parser.add_argument('--run-tests', action='store_true', |
| 687 help='run tests after building; only for local builds') |
| 688 parser.add_argument('--tools', nargs='*', |
| 689 help='select which chrome tools to build', |
| 690 default=['plugins', 'blink_gc_plugin']) |
| 691 parser.add_argument('--without-patches', action='store_false', |
| 692 help="don't apply patches (default)", dest='with_patches', |
| 693 default=True) |
| 694 |
| 695 # For now, these flags are only used for the non-Windows flow, but argparser |
| 696 # gets mad if it sees a flag it doesn't recognize. |
| 697 parser.add_argument('--no-stdin-hack', action='store_true') |
| 698 |
| 699 args = parser.parse_args() |
| 700 |
| 701 if args.if_needed: |
| 702 is_clang_required = False |
| 703 # clang is always used on Mac and Linux. |
| 704 if sys.platform == 'darwin' or sys.platform.startswith('linux'): |
| 705 is_clang_required = True |
| 706 # clang requested via $GYP_DEFINES. |
| 707 if re.search(r'\b(clang|asan|lsan|msan|tsan)=1', |
| 708 os.environ.get('GYP_DEFINES', '')): |
| 709 is_clang_required = True |
| 710 # clang previously downloaded, keep it up-to-date. |
| 711 # If you don't want this, delete third_party/llvm-build on your machine. |
| 712 if os.path.isdir(LLVM_BUILD_DIR): |
| 713 is_clang_required = True |
| 714 if not is_clang_required: |
| 715 return 0 |
| 716 if re.search(r'\b(make_clang_dir)=', os.environ.get('GYP_DEFINES', '')): |
| 717 print 'Skipping Clang update (make_clang_dir= was set in GYP_DEFINES).' |
| 718 return 0 |
| 719 |
| 720 global LLVM_WIN_REVISION, PACKAGE_VERSION |
| 721 if args.print_revision: |
| 722 if use_head_revision: |
| 723 print GetSvnRevision(LLVM_DIR) |
| 724 else: |
| 725 print PACKAGE_VERSION |
| 726 return 0 |
| 727 |
| 728 if args.print_clang_version: |
| 729 sys.stdout.write(VERSION) |
| 730 return 0 |
| 731 |
| 732 # Don't buffer stdout, so that print statements are immediately flushed. |
| 733 # Do this only after --print-revision has been handled, else we'll get |
| 734 # an error message when this script is run from gn for some reason. |
| 735 sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0) |
| 736 |
| 737 if use_head_revision: |
| 738 # Use a real revision number rather than HEAD to make sure that the stamp |
| 739 # file logic works. |
| 740 LLVM_WIN_REVISION = GetSvnRevision(LLVM_REPO_URL) |
| 741 PACKAGE_VERSION = LLVM_WIN_REVISION + '-0' |
| 742 |
| 743 args.force_local_build = True |
| 744 # Skip local patches when using HEAD: they probably don't apply anymore. |
| 745 args.with_patches = False |
| 746 |
| 747 return UpdateClang(args) |
| 748 |
| 749 |
| 750 if __name__ == '__main__': |
| 751 sys.exit(main()) |
OLD | NEW |