update.py: Check out and build libc++ libc++abi on OS X, set deployment target.

tools/clang/scripts/update.py

 #!/usr/bin/env python
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Windows can't run .sh files, so this is a Python implementation of
 update.sh. This script should replace update.sh on all platforms eventually."""
 
 import argparse
 import contextlib
@@ skipping 34 matching lines @@
 LLVM_BOOTSTRAP_DIR = os.path.join(THIRD_PARTY_DIR, 'llvm-bootstrap')
 LLVM_BOOTSTRAP_INSTALL_DIR = os.path.join(THIRD_PARTY_DIR,
                                           'llvm-bootstrap-install')
 CHROME_TOOLS_SHIM_DIR = os.path.join(LLVM_DIR, 'tools', 'chrometools')
 LLVM_BUILD_DIR = os.path.join(CHROMIUM_DIR, 'third_party', 'llvm-build',
                               'Release+Asserts')
 COMPILER_RT_BUILD_DIR = os.path.join(LLVM_BUILD_DIR, '32bit-compiler-rt')
 CLANG_DIR = os.path.join(LLVM_DIR, 'tools', 'clang')
 LLD_DIR = os.path.join(LLVM_DIR, 'tools', 'lld')
 COMPILER_RT_DIR = os.path.join(LLVM_DIR, 'projects', 'compiler-rt')
+LIBCXX_DIR = os.path.join(LLVM_DIR, 'projects', 'libcxx')
+LIBCXXABI_DIR = os.path.join(LLVM_DIR, 'projects', 'libcxxabi')
 LLVM_BUILD_TOOLS_DIR = os.path.abspath(
     os.path.join(LLVM_DIR, '..', 'llvm-build-tools'))
 STAMP_FILE = os.path.join(LLVM_DIR, '..', 'llvm-build', 'cr_build_revision')
 BINUTILS_DIR = os.path.join(THIRD_PARTY_DIR, 'binutils')
 VERSION = '3.7.0'
 
 # URL for pre-built binaries.
 CDS_URL = 'https://commondatastorage.googleapis.com/chromium-browser-clang'
 
 LLVM_REPO_URL='https://llvm.org/svn/llvm-project'
@@ skipping 28 matching lines @@
   """Return the contents of the stamp file, or '' if it doesn't exist."""
   try:
     with open(STAMP_FILE, 'r') as f:
       return f.read();
   except IOError:
     return ''
 
 
 def WriteStampFile(s):
   """Write s to the stamp file."""
-  if not os.path.exists(LLVM_BUILD_DIR):
-    os.makedirs(LLVM_BUILD_DIR)
+  if not os.path.exists(os.path.dirname(STAMP_FILE)):
+    os.makedirs(os.path.dirname(STAMP_FILE))
   with open(STAMP_FILE, 'w') as f:
     f.write(s)
 
 
 def GetSvnRevision(svn_repo):
   """Returns current revision of the svn repo at svn_repo."""
-  svn_info = subprocess.check_output(['svn', 'info', svn_repo], shell=True)
+  svn_info = subprocess.check_output('svn info ' + svn_repo, shell=True)

[hans, 2015/06/09 23:57:45]

Living and learning..

   m = re.search(r'Revision: (\d+)', svn_info)
   return m.group(1)
 
 
 def RmTree(dir):
   """Delete dir."""
   def ChmodAndRetry(func, path, _):
     # Subversion can leave read-only files around.
     if not os.access(path, os.W_OK):
       os.chmod(path, stat.S_IWUSR)
       return func(path)
     raise
 
   shutil.rmtree(dir, onerror=ChmodAndRetry)
 
 
-def RunCommand(command, fail_hard=True):
+def RunCommand(command, env=None, fail_hard=True):
   """Run command and return success (True) or failure; or if fail_hard is
      True, exit on failure."""
 
   print 'Running %s' % (str(command))
-  if subprocess.call(command, shell=True) == 0:
+  if subprocess.call(' '.join(command), env=env, shell=True) == 0:

[hans, 2015/06/09 23:57:45]

I guess this makes us fail if we ever have files with spaces anywhere :-/

     return True
   print 'Failed.'
   if fail_hard:
     sys.exit(1)
   return False
 
 
 def CopyFile(src, dst):
   """Copy a file from src to dst."""
   shutil.copy(src, dst)
@@ skipping 70 matching lines @@
   if not os.path.exists(cmake_dir):
     if not os.path.exists(LLVM_BUILD_TOOLS_DIR):
       os.makedirs(LLVM_BUILD_TOOLS_DIR)
     # The cmake archive is smaller than 20 MB, small enough to keep in memory:
     with contextlib.closing(cStringIO.StringIO()) as f:
       DownloadUrl(CDS_URL + '/tools/' + zip_name, f)
       f.seek(0)
       if zip_name.endswith('.zip'):
         zipfile.ZipFile(f).extractall(path=LLVM_BUILD_TOOLS_DIR)
       else:
-        tarfile.open(mode='r:gz', fileobj=f).extractall(path=LLVM_BUILD_DIR)
+        tarfile.open(mode='r:gz', fileobj=f).extractall(path=
+            LLVM_BUILD_TOOLS_DIR)
   os.environ['PATH'] = cmake_dir + os.pathsep + os.environ.get('PATH', '')
 
 vs_version = None
 def GetVSVersion():
   global vs_version
   if vs_version:
     return vs_version
 
   # Try using the toolchain in depot_tools.
   # This sets environment variables used by SelectVisualStudioVersion below.
@@ skipping 37 matching lines @@
         WriteStampFile(PACKAGE_VERSION)
         return 0
       except urllib2.HTTPError:
         print 'Did not find prebuilt clang %s, building locally' % cds_file
 
   AddCMakeToPath()
 
   DeleteChromeToolsShim();
   Checkout('LLVM', LLVM_REPO_URL + '/llvm/trunk', LLVM_DIR)
   Checkout('Clang', LLVM_REPO_URL + '/cfe/trunk', CLANG_DIR)
-  Checkout('LLD', LLVM_REPO_URL + '/lld/trunk', LLD_DIR)
+  if sys.platform == 'win32':
+    Checkout('LLD', LLVM_REPO_URL + '/lld/trunk', LLD_DIR)
   Checkout('compiler-rt', LLVM_REPO_URL + '/compiler-rt/trunk', COMPILER_RT_DIR)
+  if sys.platform == 'darwin':
+    # clang needs a libc++ checkout, else -stdlib=libc++ won't find includes
+    # (i.e. this is needed for bootstrap builds).
+    Checkout('libcxx', LLVM_REPO_URL + '/libcxx/trunk', LIBCXX_DIR)
+    # While we're bundling our own libc++ on OS X, we need to compile libc++abi
+    # into it too (since OS X 10.6 doesn't have libc++abi.dylib either).
+    Checkout('libcxxabi', LLVM_REPO_URL + '/libcxxabi/trunk', LIBCXXABI_DIR)
+
   CreateChromeToolsShim();
 
-  # If building at head, define a macro that plugins can use for #ifdefing
-  # out code that builds at head, but not at CLANG_REVISION or vice versa.
-  cflags = cxxflags = ''
+  cc = os.environ.get('CC')
+  cxx = os.environ.get('CXX')

[hans, 2015/06/09 23:57:45]

do we really need this?

[Nico, 2015/06/10 00:03:16]

guess not. removed, thanks.

 
-  # If building at head, define a macro that plugins can use for #ifdefing
-  # out code that builds at head, but not at LLVM_WIN_REVISION or vice versa.
-  if use_head_revision:
-    cflags += ' -DLLVM_FORCE_HEAD_REVISION'
-    cxxflags += ' -DLLVM_FORCE_HEAD_REVISION'
+  cflags = cxxflags = ldflags = ''
+
+  # LLVM uses C++11 starting in llvm 3.5. On Linux, this means libstdc++4.7+ is
+  # needed, on OS X it requires libc++. clang only automatically links to libc++
+  # when targeting OS X 10.9+, so add stdlib=libc++ explicitly so clang can run
+  # on OS X versions as old as 10.7.
+  # TODO(thakis): Some bots are still on 10.6 (nacl...), so for now bundle
+  # libc++.dylib.  Remove this once all bots are on 10.7+, then use
+  # -DLLVM_ENABLE_LIBCXX=ON and change deployment_target to 10.7.
+  deployment_target = ''
+
+  if sys.platform == 'darwin':
+    # When building on 10.9, /usr/include usually doesn't exist, and while
+    # Xcode's clang automatically sets a sysroot, self-built clangs don't.
+    cflags = "-isysroot " + subprocess.check_output(
+        ['xcrun', '--show-sdk-path']).rstrip()
+    cxxflags = '-stdlib=libc++ -nostdinc++ -I%s %s' % (
+        os.path.join(LIBCXX_DIR, 'include'), cflags)
+    if args.bootstrap:
+      deployment_target = '10.6'
 
   base_cmake_args = ['-GNinja',
                      '-DCMAKE_BUILD_TYPE=Release',
                      '-DLLVM_ENABLE_ASSERTIONS=ON',
                      '-DLLVM_ENABLE_THREADS=OFF',
                      ]
 
-  cc, cxx = None, None
   if args.bootstrap:
     print 'Building bootstrap compiler'
     if not os.path.exists(LLVM_BOOTSTRAP_DIR):
       os.makedirs(LLVM_BOOTSTRAP_DIR)
     os.chdir(LLVM_BOOTSTRAP_DIR)
     bootstrap_args = base_cmake_args + [
         '-DLLVM_TARGETS_TO_BUILD=host',
         '-DCMAKE_INSTALL_PREFIX=' + LLVM_BOOTSTRAP_INSTALL_DIR,
         '-DCMAKE_C_FLAGS=' + cflags,
         '-DCMAKE_CXX_FLAGS=' + cxxflags,
@@ skipping 10 matching lines @@
     # TODO(thakis): Set these to clang / clang++ on posix once this script
     # is used on posix.
     cc = os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'bin', 'clang-cl.exe')
     cxx = os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'bin', 'clang-cl.exe')
     # CMake has a hard time with backslashes in compiler paths:
     # https://stackoverflow.com/questions/13050827
     cc = cc.replace('\\', '/')
     cxx = cxx.replace('\\', '/')
     print 'Building final compiler'
 
+  if sys.platform == 'darwin':
+    # Build libc++.dylib while some bots are still on OS X 10.6.
+    RmTree(os.path.join(LLVM_BUILD_DIR, 'libcxxbuild'))
+    libcxxflags = "-O3 -std=c++11 -fstrict-aliasing"
+
+    # libcxx and libcxxabi both have a file stdexcept.cpp, so put their .o files
+    # into different subdirectories.
+    os.makedirs(os.path.join(LLVM_BUILD_DIR, 'libcxxbuild', 'libcxx'))
+    os.chdir(os.path.join(LLVM_BUILD_DIR, 'libcxxbuild', 'libcxx'))
+    RunCommand(['c++', '-c', cxxflags, libcxxflags,
+                os.path.join(LIBCXX_DIR, 'src', '*.cpp')])
+
+    os.makedirs(os.path.join(LLVM_BUILD_DIR, 'libcxxbuild', 'libcxxabi'))
+    os.chdir(os.path.join(LLVM_BUILD_DIR, 'libcxxbuild', 'libcxxabi'))
+    RunCommand(['c++', '-c', cxxflags, libcxxflags,
+                os.path.join(LIBCXXABI_DIR, 'src', '*.cpp'),
+                '-I' + os.path.join(LIBCXXABI_DIR, 'include')])
+
+    os.chdir(os.path.join(LLVM_BUILD_DIR, 'libcxxbuild'))
+    libdir = os.path.join(LIBCXX_DIR, 'lib')
+    RunCommand(['cc', 'libcxx/*.o', 'libcxxabi/*.o', '-o', 'libc++.1.dylib',
+        '-dynamiclib', '-nodefaultlibs', '-current_version', '1',
+        '-compatibility_version', '1', '-lSystem', '-install_name',
+        '@executable_path/libc++.dylib',
+        '-Wl,-unexported_symbols_list,' + libdir + '/libc++unexp.exp',
+        '-Wl,-force_symbols_not_weak_list,' + libdir + '/notweak.exp',
+        '-Wl,-force_symbols_weak_list,' + libdir + '/weak.exp'])
+    if os.path.exists('libc++.dylib'):
+      os.remove('libc++.dylib')
+    os.symlink('libc++.1.dylib', 'libc++.dylib')
+    ldflags += '-stdlib=libc++ -L' + os.path.join(LLVM_BUILD_DIR, 'libcxxbuild')
+
   # Build clang.
   binutils_incdir = ''
   if sys.platform.startswith('linux'):
     binutils_incdir = os.path.join(BINUTILS_DIR, 'Linux_x64/Release/include')
 
+  # If building at head, define a macro that plugins can use for #ifdefing
+  # out code that builds at head, but not at LLVM_WIN_REVISION or vice versa.
+  if use_head_revision:
+    cflags += ' -DLLVM_FORCE_HEAD_REVISION'
+    cxxflags += ' -DLLVM_FORCE_HEAD_REVISION'
+
+  deployment_env = os.environ.copy()
+  if deployment_target:
+    deployment_env['MACOSX_DEPLOYMENT_TARGET'] = deployment_target
+
   cmake_args = base_cmake_args + [
       '-DLLVM_BINUTILS_INCDIR=' + binutils_incdir,
       '-DCMAKE_C_FLAGS=' + cflags,
       '-DCMAKE_CXX_FLAGS=' + cxxflags,
+      '-DCMAKE_EXE_LINKER_FLAGS="' + ldflags,
+      '-DCMAKE_SHARED_LINKER_FLAGS="' + ldflags,
+      '-DCMAKE_MODULE_LINKER_FLAGS="' + ldflags,
       '-DCHROMIUM_TOOLS_SRC=%s' % os.path.join(CHROMIUM_DIR, 'tools', 'clang'),
       '-DCHROMIUM_TOOLS=%s' % ';'.join(args.tools)]
   # TODO(thakis): Append this to base_cmake_args instead once compiler-rt
   # can build with clang-cl (http://llvm.org/PR23698)
   if cc is not None:  cmake_args.append('-DCMAKE_C_COMPILER=' + cc)
   if cxx is not None: cmake_args.append('-DCMAKE_CXX_COMPILER=' + cxx)
 
   if not os.path.exists(LLVM_BUILD_DIR):
     os.makedirs(LLVM_BUILD_DIR)
   os.chdir(LLVM_BUILD_DIR)
-  RunCommand(GetVSVersion().SetupScript('x64') +
-             ['&&', 'cmake'] + cmake_args + [LLVM_DIR])
+  RunCommand(GetVSVersion().SetupScript('x64') + ['&&', 'cmake'] + cmake_args +
+             [LLVM_DIR], env=deployment_env)
   RunCommand(GetVSVersion().SetupScript('x64') + ['&&', 'ninja', 'all'])
 
+  # TODO(thakis): Run `strip bin/clang` on posix (with -x on darwin)
+
+  if sys.platform == 'darwin':
+    CopyFile(os.path.join(LLVM_BUILD_DIR, 'libc++.1.dylib'),
+             os.path.join(LLVM_BUILD_DIR, 'bin'))
+
   # Do an x86 build of compiler-rt to get the 32-bit ASan run-time.
   # TODO(hans): Remove once the regular build above produces this.
   if not os.path.exists(COMPILER_RT_BUILD_DIR):
     os.makedirs(COMPILER_RT_BUILD_DIR)
   os.chdir(COMPILER_RT_BUILD_DIR)
   # TODO(thakis): Add this once compiler-rt can build with clang-cl (see
   # above).
   #if args.bootstrap:
     # The bootstrap compiler produces 64-bit binaries by default.
     #cflags += ' -m32'
     #cxxflags += ' -m32'
   compiler_rt_args = base_cmake_args + [
       '-DCMAKE_C_FLAGS=' + cflags,
       '-DCMAKE_CXX_FLAGS=' + cxxflags]
-  RunCommand(GetVSVersion().SetupScript('x86') +
-             ['&&', 'cmake'] + compiler_rt_args + [LLVM_DIR])
+  RunCommand(GetVSVersion().SetupScript('x86') + ['&&', 'cmake'] +
+             compiler_rt_args + [LLVM_DIR], env=deployment_env)
   RunCommand(GetVSVersion().SetupScript('x86') + ['&&', 'ninja', 'compiler-rt'])
 
   # TODO(hans): Make this (and the .gypi and .isolate files) version number
   # independent.
   asan_rt_lib_src_dir = os.path.join(COMPILER_RT_BUILD_DIR, 'lib', 'clang',
                                      VERSION, 'lib', 'windows')
   asan_rt_lib_dst_dir = os.path.join(LLVM_BUILD_DIR, 'lib', 'clang',
                                      VERSION, 'lib', 'windows')
   CopyDirectoryContents(asan_rt_lib_src_dir, asan_rt_lib_dst_dir,
                         r'^.*-i386\.lib$')
@@ skipping 109 matching lines @@
     # Use a real revision number rather than HEAD to make sure that the stamp
     # file logic works.
     LLVM_WIN_REVISION = GetSvnRevision(LLVM_REPO_URL)
     PACKAGE_VERSION = LLVM_WIN_REVISION + '-0'
 
   return UpdateClang(args)
 
 
 if __name__ == '__main__':
   sys.exit(main())