Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(280)

Side by Side Diff: tools/clang/scripts/update.py

Issue 2858873005: [infra] Roll clang to match the version used by Flutter (Closed)
Patch Set: Fix Mac Android build Created 3 years, 7 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
(Empty)
1 #!/usr/bin/env python
2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file.
5
6 """This script is used to download prebuilt clang binaries.
7
8 It is also used by package.py to build the prebuilt clang binaries."""
9
10 import argparse
11 import distutils.spawn
12 import glob
13 import os
14 import pipes
15 import re
16 import shutil
17 import subprocess
18 import stat
19 import sys
20 import tarfile
21 import tempfile
22 import time
23 import urllib2
24 import zipfile
25
26
27 # Do NOT CHANGE this if you don't know what you're doing -- see
28 # https://chromium.googlesource.com/chromium/src/+/master/docs/updating_clang.md
29 # Reverting problematic clang rolls is safe, though.
30 CLANG_REVISION = '282487'
31
32 use_head_revision = 'LLVM_FORCE_HEAD_REVISION' in os.environ
33 if use_head_revision:
34 CLANG_REVISION = 'HEAD'
35
36 # This is incremented when pushing a new build of Clang at the same revision.
37 CLANG_SUB_REVISION=1
38
39 PACKAGE_VERSION = "%s-%s" % (CLANG_REVISION, CLANG_SUB_REVISION)
40
41 # Path constants. (All of these should be absolute paths.)
42 THIS_DIR = os.path.abspath(os.path.dirname(__file__))
43 CHROMIUM_DIR = os.path.abspath(os.path.join(THIS_DIR, '..', '..', '..'))
44 THIRD_PARTY_DIR = os.path.join(CHROMIUM_DIR, 'third_party')
45 LLVM_DIR = os.path.join(THIRD_PARTY_DIR, 'llvm')
46 LLVM_BOOTSTRAP_DIR = os.path.join(THIRD_PARTY_DIR, 'llvm-bootstrap')
47 LLVM_BOOTSTRAP_INSTALL_DIR = os.path.join(THIRD_PARTY_DIR,
48 'llvm-bootstrap-install')
49 LLVM_LTO_GOLD_PLUGIN_DIR = os.path.join(THIRD_PARTY_DIR, 'llvm-lto-gold-plugin')
50 CHROME_TOOLS_SHIM_DIR = os.path.join(LLVM_DIR, 'tools', 'chrometools')
51 LLVM_BUILD_DIR = os.path.join(CHROMIUM_DIR, 'third_party', 'llvm-build',
52 'Release+Asserts')
53 COMPILER_RT_BUILD_DIR = os.path.join(LLVM_BUILD_DIR, 'compiler-rt')
54 CLANG_DIR = os.path.join(LLVM_DIR, 'tools', 'clang')
55 LLD_DIR = os.path.join(LLVM_DIR, 'tools', 'lld')
56 # compiler-rt is built as part of the regular LLVM build on Windows to get
57 # the 64-bit runtime, and out-of-tree elsewhere.
58 # TODO(thakis): Try to unify this.
59 if sys.platform == 'win32':
60 COMPILER_RT_DIR = os.path.join(LLVM_DIR, 'projects', 'compiler-rt')
61 else:
62 COMPILER_RT_DIR = os.path.join(LLVM_DIR, 'compiler-rt')
63 LIBCXX_DIR = os.path.join(LLVM_DIR, 'projects', 'libcxx')
64 LIBCXXABI_DIR = os.path.join(LLVM_DIR, 'projects', 'libcxxabi')
65 LLVM_BUILD_TOOLS_DIR = os.path.abspath(
66 os.path.join(LLVM_DIR, '..', 'llvm-build-tools'))
67 STAMP_FILE = os.path.normpath(
68 os.path.join(LLVM_DIR, '..', 'llvm-build', 'cr_build_revision'))
69 BINUTILS_DIR = os.path.join(THIRD_PARTY_DIR, 'binutils')
70 BINUTILS_BIN_DIR = os.path.join(BINUTILS_DIR, BINUTILS_DIR,
71 'Linux_x64', 'Release', 'bin')
72 BFD_PLUGINS_DIR = os.path.join(BINUTILS_DIR, 'Linux_x64', 'Release',
73 'lib', 'bfd-plugins')
74 VERSION = '4.0.0'
75 ANDROID_NDK_DIR = os.path.join(
76 CHROMIUM_DIR, 'third_party', 'android_tools', 'ndk')
77
78 # URL for pre-built binaries.
79 CDS_URL = os.environ.get('CDS_CLANG_BUCKET_OVERRIDE',
80 'https://commondatastorage.googleapis.com/chromium-browser-clang')
81
82 LLVM_REPO_URL='https://llvm.org/svn/llvm-project'
83 if 'LLVM_REPO_URL' in os.environ:
84 LLVM_REPO_URL = os.environ['LLVM_REPO_URL']
85
86 # Bump after VC updates.
87 DIA_DLL = {
88 '2013': 'msdia120.dll',
89 '2015': 'msdia140.dll',
90 }
91
92
93 def DownloadUrl(url, output_file):
94 """Download url into output_file."""
95 CHUNK_SIZE = 4096
96 TOTAL_DOTS = 10
97 num_retries = 3
98 retry_wait_s = 5 # Doubled at each retry.
99
100 while True:
101 try:
102 sys.stdout.write('Downloading %s ' % url)
103 sys.stdout.flush()
104 response = urllib2.urlopen(url)
105 total_size = int(response.info().getheader('Content-Length').strip())
106 bytes_done = 0
107 dots_printed = 0
108 while True:
109 chunk = response.read(CHUNK_SIZE)
110 if not chunk:
111 break
112 output_file.write(chunk)
113 bytes_done += len(chunk)
114 num_dots = TOTAL_DOTS * bytes_done / total_size
115 sys.stdout.write('.' * (num_dots - dots_printed))
116 sys.stdout.flush()
117 dots_printed = num_dots
118 if bytes_done != total_size:
119 raise urllib2.URLError("only got %d of %d bytes" %
120 (bytes_done, total_size))
121 print ' Done.'
122 return
123 except urllib2.URLError as e:
124 sys.stdout.write('\n')
125 print e
126 if num_retries == 0 or isinstance(e, urllib2.HTTPError) and e.code == 404:
127 raise e
128 num_retries -= 1
129 print 'Retrying in %d s ...' % retry_wait_s
130 time.sleep(retry_wait_s)
131 retry_wait_s *= 2
132
133
134 def EnsureDirExists(path):
135 if not os.path.exists(path):
136 print "Creating directory %s" % path
137 os.makedirs(path)
138
139
140 def DownloadAndUnpack(url, output_dir):
141 with tempfile.TemporaryFile() as f:
142 DownloadUrl(url, f)
143 f.seek(0)
144 EnsureDirExists(output_dir)
145 if url.endswith('.zip'):
146 zipfile.ZipFile(f).extractall(path=output_dir)
147 else:
148 tarfile.open(mode='r:gz', fileobj=f).extractall(path=output_dir)
149
150
151 def ReadStampFile(path=STAMP_FILE):
152 """Return the contents of the stamp file, or '' if it doesn't exist."""
153 try:
154 with open(path, 'r') as f:
155 return f.read().rstrip()
156 except IOError:
157 return ''
158
159
160 def WriteStampFile(s, path=STAMP_FILE):
161 """Write s to the stamp file."""
162 EnsureDirExists(os.path.dirname(path))
163 with open(path, 'w') as f:
164 f.write(s)
165 f.write('\n')
166
167
168 def GetSvnRevision(svn_repo):
169 """Returns current revision of the svn repo at svn_repo."""
170 if sys.platform == 'darwin':
171 # mac_files toolchain must be set for hermetic builds.
172 root = os.path.dirname(os.path.dirname(os.path.dirname(
173 os.path.dirname(__file__))))
174 sys.path.append(os.path.join(root, 'build'))
175 import mac_toolchain
176
177 mac_toolchain.SetToolchainEnvironment()
178 svn_info = subprocess.check_output('svn info ' + svn_repo, shell=True)
179 m = re.search(r'Revision: (\d+)', svn_info)
180 return m.group(1)
181
182
183 def RmTree(dir):
184 """Delete dir."""
185 def ChmodAndRetry(func, path, _):
186 # Subversion can leave read-only files around.
187 if not os.access(path, os.W_OK):
188 os.chmod(path, stat.S_IWUSR)
189 return func(path)
190 raise
191
192 shutil.rmtree(dir, onerror=ChmodAndRetry)
193
194
195 def RmCmakeCache(dir):
196 """Delete CMake cache related files from dir."""
197 for dirpath, dirs, files in os.walk(dir):
198 if 'CMakeCache.txt' in files:
199 os.remove(os.path.join(dirpath, 'CMakeCache.txt'))
200 if 'CMakeFiles' in dirs:
201 RmTree(os.path.join(dirpath, 'CMakeFiles'))
202
203
204 def RunCommand(command, msvc_arch=None, env=None, fail_hard=True):
205 """Run command and return success (True) or failure; or if fail_hard is
206 True, exit on failure. If msvc_arch is set, runs the command in a
207 shell with the msvc tools for that architecture."""
208
209 if msvc_arch and sys.platform == 'win32':
210 command = GetVSVersion().SetupScript(msvc_arch) + ['&&'] + command
211
212 # https://docs.python.org/2/library/subprocess.html:
213 # "On Unix with shell=True [...] if args is a sequence, the first item
214 # specifies the command string, and any additional items will be treated as
215 # additional arguments to the shell itself. That is to say, Popen does the
216 # equivalent of:
217 # Popen(['/bin/sh', '-c', args[0], args[1], ...])"
218 #
219 # We want to pass additional arguments to command[0], not to the shell,
220 # so manually join everything into a single string.
221 # Annoyingly, for "svn co url c:\path", pipes.quote() thinks that it should
222 # quote c:\path but svn can't handle quoted paths on Windows. Since on
223 # Windows follow-on args are passed to args[0] instead of the shell, don't
224 # do the single-string transformation there.
225 if sys.platform != 'win32':
226 command = ' '.join([pipes.quote(c) for c in command])
227 print 'Running', command
228 if subprocess.call(command, env=env, shell=True) == 0:
229 return True
230 print 'Failed.'
231 if fail_hard:
232 sys.exit(1)
233 return False
234
235
236 def CopyFile(src, dst):
237 """Copy a file from src to dst."""
238 print "Copying %s to %s" % (src, dst)
239 shutil.copy(src, dst)
240
241
242 def CopyDirectoryContents(src, dst, filename_filter=None):
243 """Copy the files from directory src to dst
244 with an optional filename filter."""
245 dst = os.path.realpath(dst) # realpath() in case dst ends in /..
246 EnsureDirExists(dst)
247 for root, _, files in os.walk(src):
248 for f in files:
249 if filename_filter and not re.match(filename_filter, f):
250 continue
251 CopyFile(os.path.join(root, f), dst)
252
253
254 def Checkout(name, url, dir):
255 """Checkout the SVN module at url into dir. Use name for the log message."""
256 print "Checking out %s r%s into '%s'" % (name, CLANG_REVISION, dir)
257
258 command = ['svn', 'checkout', '--force', url + '@' + CLANG_REVISION, dir]
259 if RunCommand(command, fail_hard=False):
260 return
261
262 if os.path.isdir(dir):
263 print "Removing %s." % (dir)
264 RmTree(dir)
265
266 print "Retrying."
267 RunCommand(command)
268
269
270 def DeleteChromeToolsShim():
271 OLD_SHIM_DIR = os.path.join(LLVM_DIR, 'tools', 'zzz-chrometools')
272 shutil.rmtree(OLD_SHIM_DIR, ignore_errors=True)
273 shutil.rmtree(CHROME_TOOLS_SHIM_DIR, ignore_errors=True)
274
275
276 def CreateChromeToolsShim():
277 """Hooks the Chrome tools into the LLVM build.
278
279 Several Chrome tools have dependencies on LLVM/Clang libraries. The LLVM build
280 detects implicit tools in the tools subdirectory, so this helper install a
281 shim CMakeLists.txt that forwards to the real directory for the Chrome tools.
282
283 Note that the shim directory name intentionally has no - or _. The implicit
284 tool detection logic munges them in a weird way."""
285 assert not any(i in os.path.basename(CHROME_TOOLS_SHIM_DIR) for i in '-_')
286 os.mkdir(CHROME_TOOLS_SHIM_DIR)
287 with file(os.path.join(CHROME_TOOLS_SHIM_DIR, 'CMakeLists.txt'), 'w') as f:
288 f.write('# Automatically generated by tools/clang/scripts/update.py. ' +
289 'Do not edit.\n')
290 f.write('# Since tools/clang is located in another directory, use the \n')
291 f.write('# two arg version to specify where build artifacts go. CMake\n')
292 f.write('# disallows reuse of the same binary dir for multiple source\n')
293 f.write('# dirs, so the build artifacts need to go into a subdirectory.\n')
294 f.write('# dirs, so the build artifacts need to go into a subdirectory.\n')
295 f.write('if (CHROMIUM_TOOLS_SRC)\n')
296 f.write(' add_subdirectory(${CHROMIUM_TOOLS_SRC} ' +
297 '${CMAKE_CURRENT_BINARY_DIR}/a)\n')
298 f.write('endif (CHROMIUM_TOOLS_SRC)\n')
299
300
301 def DownloadHostGcc(args):
302 """Downloads gcc 4.8.5 and makes sure args.gcc_toolchain is set."""
303 if not sys.platform.startswith('linux') or args.gcc_toolchain:
304 return
305 # Unconditionally download a prebuilt gcc to guarantee the included libstdc++
306 # works on Ubuntu Precise.
307 gcc_dir = os.path.join(LLVM_BUILD_TOOLS_DIR, 'gcc485precise')
308 if not os.path.exists(gcc_dir):
309 print 'Downloading pre-built GCC 4.8.5...'
310 DownloadAndUnpack(
311 CDS_URL + '/tools/gcc485precise.tgz', LLVM_BUILD_TOOLS_DIR)
312 args.gcc_toolchain = gcc_dir
313
314
315 def AddCMakeToPath():
316 """Download CMake and add it to PATH."""
317 if sys.platform == 'win32':
318 zip_name = 'cmake-3.4.3-win32-x86.zip'
319 cmake_dir = os.path.join(LLVM_BUILD_TOOLS_DIR,
320 'cmake-3.4.3-win32-x86', 'bin')
321 else:
322 suffix = 'Darwin' if sys.platform == 'darwin' else 'Linux'
323 zip_name = 'cmake343_%s.tgz' % suffix
324 cmake_dir = os.path.join(LLVM_BUILD_TOOLS_DIR, 'cmake343', 'bin')
325 if not os.path.exists(cmake_dir):
326 DownloadAndUnpack(CDS_URL + '/tools/' + zip_name, LLVM_BUILD_TOOLS_DIR)
327 os.environ['PATH'] = cmake_dir + os.pathsep + os.environ.get('PATH', '')
328
329
330 def AddGnuWinToPath():
331 """Download some GNU win tools and add them to PATH."""
332 if sys.platform != 'win32':
333 return
334
335 gnuwin_dir = os.path.join(LLVM_BUILD_TOOLS_DIR, 'gnuwin')
336 GNUWIN_VERSION = '5'
337 GNUWIN_STAMP = os.path.join(gnuwin_dir, 'stamp')
338 if ReadStampFile(GNUWIN_STAMP) == GNUWIN_VERSION:
339 print 'GNU Win tools already up to date.'
340 else:
341 zip_name = 'gnuwin-%s.zip' % GNUWIN_VERSION
342 DownloadAndUnpack(CDS_URL + '/tools/' + zip_name, LLVM_BUILD_TOOLS_DIR)
343 WriteStampFile(GNUWIN_VERSION, GNUWIN_STAMP)
344
345 os.environ['PATH'] = gnuwin_dir + os.pathsep + os.environ.get('PATH', '')
346
347
348 vs_version = None
349 def GetVSVersion():
350 global vs_version
351 if vs_version:
352 return vs_version
353
354 # Try using the toolchain in depot_tools.
355 # This sets environment variables used by SelectVisualStudioVersion below.
356 sys.path.append(os.path.join(CHROMIUM_DIR, 'build'))
357 import vs_toolchain
358 vs_toolchain.SetEnvironmentAndGetRuntimeDllDirs()
359
360 # Use gyp to find the MSVS installation, either in depot_tools as per above,
361 # or a system-wide installation otherwise.
362 sys.path.append(os.path.join(CHROMIUM_DIR, 'tools', 'gyp', 'pylib'))
363 import gyp.MSVSVersion
364 vs_version = gyp.MSVSVersion.SelectVisualStudioVersion(
365 vs_toolchain.GetVisualStudioVersion())
366 return vs_version
367
368
369 def CopyDiaDllTo(target_dir):
370 # This script always wants to use the 64-bit msdia*.dll.
371 dia_path = os.path.join(GetVSVersion().Path(), 'DIA SDK', 'bin', 'amd64')
372 dia_dll = os.path.join(dia_path, DIA_DLL[GetVSVersion().ShortName()])
373 CopyFile(dia_dll, target_dir)
374
375
376 def VeryifyVersionOfBuiltClangMatchesVERSION():
377 """Checks that `clang --version` outputs VERSION. If this fails, VERSION
378 in this file is out-of-date and needs to be updated (possibly in an
379 `if use_head_revision:` block in main() first)."""
380 clang = os.path.join(LLVM_BUILD_DIR, 'bin', 'clang')
381 if sys.platform == 'win32':
382 # TODO: Parse `clang-cl /?` output for built clang's version and check that
383 # to check the binary we're actually shipping? But clang-cl.exe is just
384 # a copy of clang.exe, so this does check the same thing.
385 clang += '.exe'
386 version_out = subprocess.check_output([clang, '--version'])
387 version_out = re.match(r'clang version ([0-9.]+)', version_out).group(1)
388 if version_out != VERSION:
389 print ('unexpected clang version %s (not %s), update VERSION in update.py'
390 % (version_out, VERSION))
391 sys.exit(1)
392
393
394 def UpdateClang(args):
395 print 'Updating Clang to %s...' % PACKAGE_VERSION
396
397 need_gold_plugin = 'LLVM_DOWNLOAD_GOLD_PLUGIN' in os.environ or (
398 sys.platform.startswith('linux') and
399 'buildtype=Official' in os.environ.get('GYP_DEFINES', '') and
400 'branding=Chrome' in os.environ.get('GYP_DEFINES', ''))
401
402 if ReadStampFile() == PACKAGE_VERSION and not args.force_local_build:
403 print 'Clang is already up to date.'
404 if not need_gold_plugin or os.path.exists(
405 os.path.join(LLVM_BUILD_DIR, "lib/LLVMgold.so")):
406 return 0
407
408 # Reset the stamp file in case the build is unsuccessful.
409 WriteStampFile('')
410
411 if not args.force_local_build:
412 cds_file = "clang-%s.tgz" % PACKAGE_VERSION
413 if sys.platform == 'win32' or sys.platform == 'cygwin':
414 cds_full_url = CDS_URL + '/Win/' + cds_file
415 elif sys.platform == 'darwin':
416 cds_full_url = CDS_URL + '/Mac/' + cds_file
417 else:
418 assert sys.platform.startswith('linux')
419 cds_full_url = CDS_URL + '/Linux_x64/' + cds_file
420
421 print 'Downloading prebuilt clang'
422 if os.path.exists(LLVM_BUILD_DIR):
423 RmTree(LLVM_BUILD_DIR)
424 try:
425 DownloadAndUnpack(cds_full_url, LLVM_BUILD_DIR)
426 print 'clang %s unpacked' % PACKAGE_VERSION
427 if sys.platform == 'win32':
428 CopyDiaDllTo(os.path.join(LLVM_BUILD_DIR, 'bin'))
429 # Download the gold plugin if requested to by an environment variable.
430 # This is used by the CFI ClusterFuzz bot, and it's required for official
431 # builds on linux.
432 if need_gold_plugin:
433 RunCommand(['python', CHROMIUM_DIR+'/build/download_gold_plugin.py'])
434 WriteStampFile(PACKAGE_VERSION)
435 return 0
436 except urllib2.URLError:
437 print 'Failed to download prebuilt clang %s' % cds_file
438 print 'Use --force-local-build if you want to build locally.'
439 print 'Exiting.'
440 return 1
441
442 if args.with_android and not os.path.exists(ANDROID_NDK_DIR):
443 print 'Android NDK not found at ' + ANDROID_NDK_DIR
444 print 'The Android NDK is needed to build a Clang whose -fsanitize=address'
445 print 'works on Android. See '
446 print 'https://www.chromium.org/developers/how-tos/android-build-instruction s'
447 print 'for how to install the NDK, or pass --without-android.'
448 return 1
449
450 DownloadHostGcc(args)
451 AddCMakeToPath()
452 AddGnuWinToPath()
453
454 DeleteChromeToolsShim()
455
456 Checkout('LLVM', LLVM_REPO_URL + '/llvm/trunk', LLVM_DIR)
457 Checkout('Clang', LLVM_REPO_URL + '/cfe/trunk', CLANG_DIR)
458 if sys.platform == 'win32' or use_head_revision:
459 Checkout('LLD', LLVM_REPO_URL + '/lld/trunk', LLD_DIR)
460 Checkout('compiler-rt', LLVM_REPO_URL + '/compiler-rt/trunk', COMPILER_RT_DIR)
461 if sys.platform == 'darwin':
462 # clang needs a libc++ checkout, else -stdlib=libc++ won't find includes
463 # (i.e. this is needed for bootstrap builds).
464 Checkout('libcxx', LLVM_REPO_URL + '/libcxx/trunk', LIBCXX_DIR)
465 # We used to check out libcxxabi on OS X; we no longer need that.
466 if os.path.exists(LIBCXXABI_DIR):
467 RmTree(LIBCXXABI_DIR)
468
469 cc, cxx = None, None
470 libstdcpp = None
471 if args.gcc_toolchain: # This option is only used on Linux.
472 # Use the specified gcc installation for building.
473 cc = os.path.join(args.gcc_toolchain, 'bin', 'gcc')
474 cxx = os.path.join(args.gcc_toolchain, 'bin', 'g++')
475
476 if not os.access(cc, os.X_OK):
477 print 'Invalid --gcc-toolchain: "%s"' % args.gcc_toolchain
478 print '"%s" does not appear to be valid.' % cc
479 return 1
480
481 # Set LD_LIBRARY_PATH to make auxiliary targets (tablegen, bootstrap
482 # compiler, etc.) find the .so.
483 libstdcpp = subprocess.check_output(
484 [cxx, '-print-file-name=libstdc++.so.6']).rstrip()
485 os.environ['LD_LIBRARY_PATH'] = os.path.dirname(libstdcpp)
486
487 cflags = []
488 cxxflags = []
489 ldflags = []
490
491 base_cmake_args = ['-GNinja',
492 '-DCMAKE_BUILD_TYPE=Release',
493 '-DLLVM_ENABLE_ASSERTIONS=ON',
494 '-DLLVM_ENABLE_THREADS=OFF',
495 '-DLLVM_ENABLE_TIMESTAMPS=OFF',
496 # Statically link MSVCRT to avoid DLL dependencies.
497 '-DLLVM_USE_CRT_RELEASE=MT',
498 ]
499
500 binutils_incdir = ''
501 if sys.platform.startswith('linux'):
502 binutils_incdir = os.path.join(BINUTILS_DIR, 'Linux_x64/Release/include')
503
504 if args.bootstrap:
505 print 'Building bootstrap compiler'
506 EnsureDirExists(LLVM_BOOTSTRAP_DIR)
507 os.chdir(LLVM_BOOTSTRAP_DIR)
508 bootstrap_args = base_cmake_args + [
509 '-DLLVM_BINUTILS_INCDIR=' + binutils_incdir,
510 '-DLLVM_TARGETS_TO_BUILD=host',
511 '-DCMAKE_INSTALL_PREFIX=' + LLVM_BOOTSTRAP_INSTALL_DIR,
512 '-DCMAKE_C_FLAGS=' + ' '.join(cflags),
513 '-DCMAKE_CXX_FLAGS=' + ' '.join(cxxflags),
514 ]
515 if cc is not None: bootstrap_args.append('-DCMAKE_C_COMPILER=' + cc)
516 if cxx is not None: bootstrap_args.append('-DCMAKE_CXX_COMPILER=' + cxx)
517 RmCmakeCache('.')
518 RunCommand(['cmake'] + bootstrap_args + [LLVM_DIR], msvc_arch='x64')
519 RunCommand(['ninja'], msvc_arch='x64')
520 if args.run_tests:
521 if sys.platform == 'win32':
522 CopyDiaDllTo(os.path.join(LLVM_BOOTSTRAP_DIR, 'bin'))
523 RunCommand(['ninja', 'check-all'], msvc_arch='x64')
524 RunCommand(['ninja', 'install'], msvc_arch='x64')
525 if args.gcc_toolchain:
526 # Copy that gcc's stdlibc++.so.6 to the build dir, so the bootstrap
527 # compiler can start.
528 CopyFile(libstdcpp, os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'lib'))
529
530 if sys.platform == 'win32':
531 cc = os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'bin', 'clang-cl.exe')
532 cxx = os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'bin', 'clang-cl.exe')
533 # CMake has a hard time with backslashes in compiler paths:
534 # https://stackoverflow.com/questions/13050827
535 cc = cc.replace('\\', '/')
536 cxx = cxx.replace('\\', '/')
537 else:
538 cc = os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'bin', 'clang')
539 cxx = os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'bin', 'clang++')
540
541 if args.gcc_toolchain:
542 # Tell the bootstrap compiler to use a specific gcc prefix to search
543 # for standard library headers and shared object files.
544 cflags = ['--gcc-toolchain=' + args.gcc_toolchain]
545 cxxflags = ['--gcc-toolchain=' + args.gcc_toolchain]
546 print 'Building final compiler'
547
548 # Build LLVM gold plugin with LTO. That speeds up the linker by ~10%.
549 # We only use LTO for Linux now.
550 if args.bootstrap and args.lto_gold_plugin:
551 print 'Building LTO LLVM Gold plugin'
552 if os.path.exists(LLVM_LTO_GOLD_PLUGIN_DIR):
553 RmTree(LLVM_LTO_GOLD_PLUGIN_DIR)
554 EnsureDirExists(LLVM_LTO_GOLD_PLUGIN_DIR)
555 os.chdir(LLVM_LTO_GOLD_PLUGIN_DIR)
556
557 # Create a symlink to LLVMgold.so build in the previous step so that ar
558 # and ranlib could find it while linking LLVMgold.so with LTO.
559 EnsureDirExists(BFD_PLUGINS_DIR)
560 RunCommand(['ln', '-sf',
561 os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'lib', 'LLVMgold.so'),
562 os.path.join(BFD_PLUGINS_DIR, 'LLVMgold.so')])
563
564 lto_cflags = ['-flto']
565 lto_ldflags = ['-fuse-ld=gold']
566 if args.gcc_toolchain:
567 # Tell the bootstrap compiler to use a specific gcc prefix to search
568 # for standard library headers and shared object files.
569 lto_cflags += ['--gcc-toolchain=' + args.gcc_toolchain]
570 lto_cmake_args = base_cmake_args + [
571 '-DLLVM_BINUTILS_INCDIR=' + binutils_incdir,
572 '-DCMAKE_C_COMPILER=' + cc,
573 '-DCMAKE_CXX_COMPILER=' + cxx,
574 '-DCMAKE_C_FLAGS=' + ' '.join(lto_cflags),
575 '-DCMAKE_CXX_FLAGS=' + ' '.join(lto_cflags),
576 '-DCMAKE_EXE_LINKER_FLAGS=' + ' '.join(lto_ldflags),
577 '-DCMAKE_SHARED_LINKER_FLAGS=' + ' '.join(lto_ldflags),
578 '-DCMAKE_MODULE_LINKER_FLAGS=' + ' '.join(lto_ldflags)]
579
580 # We need to use the proper binutils which support LLVM Gold plugin.
581 lto_env = os.environ.copy()
582 lto_env['PATH'] = BINUTILS_BIN_DIR + os.pathsep + lto_env.get('PATH', '')
583
584 RmCmakeCache('.')
585 RunCommand(['cmake'] + lto_cmake_args + [LLVM_DIR], env=lto_env)
586 RunCommand(['ninja', 'LLVMgold'], env=lto_env)
587
588
589 # LLVM uses C++11 starting in llvm 3.5. On Linux, this means libstdc++4.7+ is
590 # needed, on OS X it requires libc++. clang only automatically links to libc++
591 # when targeting OS X 10.9+, so add stdlib=libc++ explicitly so clang can run
592 # on OS X versions as old as 10.7.
593 deployment_target = ''
594
595 if sys.platform == 'darwin' and args.bootstrap:
596 # When building on 10.9, /usr/include usually doesn't exist, and while
597 # Xcode's clang automatically sets a sysroot, self-built clangs don't.
598 cflags = ['-isysroot', subprocess.check_output(
599 ['xcrun', '--show-sdk-path']).rstrip()]
600 cxxflags = ['-stdlib=libc++'] + cflags
601 ldflags += ['-stdlib=libc++']
602 deployment_target = '10.7'
603 # Running libc++ tests takes a long time. Since it was only needed for
604 # the install step above, don't build it as part of the main build.
605 # This makes running package.py over 10% faster (30 min instead of 34 min)
606 RmTree(LIBCXX_DIR)
607
608 # Build clang.
609
610 # If building at head, define a macro that plugins can use for #ifdefing
611 # out code that builds at head, but not at CLANG_REVISION or vice versa.
612 if use_head_revision:
613 cflags += ['-DLLVM_FORCE_HEAD_REVISION']
614 cxxflags += ['-DLLVM_FORCE_HEAD_REVISION']
615
616 CreateChromeToolsShim()
617
618 deployment_env = None
619 if deployment_target:
620 deployment_env = os.environ.copy()
621 deployment_env['MACOSX_DEPLOYMENT_TARGET'] = deployment_target
622
623 cmake_args = []
624 # TODO(thakis): Unconditionally append this to base_cmake_args instead once
625 # compiler-rt can build with clang-cl on Windows (http://llvm.org/PR23698)
626 cc_args = base_cmake_args if sys.platform != 'win32' else cmake_args
627 if cc is not None: cc_args.append('-DCMAKE_C_COMPILER=' + cc)
628 if cxx is not None: cc_args.append('-DCMAKE_CXX_COMPILER=' + cxx)
629 cmake_args += base_cmake_args + [
630 '-DLLVM_BINUTILS_INCDIR=' + binutils_incdir,
631 '-DCMAKE_C_FLAGS=' + ' '.join(cflags),
632 '-DCMAKE_CXX_FLAGS=' + ' '.join(cxxflags),
633 '-DCMAKE_EXE_LINKER_FLAGS=' + ' '.join(ldflags),
634 '-DCMAKE_SHARED_LINKER_FLAGS=' + ' '.join(ldflags),
635 '-DCMAKE_MODULE_LINKER_FLAGS=' + ' '.join(ldflags),
636 '-DCMAKE_INSTALL_PREFIX=' + LLVM_BUILD_DIR,
637 # TODO(thakis): Remove this once official builds pass -Wl,--build-id
638 # explicitly, https://crbug.com/622775
639 '-DENABLE_LINKER_BUILD_ID=ON',
640 '-DCHROMIUM_TOOLS_SRC=%s' % os.path.join(CHROMIUM_DIR, 'tools', 'clang'),
641 '-DCHROMIUM_TOOLS=%s' % ';'.join(args.tools)]
642
643 EnsureDirExists(LLVM_BUILD_DIR)
644 os.chdir(LLVM_BUILD_DIR)
645 RmCmakeCache('.')
646 RunCommand(['cmake'] + cmake_args + [LLVM_DIR],
647 msvc_arch='x64', env=deployment_env)
648
649 if args.gcc_toolchain:
650 # Copy in the right stdlibc++.so.6 so clang can start.
651 if not os.path.exists(os.path.join(LLVM_BUILD_DIR, 'lib')):
652 os.mkdir(os.path.join(LLVM_BUILD_DIR, 'lib'))
653 libstdcpp = subprocess.check_output(
654 [cxx] + cxxflags + ['-print-file-name=libstdc++.so.6']).rstrip()
655 CopyFile(libstdcpp, os.path.join(LLVM_BUILD_DIR, 'lib'))
656
657 RunCommand(['ninja'], msvc_arch='x64')
658
659 if args.tools:
660 # If any Chromium tools were built, install those now.
661 RunCommand(['ninja', 'cr-install'], msvc_arch='x64')
662
663 if sys.platform == 'darwin':
664 # See http://crbug.com/256342
665 RunCommand(['strip', '-x', os.path.join(LLVM_BUILD_DIR, 'bin', 'clang')])
666 elif sys.platform.startswith('linux'):
667 RunCommand(['strip', os.path.join(LLVM_BUILD_DIR, 'bin', 'clang')])
668
669 VeryifyVersionOfBuiltClangMatchesVERSION()
670
671 # Do an out-of-tree build of compiler-rt.
672 # On Windows, this is used to get the 32-bit ASan run-time.
673 # TODO(hans): Remove once the regular build above produces this.
674 # On Mac and Linux, this is used to get the regular 64-bit run-time.
675 # Do a clobbered build due to cmake changes.
676 if os.path.isdir(COMPILER_RT_BUILD_DIR):
677 RmTree(COMPILER_RT_BUILD_DIR)
678 os.makedirs(COMPILER_RT_BUILD_DIR)
679 os.chdir(COMPILER_RT_BUILD_DIR)
680 # TODO(thakis): Add this once compiler-rt can build with clang-cl (see
681 # above).
682 #if args.bootstrap and sys.platform == 'win32':
683 # The bootstrap compiler produces 64-bit binaries by default.
684 #cflags += ['-m32']
685 #cxxflags += ['-m32']
686 compiler_rt_args = base_cmake_args + [
687 '-DCMAKE_C_FLAGS=' + ' '.join(cflags),
688 '-DCMAKE_CXX_FLAGS=' + ' '.join(cxxflags)]
689 if sys.platform == 'darwin':
690 compiler_rt_args += ['-DCOMPILER_RT_ENABLE_IOS=ON']
691 if sys.platform != 'win32':
692 compiler_rt_args += ['-DLLVM_CONFIG_PATH=' +
693 os.path.join(LLVM_BUILD_DIR, 'bin', 'llvm-config'),
694 '-DSANITIZER_MIN_OSX_VERSION="10.7"']
695 # compiler-rt is part of the llvm checkout on Windows but a stand-alone
696 # directory elsewhere, see the TODO above COMPILER_RT_DIR.
697 RmCmakeCache('.')
698 RunCommand(['cmake'] + compiler_rt_args +
699 [LLVM_DIR if sys.platform == 'win32' else COMPILER_RT_DIR],
700 msvc_arch='x86', env=deployment_env)
701 RunCommand(['ninja', 'compiler-rt'], msvc_arch='x86')
702
703 # Copy select output to the main tree.
704 # TODO(hans): Make this (and the .gypi and .isolate files) version number
705 # independent.
706 if sys.platform == 'win32':
707 platform = 'windows'
708 elif sys.platform == 'darwin':
709 platform = 'darwin'
710 else:
711 assert sys.platform.startswith('linux')
712 platform = 'linux'
713 asan_rt_lib_src_dir = os.path.join(COMPILER_RT_BUILD_DIR, 'lib', platform)
714 if sys.platform == 'win32':
715 # TODO(thakis): This too is due to compiler-rt being part of the checkout
716 # on Windows, see TODO above COMPILER_RT_DIR.
717 asan_rt_lib_src_dir = os.path.join(COMPILER_RT_BUILD_DIR, 'lib', 'clang',
718 VERSION, 'lib', platform)
719 asan_rt_lib_dst_dir = os.path.join(LLVM_BUILD_DIR, 'lib', 'clang',
720 VERSION, 'lib', platform)
721 # Blacklists:
722 CopyDirectoryContents(os.path.join(asan_rt_lib_src_dir, '..', '..'),
723 os.path.join(asan_rt_lib_dst_dir, '..', '..'),
724 r'^.*blacklist\.txt$')
725 # Headers:
726 if sys.platform != 'win32':
727 CopyDirectoryContents(
728 os.path.join(COMPILER_RT_BUILD_DIR, 'include/sanitizer'),
729 os.path.join(LLVM_BUILD_DIR, 'lib/clang', VERSION, 'include/sanitizer'))
730 # Static and dynamic libraries:
731 CopyDirectoryContents(asan_rt_lib_src_dir, asan_rt_lib_dst_dir)
732 if sys.platform == 'darwin':
733 for dylib in glob.glob(os.path.join(asan_rt_lib_dst_dir, '*.dylib')):
734 # Fix LC_ID_DYLIB for the ASan dynamic libraries to be relative to
735 # @executable_path.
736 # TODO(glider): this is transitional. We'll need to fix the dylib
737 # name either in our build system, or in Clang. See also
738 # http://crbug.com/344836.
739 subprocess.call(['install_name_tool', '-id',
740 '@executable_path/' + os.path.basename(dylib), dylib])
741
742
743 if sys.platform == 'win32':
744 # Make an extra copy of the sanitizer headers, to be put on the include path
745 # of the fallback compiler.
746 sanitizer_include_dir = os.path.join(LLVM_BUILD_DIR, 'lib', 'clang',
747 VERSION, 'include', 'sanitizer')
748 aux_sanitizer_include_dir = os.path.join(LLVM_BUILD_DIR, 'lib', 'clang',
749 VERSION, 'include_sanitizer',
750 'sanitizer')
751 EnsureDirExists(aux_sanitizer_include_dir)
752 for _, _, files in os.walk(sanitizer_include_dir):
753 for f in files:
754 CopyFile(os.path.join(sanitizer_include_dir, f),
755 aux_sanitizer_include_dir)
756
757 if args.with_android:
758 make_toolchain = os.path.join(
759 ANDROID_NDK_DIR, 'build', 'tools', 'make_standalone_toolchain.py')
760 for target_arch in ['aarch64', 'arm', 'i686']:
761 # Make standalone Android toolchain for target_arch.
762 toolchain_dir = os.path.join(
763 LLVM_BUILD_DIR, 'android-toolchain-' + target_arch)
764 RunCommand([
765 make_toolchain,
766 '--api=' + ('21' if target_arch == 'aarch64' else '19'),
767 '--force',
768 '--install-dir=%s' % toolchain_dir,
769 '--stl=stlport',
770 '--arch=' + {
771 'aarch64': 'arm64',
772 'arm': 'arm',
773 'i686': 'x86',
774 }[target_arch]])
775 # Android NDK r9d copies a broken unwind.h into the toolchain, see
776 # http://crbug.com/357890
777 for f in glob.glob(os.path.join(toolchain_dir, 'include/c++/*/unwind.h')):
778 os.remove(f)
779
780 # Build ASan runtime for Android in a separate build tree.
781 build_dir = os.path.join(LLVM_BUILD_DIR, 'android-' + target_arch)
782 if not os.path.exists(build_dir):
783 os.mkdir(os.path.join(build_dir))
784 os.chdir(build_dir)
785 cflags = ['--target=%s-linux-androideabi' % target_arch,
786 '--sysroot=%s/sysroot' % toolchain_dir,
787 '-B%s' % toolchain_dir]
788 android_args = base_cmake_args + [
789 '-DCMAKE_C_COMPILER=' + os.path.join(LLVM_BUILD_DIR, 'bin/clang'),
790 '-DCMAKE_CXX_COMPILER=' + os.path.join(LLVM_BUILD_DIR, 'bin/clang++'),
791 '-DLLVM_CONFIG_PATH=' + os.path.join(LLVM_BUILD_DIR, 'bin/llvm-config'),
792 '-DCMAKE_C_FLAGS=' + ' '.join(cflags),
793 '-DCMAKE_CXX_FLAGS=' + ' '.join(cflags),
794 '-DANDROID=1']
795 RmCmakeCache('.')
796 RunCommand(['cmake'] + android_args + [COMPILER_RT_DIR])
797 RunCommand(['ninja', 'libclang_rt.asan-%s-android.so' % target_arch])
798
799 # And copy it into the main build tree.
800 runtime = 'libclang_rt.asan-%s-android.so' % target_arch
801 for root, _, files in os.walk(build_dir):
802 if runtime in files:
803 shutil.copy(os.path.join(root, runtime), asan_rt_lib_dst_dir)
804
805 # Run tests.
806 if args.run_tests or use_head_revision:
807 os.chdir(LLVM_BUILD_DIR)
808 RunCommand(['ninja', 'cr-check-all'], msvc_arch='x64')
809 if args.run_tests:
810 if sys.platform == 'win32':
811 CopyDiaDllTo(os.path.join(LLVM_BUILD_DIR, 'bin'))
812 os.chdir(LLVM_BUILD_DIR)
813 RunCommand(['ninja', 'check-all'], msvc_arch='x64')
814
815 WriteStampFile(PACKAGE_VERSION)
816 print 'Clang update was successful.'
817 return 0
818
819
820 def main():
821 parser = argparse.ArgumentParser(description='Build Clang.')
822 parser.add_argument('--bootstrap', action='store_true',
823 help='first build clang with CC, then with itself.')
824 parser.add_argument('--if-needed', action='store_true',
825 help="run only if the script thinks clang is needed")
826 parser.add_argument('--force-local-build', action='store_true',
827 help="don't try to download prebuild binaries")
828 parser.add_argument('--gcc-toolchain', help='set the version for which gcc '
829 'version be used for building; --gcc-toolchain=/opt/foo '
830 'picks /opt/foo/bin/gcc')
831 parser.add_argument('--lto-gold-plugin', action='store_true',
832 help='build LLVM Gold plugin with LTO')
833 parser.add_argument('--llvm-force-head-revision', action='store_true',
834 help=('use the revision in the repo when printing '
835 'the revision'))
836 parser.add_argument('--print-revision', action='store_true',
837 help='print current clang revision and exit.')
838 parser.add_argument('--print-clang-version', action='store_true',
839 help='print current clang version (e.g. x.y.z) and exit.')
840 parser.add_argument('--run-tests', action='store_true',
841 help='run tests after building; only for local builds')
842 parser.add_argument('--tools', nargs='*',
843 help='select which chrome tools to build',
844 default=['plugins', 'blink_gc_plugin'])
845 parser.add_argument('--without-android', action='store_false',
846 help='don\'t build Android ASan runtime (linux only)',
847 dest='with_android',
848 default=sys.platform.startswith('linux'))
849 args = parser.parse_args()
850
851 if args.lto_gold_plugin and not args.bootstrap:
852 print '--lto-gold-plugin requires --bootstrap'
853 return 1
854 if args.lto_gold_plugin and not sys.platform.startswith('linux'):
855 print '--lto-gold-plugin is only effective on Linux. Ignoring the option.'
856 args.lto_gold_plugin = False
857
858 if args.if_needed:
859 is_clang_required = False
860 # clang is always used on Mac and Linux.
861 if sys.platform == 'darwin' or sys.platform.startswith('linux'):
862 is_clang_required = True
863 # clang requested via $GYP_DEFINES.
864 if re.search(r'\b(clang|asan|lsan|msan|tsan)=1',
865 os.environ.get('GYP_DEFINES', '')):
866 is_clang_required = True
867 # clang previously downloaded, keep it up to date.
868 # If you don't want this, delete third_party/llvm-build on your machine.
869 if os.path.isdir(LLVM_BUILD_DIR):
870 is_clang_required = True
871 if not is_clang_required:
872 return 0
873 if re.search(r'\b(make_clang_dir)=', os.environ.get('GYP_DEFINES', '')):
874 print 'Skipping Clang update (make_clang_dir= was set in GYP_DEFINES).'
875 return 0
876
877 global CLANG_REVISION, PACKAGE_VERSION
878 if args.print_revision:
879 if use_head_revision or args.llvm_force_head_revision:
880 print GetSvnRevision(LLVM_DIR)
881 else:
882 print PACKAGE_VERSION
883 return 0
884
885 if args.print_clang_version:
886 sys.stdout.write(VERSION)
887 return 0
888
889 # Don't buffer stdout, so that print statements are immediately flushed.
890 # Do this only after --print-revision has been handled, else we'll get
891 # an error message when this script is run from gn for some reason.
892 sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
893
894 if use_head_revision:
895 # Use a real revision number rather than HEAD to make sure that the stamp
896 # file logic works.
897 CLANG_REVISION = GetSvnRevision(LLVM_REPO_URL)
898 PACKAGE_VERSION = CLANG_REVISION + '-0'
899
900 args.force_local_build = True
901 if 'OS=android' not in os.environ.get('GYP_DEFINES', ''):
902 # Only build the Android ASan rt on ToT bots when targetting Android.
903 args.with_android = False
904
905 return UpdateClang(args)
906
907
908 if __name__ == '__main__':
909 sys.exit(main())
OLDNEW

Powered by Google App Engine
This is Rietveld 408576698