| OLD | NEW |
| 1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
| 2 # Copyright (c) 2015 The Chromium Authors. All rights reserved. | 2 # Copyright (c) 2015 The Chromium Authors. All rights reserved. |
| 3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
| 4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
| 5 | 5 |
| 6 """This script will check out llvm and clang, and then package the results up | 6 """This script will check out llvm and clang, and then package the results up |
| 7 to a tgz file.""" | 7 to a tgz file.""" |
| 8 | 8 |
| 9 import argparse | 9 import argparse |
| 10 import fnmatch | 10 import fnmatch |
| 11 import itertools | 11 import itertools |
| 12 import os | 12 import os |
| 13 import shutil | 13 import shutil |
| 14 import subprocess | 14 import subprocess |
| 15 import sys | 15 import sys |
| 16 import tarfile | 16 import tarfile |
| 17 | 17 |
| 18 # Path constants. | 18 # Path constants. |
| 19 THIS_DIR = os.path.dirname(__file__) | 19 THIS_DIR = os.path.dirname(__file__) |
| 20 CHROMIUM_DIR = os.path.abspath(os.path.join(THIS_DIR, '..', '..', '..')) | 20 CHROMIUM_DIR = os.path.abspath(os.path.join(THIS_DIR, '..', '..', '..')) |
| 21 THIRD_PARTY_DIR = os.path.join(THIS_DIR, '..', '..', '..', 'third_party') | 21 THIRD_PARTY_DIR = os.path.join(THIS_DIR, '..', '..', '..', 'third_party') |
| 22 LLVM_DIR = os.path.join(THIRD_PARTY_DIR, 'llvm') | 22 LLVM_DIR = os.path.join(THIRD_PARTY_DIR, 'llvm') |
| 23 LLVM_BOOTSTRAP_DIR = os.path.join(THIRD_PARTY_DIR, 'llvm-bootstrap') | 23 LLVM_BOOTSTRAP_DIR = os.path.join(THIRD_PARTY_DIR, 'llvm-bootstrap') |
| 24 LLVM_BOOTSTRAP_INSTALL_DIR = os.path.join(THIRD_PARTY_DIR, | 24 LLVM_BOOTSTRAP_INSTALL_DIR = os.path.join(THIRD_PARTY_DIR, |
| 25 'llvm-bootstrap-install') | 25 'llvm-bootstrap-install') |
| 26 LLVM_BUILD_DIR = os.path.join(THIRD_PARTY_DIR, 'llvm-build') | 26 LLVM_BUILD_DIR = os.path.join(THIRD_PARTY_DIR, 'llvm-build') |
| 27 LLVM_RELEASE_DIR = os.path.join(LLVM_BUILD_DIR, 'Release+Asserts') | 27 LLVM_RELEASE_DIR = os.path.join(LLVM_BUILD_DIR, 'Release+Asserts') |
| 28 LLVM_LTO_GOLD_PLUGIN_DIR = os.path.join(THIRD_PARTY_DIR, 'llvm-lto-gold-plugin') | 28 LLVM_LTO_LLD_DIR = os.path.join(THIRD_PARTY_DIR, 'llvm-lto-lld') |
| 29 BINUTILS_LIB_DIR = os.path.join(THIRD_PARTY_DIR, 'binutils', 'Linux_x64', | |
| 30 'Release', 'lib') | |
| 31 STAMP_FILE = os.path.join(LLVM_BUILD_DIR, 'cr_build_revision') | 29 STAMP_FILE = os.path.join(LLVM_BUILD_DIR, 'cr_build_revision') |
| 32 | 30 |
| 33 | 31 |
| 34 def Tee(output, logfile): | 32 def Tee(output, logfile): |
| 35 logfile.write(output) | 33 logfile.write(output) |
| 36 print output, | 34 print output, |
| 37 | 35 |
| 38 | 36 |
| 39 def TeeCmd(cmd, logfile, fail_hard=True): | 37 def TeeCmd(cmd, logfile, fail_hard=True): |
| 40 """Runs cmd and writes the output to both stdout and logfile.""" | 38 """Runs cmd and writes the output to both stdout and logfile.""" |
| (...skipping 128 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 169 | 167 |
| 170 # Check that the script is not going to upload a toolchain built from HEAD. | 168 # Check that the script is not going to upload a toolchain built from HEAD. |
| 171 use_head_revision = 'LLVM_FORCE_HEAD_REVISION' in os.environ | 169 use_head_revision = 'LLVM_FORCE_HEAD_REVISION' in os.environ |
| 172 if args.upload and use_head_revision: | 170 if args.upload and use_head_revision: |
| 173 print ("--upload and LLVM_FORCE_HEAD_REVISION could not be used " | 171 print ("--upload and LLVM_FORCE_HEAD_REVISION could not be used " |
| 174 "at the same time.") | 172 "at the same time.") |
| 175 return 1 | 173 return 1 |
| 176 | 174 |
| 177 expected_stamp = GetExpectedStamp() | 175 expected_stamp = GetExpectedStamp() |
| 178 pdir = 'clang-' + expected_stamp | 176 pdir = 'clang-' + expected_stamp |
| 179 golddir = 'llvmgold-' + expected_stamp | |
| 180 print pdir | 177 print pdir |
| 181 | 178 |
| 182 if sys.platform == 'darwin': | 179 if sys.platform == 'darwin': |
| 183 platform = 'Mac' | 180 platform = 'Mac' |
| 184 elif sys.platform == 'win32': | 181 elif sys.platform == 'win32': |
| 185 platform = 'Win' | 182 platform = 'Win' |
| 186 else: | 183 else: |
| 187 platform = 'Linux_x64' | 184 platform = 'Linux_x64' |
| 188 | 185 |
| 189 # Check if Google Cloud Storage already has the artifacts we want to build. | 186 # Check if Google Cloud Storage already has the artifacts we want to build. |
| 190 if (args.upload and GsutilArchiveExists(pdir, platform) and | 187 if args.upload and GsutilArchiveExists(pdir, platform): |
| 191 (not sys.platform.startswith('linux') or | |
| 192 GsutilArchiveExists(golddir, platform))): | |
| 193 print ('Desired toolchain revision %s is already available ' | 188 print ('Desired toolchain revision %s is already available ' |
| 194 'in Google Cloud Storage:') % expected_stamp | 189 'in Google Cloud Storage:') % expected_stamp |
| 195 print 'gs://chromium-browser-clang-staging/%s/%s.tgz' % (platform, pdir) | 190 print 'gs://chromium-browser-clang-staging/%s/%s.tgz' % (platform, pdir) |
| 196 if sys.platform.startswith('linux'): | |
| 197 print 'gs://chromium-browser-clang-staging/%s/%s.tgz' % (platform, | |
| 198 golddir) | |
| 199 return 0 | 191 return 0 |
| 200 | 192 |
| 201 with open('buildlog.txt', 'w') as log: | 193 with open('buildlog.txt', 'w') as log: |
| 202 Tee('Diff in llvm:\n', log) | 194 Tee('Diff in llvm:\n', log) |
| 203 TeeCmd(['svn', 'stat', LLVM_DIR], log, fail_hard=False) | 195 TeeCmd(['svn', 'stat', LLVM_DIR], log, fail_hard=False) |
| 204 TeeCmd(['svn', 'diff', LLVM_DIR], log, fail_hard=False) | 196 TeeCmd(['svn', 'diff', LLVM_DIR], log, fail_hard=False) |
| 205 Tee('Diff in llvm/tools/clang:\n', log) | 197 Tee('Diff in llvm/tools/clang:\n', log) |
| 206 TeeCmd(['svn', 'stat', os.path.join(LLVM_DIR, 'tools', 'clang')], | 198 TeeCmd(['svn', 'stat', os.path.join(LLVM_DIR, 'tools', 'clang')], |
| 207 log, fail_hard=False) | 199 log, fail_hard=False) |
| 208 TeeCmd(['svn', 'diff', os.path.join(LLVM_DIR, 'tools', 'clang')], | 200 TeeCmd(['svn', 'diff', os.path.join(LLVM_DIR, 'tools', 'clang')], |
| (...skipping 14 matching lines...) Expand all Loading... |
| 223 | 215 |
| 224 Tee('Starting build\n', log) | 216 Tee('Starting build\n', log) |
| 225 | 217 |
| 226 # Do a clobber build. | 218 # Do a clobber build. |
| 227 shutil.rmtree(LLVM_BOOTSTRAP_DIR, ignore_errors=True) | 219 shutil.rmtree(LLVM_BOOTSTRAP_DIR, ignore_errors=True) |
| 228 shutil.rmtree(LLVM_BOOTSTRAP_INSTALL_DIR, ignore_errors=True) | 220 shutil.rmtree(LLVM_BOOTSTRAP_INSTALL_DIR, ignore_errors=True) |
| 229 shutil.rmtree(LLVM_BUILD_DIR, ignore_errors=True) | 221 shutil.rmtree(LLVM_BUILD_DIR, ignore_errors=True) |
| 230 | 222 |
| 231 opt_flags = [] | 223 opt_flags = [] |
| 232 if sys.platform.startswith('linux'): | 224 if sys.platform.startswith('linux'): |
| 233 opt_flags += ['--lto-gold-plugin'] | 225 opt_flags += ['--lto-lld'] |
| 234 build_cmd = [sys.executable, os.path.join(THIS_DIR, 'update.py'), | 226 build_cmd = [sys.executable, os.path.join(THIS_DIR, 'update.py'), |
| 235 '--bootstrap', '--force-local-build', | 227 '--bootstrap', '--force-local-build', |
| 236 '--run-tests'] + opt_flags | 228 '--run-tests'] + opt_flags |
| 237 TeeCmd(build_cmd, log) | 229 TeeCmd(build_cmd, log) |
| 238 | 230 |
| 239 stamp = open(STAMP_FILE).read().rstrip() | 231 stamp = open(STAMP_FILE).read().rstrip() |
| 240 if stamp != expected_stamp: | 232 if stamp != expected_stamp: |
| 241 print 'Actual stamp (%s) != expected stamp (%s).' % (stamp, expected_stamp) | 233 print 'Actual stamp (%s) != expected stamp (%s).' % (stamp, expected_stamp) |
| 242 return 1 | 234 return 1 |
| 243 | 235 |
| (...skipping 74 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 318 | 310 |
| 319 if sys.platform.startswith('linux'): | 311 if sys.platform.startswith('linux'): |
| 320 os.symlink('lld', os.path.join(pdir, 'bin', 'ld.lld')) | 312 os.symlink('lld', os.path.join(pdir, 'bin', 'ld.lld')) |
| 321 os.symlink('lld', os.path.join(pdir, 'bin', 'lld-link')) | 313 os.symlink('lld', os.path.join(pdir, 'bin', 'lld-link')) |
| 322 | 314 |
| 323 # Copy libc++ headers. | 315 # Copy libc++ headers. |
| 324 if sys.platform == 'darwin': | 316 if sys.platform == 'darwin': |
| 325 shutil.copytree(os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'include', 'c++'), | 317 shutil.copytree(os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'include', 'c++'), |
| 326 os.path.join(pdir, 'include', 'c++')) | 318 os.path.join(pdir, 'include', 'c++')) |
| 327 | 319 |
| 328 # Copy tcmalloc from the binutils package. | |
| 329 # FIXME: We should eventually be building our own copy. | |
| 330 if sys.platform.startswith('linux'): | |
| 331 shutil.copy(os.path.join(BINUTILS_LIB_DIR, 'libtcmalloc_minimal.so.4'), | |
| 332 os.path.join(pdir, 'lib')) | |
| 333 | |
| 334 # Copy buildlog over. | 320 # Copy buildlog over. |
| 335 shutil.copy('buildlog.txt', pdir) | 321 shutil.copy('buildlog.txt', pdir) |
| 336 | 322 |
| 337 # Create archive. | 323 # Create archive. |
| 338 tar_entries = ['bin', 'lib', 'buildlog.txt'] | 324 tar_entries = ['bin', 'lib', 'buildlog.txt'] |
| 339 if sys.platform == 'darwin': | 325 if sys.platform == 'darwin': |
| 340 tar_entries += ['include'] | 326 tar_entries += ['include'] |
| 341 with tarfile.open(pdir + '.tgz', 'w:gz') as tar: | 327 with tarfile.open(pdir + '.tgz', 'w:gz') as tar: |
| 342 for entry in tar_entries: | 328 for entry in tar_entries: |
| 343 tar.add(os.path.join(pdir, entry), arcname=entry, filter=PrintTarProgress) | 329 tar.add(os.path.join(pdir, entry), arcname=entry, filter=PrintTarProgress) |
| 344 | 330 |
| 345 MaybeUpload(args, pdir, platform) | 331 MaybeUpload(args, pdir, platform) |
| 346 | 332 |
| 347 # Zip up gold plugin on Linux. | |
| 348 if sys.platform.startswith('linux'): | |
| 349 shutil.rmtree(golddir, ignore_errors=True) | |
| 350 os.makedirs(os.path.join(golddir, 'lib')) | |
| 351 shutil.copy(os.path.join(LLVM_LTO_GOLD_PLUGIN_DIR, 'lib', 'LLVMgold.so'), | |
| 352 os.path.join(golddir, 'lib')) | |
| 353 with tarfile.open(golddir + '.tgz', 'w:gz') as tar: | |
| 354 tar.add(os.path.join(golddir, 'lib'), arcname='lib', | |
| 355 filter=PrintTarProgress) | |
| 356 MaybeUpload(args, golddir, platform) | |
| 357 | |
| 358 # Zip up llvm-objdump for sanitizer coverage. | 333 # Zip up llvm-objdump for sanitizer coverage. |
| 359 objdumpdir = 'llvmobjdump-' + stamp | 334 objdumpdir = 'llvmobjdump-' + stamp |
| 360 shutil.rmtree(objdumpdir, ignore_errors=True) | 335 shutil.rmtree(objdumpdir, ignore_errors=True) |
| 361 os.makedirs(os.path.join(objdumpdir, 'bin')) | 336 os.makedirs(os.path.join(objdumpdir, 'bin')) |
| 362 shutil.copy(os.path.join(LLVM_RELEASE_DIR, 'bin', 'llvm-objdump' + exe_ext), | 337 shutil.copy(os.path.join(LLVM_RELEASE_DIR, 'bin', 'llvm-objdump' + exe_ext), |
| 363 os.path.join(objdumpdir, 'bin')) | 338 os.path.join(objdumpdir, 'bin')) |
| 364 with tarfile.open(objdumpdir + '.tgz', 'w:gz') as tar: | 339 with tarfile.open(objdumpdir + '.tgz', 'w:gz') as tar: |
| 365 tar.add(os.path.join(objdumpdir, 'bin'), arcname='bin', | 340 tar.add(os.path.join(objdumpdir, 'bin'), arcname='bin', |
| 366 filter=PrintTarProgress) | 341 filter=PrintTarProgress) |
| 367 MaybeUpload(args, objdumpdir, platform) | 342 MaybeUpload(args, objdumpdir, platform) |
| (...skipping 11 matching lines...) Expand all Loading... |
| 379 MaybeUpload(args, translation_unit_dir, platform) | 354 MaybeUpload(args, translation_unit_dir, platform) |
| 380 | 355 |
| 381 if sys.platform == 'win32' and args.upload: | 356 if sys.platform == 'win32' and args.upload: |
| 382 UploadPDBToSymbolServer() | 357 UploadPDBToSymbolServer() |
| 383 | 358 |
| 384 # FIXME: Warn if the file already exists on the server. | 359 # FIXME: Warn if the file already exists on the server. |
| 385 | 360 |
| 386 | 361 |
| 387 if __name__ == '__main__': | 362 if __name__ == '__main__': |
| 388 sys.exit(main()) | 363 sys.exit(main()) |
| OLD | NEW |