OLD | NEW |
1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
2 # Copyright (c) 2015 The Chromium Authors. All rights reserved. | 2 # Copyright (c) 2015 The Chromium Authors. All rights reserved. |
3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
5 | 5 |
6 """This script will check out llvm and clang, and then package the results up | 6 """This script will check out llvm and clang, and then package the results up |
7 to a tgz file.""" | 7 to a tgz file.""" |
8 | 8 |
9 import argparse | 9 import argparse |
10 import fnmatch | 10 import fnmatch |
11 import itertools | 11 import itertools |
12 import os | 12 import os |
13 import shutil | 13 import shutil |
14 import subprocess | 14 import subprocess |
15 import sys | 15 import sys |
16 import tarfile | 16 import tarfile |
17 | 17 |
18 # Path constants. | 18 # Path constants. |
19 THIS_DIR = os.path.dirname(__file__) | 19 THIS_DIR = os.path.dirname(__file__) |
20 CHROMIUM_DIR = os.path.abspath(os.path.join(THIS_DIR, '..', '..', '..')) | 20 CHROMIUM_DIR = os.path.abspath(os.path.join(THIS_DIR, '..', '..', '..')) |
21 THIRD_PARTY_DIR = os.path.join(THIS_DIR, '..', '..', '..', 'third_party') | 21 THIRD_PARTY_DIR = os.path.join(THIS_DIR, '..', '..', '..', 'third_party') |
22 LLVM_DIR = os.path.join(THIRD_PARTY_DIR, 'llvm') | 22 LLVM_DIR = os.path.join(THIRD_PARTY_DIR, 'llvm') |
23 LLVM_BOOTSTRAP_DIR = os.path.join(THIRD_PARTY_DIR, 'llvm-bootstrap') | 23 LLVM_BOOTSTRAP_DIR = os.path.join(THIRD_PARTY_DIR, 'llvm-bootstrap') |
24 LLVM_BOOTSTRAP_INSTALL_DIR = os.path.join(THIRD_PARTY_DIR, | 24 LLVM_BOOTSTRAP_INSTALL_DIR = os.path.join(THIRD_PARTY_DIR, |
25 'llvm-bootstrap-install') | 25 'llvm-bootstrap-install') |
26 LLVM_BUILD_DIR = os.path.join(THIRD_PARTY_DIR, 'llvm-build') | 26 LLVM_BUILD_DIR = os.path.join(THIRD_PARTY_DIR, 'llvm-build') |
27 LLVM_RELEASE_DIR = os.path.join(LLVM_BUILD_DIR, 'Release+Asserts') | 27 LLVM_RELEASE_DIR = os.path.join(LLVM_BUILD_DIR, 'Release+Asserts') |
28 LLVM_LTO_GOLD_PLUGIN_DIR = os.path.join(THIRD_PARTY_DIR, 'llvm-lto-gold-plugin') | 28 LLVM_LTO_LLD_DIR = os.path.join(THIRD_PARTY_DIR, 'llvm-lto-lld') |
29 BINUTILS_LIB_DIR = os.path.join(THIRD_PARTY_DIR, 'binutils', 'Linux_x64', | |
30 'Release', 'lib') | |
31 STAMP_FILE = os.path.join(LLVM_BUILD_DIR, 'cr_build_revision') | 29 STAMP_FILE = os.path.join(LLVM_BUILD_DIR, 'cr_build_revision') |
32 | 30 |
33 | 31 |
34 def Tee(output, logfile): | 32 def Tee(output, logfile): |
35 logfile.write(output) | 33 logfile.write(output) |
36 print output, | 34 print output, |
37 | 35 |
38 | 36 |
39 def TeeCmd(cmd, logfile, fail_hard=True): | 37 def TeeCmd(cmd, logfile, fail_hard=True): |
40 """Runs cmd and writes the output to both stdout and logfile.""" | 38 """Runs cmd and writes the output to both stdout and logfile.""" |
(...skipping 128 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
169 | 167 |
170 # Check that the script is not going to upload a toolchain built from HEAD. | 168 # Check that the script is not going to upload a toolchain built from HEAD. |
171 use_head_revision = 'LLVM_FORCE_HEAD_REVISION' in os.environ | 169 use_head_revision = 'LLVM_FORCE_HEAD_REVISION' in os.environ |
172 if args.upload and use_head_revision: | 170 if args.upload and use_head_revision: |
173 print ("--upload and LLVM_FORCE_HEAD_REVISION could not be used " | 171 print ("--upload and LLVM_FORCE_HEAD_REVISION could not be used " |
174 "at the same time.") | 172 "at the same time.") |
175 return 1 | 173 return 1 |
176 | 174 |
177 expected_stamp = GetExpectedStamp() | 175 expected_stamp = GetExpectedStamp() |
178 pdir = 'clang-' + expected_stamp | 176 pdir = 'clang-' + expected_stamp |
179 golddir = 'llvmgold-' + expected_stamp | |
180 print pdir | 177 print pdir |
181 | 178 |
182 if sys.platform == 'darwin': | 179 if sys.platform == 'darwin': |
183 platform = 'Mac' | 180 platform = 'Mac' |
184 elif sys.platform == 'win32': | 181 elif sys.platform == 'win32': |
185 platform = 'Win' | 182 platform = 'Win' |
186 else: | 183 else: |
187 platform = 'Linux_x64' | 184 platform = 'Linux_x64' |
188 | 185 |
189 # Check if Google Cloud Storage already has the artifacts we want to build. | 186 # Check if Google Cloud Storage already has the artifacts we want to build. |
190 if (args.upload and GsutilArchiveExists(pdir, platform) and | 187 if args.upload and GsutilArchiveExists(pdir, platform): |
191 (not sys.platform.startswith('linux') or | |
192 GsutilArchiveExists(golddir, platform))): | |
193 print ('Desired toolchain revision %s is already available ' | 188 print ('Desired toolchain revision %s is already available ' |
194 'in Google Cloud Storage:') % expected_stamp | 189 'in Google Cloud Storage:') % expected_stamp |
195 print 'gs://chromium-browser-clang-staging/%s/%s.tgz' % (platform, pdir) | 190 print 'gs://chromium-browser-clang-staging/%s/%s.tgz' % (platform, pdir) |
196 if sys.platform.startswith('linux'): | |
197 print 'gs://chromium-browser-clang-staging/%s/%s.tgz' % (platform, | |
198 golddir) | |
199 return 0 | 191 return 0 |
200 | 192 |
201 with open('buildlog.txt', 'w') as log: | 193 with open('buildlog.txt', 'w') as log: |
202 Tee('Diff in llvm:\n', log) | 194 Tee('Diff in llvm:\n', log) |
203 TeeCmd(['svn', 'stat', LLVM_DIR], log, fail_hard=False) | 195 TeeCmd(['svn', 'stat', LLVM_DIR], log, fail_hard=False) |
204 TeeCmd(['svn', 'diff', LLVM_DIR], log, fail_hard=False) | 196 TeeCmd(['svn', 'diff', LLVM_DIR], log, fail_hard=False) |
205 Tee('Diff in llvm/tools/clang:\n', log) | 197 Tee('Diff in llvm/tools/clang:\n', log) |
206 TeeCmd(['svn', 'stat', os.path.join(LLVM_DIR, 'tools', 'clang')], | 198 TeeCmd(['svn', 'stat', os.path.join(LLVM_DIR, 'tools', 'clang')], |
207 log, fail_hard=False) | 199 log, fail_hard=False) |
208 TeeCmd(['svn', 'diff', os.path.join(LLVM_DIR, 'tools', 'clang')], | 200 TeeCmd(['svn', 'diff', os.path.join(LLVM_DIR, 'tools', 'clang')], |
(...skipping 14 matching lines...) Expand all Loading... |
223 | 215 |
224 Tee('Starting build\n', log) | 216 Tee('Starting build\n', log) |
225 | 217 |
226 # Do a clobber build. | 218 # Do a clobber build. |
227 shutil.rmtree(LLVM_BOOTSTRAP_DIR, ignore_errors=True) | 219 shutil.rmtree(LLVM_BOOTSTRAP_DIR, ignore_errors=True) |
228 shutil.rmtree(LLVM_BOOTSTRAP_INSTALL_DIR, ignore_errors=True) | 220 shutil.rmtree(LLVM_BOOTSTRAP_INSTALL_DIR, ignore_errors=True) |
229 shutil.rmtree(LLVM_BUILD_DIR, ignore_errors=True) | 221 shutil.rmtree(LLVM_BUILD_DIR, ignore_errors=True) |
230 | 222 |
231 opt_flags = [] | 223 opt_flags = [] |
232 if sys.platform.startswith('linux'): | 224 if sys.platform.startswith('linux'): |
233 opt_flags += ['--lto-gold-plugin'] | 225 opt_flags += ['--lto-lld'] |
234 build_cmd = [sys.executable, os.path.join(THIS_DIR, 'update.py'), | 226 build_cmd = [sys.executable, os.path.join(THIS_DIR, 'update.py'), |
235 '--bootstrap', '--force-local-build', | 227 '--bootstrap', '--force-local-build', |
236 '--run-tests'] + opt_flags | 228 '--run-tests'] + opt_flags |
237 TeeCmd(build_cmd, log) | 229 TeeCmd(build_cmd, log) |
238 | 230 |
239 stamp = open(STAMP_FILE).read().rstrip() | 231 stamp = open(STAMP_FILE).read().rstrip() |
240 if stamp != expected_stamp: | 232 if stamp != expected_stamp: |
241 print 'Actual stamp (%s) != expected stamp (%s).' % (stamp, expected_stamp) | 233 print 'Actual stamp (%s) != expected stamp (%s).' % (stamp, expected_stamp) |
242 return 1 | 234 return 1 |
243 | 235 |
(...skipping 74 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
318 | 310 |
319 if sys.platform.startswith('linux'): | 311 if sys.platform.startswith('linux'): |
320 os.symlink('lld', os.path.join(pdir, 'bin', 'ld.lld')) | 312 os.symlink('lld', os.path.join(pdir, 'bin', 'ld.lld')) |
321 os.symlink('lld', os.path.join(pdir, 'bin', 'lld-link')) | 313 os.symlink('lld', os.path.join(pdir, 'bin', 'lld-link')) |
322 | 314 |
323 # Copy libc++ headers. | 315 # Copy libc++ headers. |
324 if sys.platform == 'darwin': | 316 if sys.platform == 'darwin': |
325 shutil.copytree(os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'include', 'c++'), | 317 shutil.copytree(os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'include', 'c++'), |
326 os.path.join(pdir, 'include', 'c++')) | 318 os.path.join(pdir, 'include', 'c++')) |
327 | 319 |
328 # Copy tcmalloc from the binutils package. | |
329 # FIXME: We should eventually be building our own copy. | |
330 if sys.platform.startswith('linux'): | |
331 shutil.copy(os.path.join(BINUTILS_LIB_DIR, 'libtcmalloc_minimal.so.4'), | |
332 os.path.join(pdir, 'lib')) | |
333 | |
334 # Copy buildlog over. | 320 # Copy buildlog over. |
335 shutil.copy('buildlog.txt', pdir) | 321 shutil.copy('buildlog.txt', pdir) |
336 | 322 |
337 # Create archive. | 323 # Create archive. |
338 tar_entries = ['bin', 'lib', 'buildlog.txt'] | 324 tar_entries = ['bin', 'lib', 'buildlog.txt'] |
339 if sys.platform == 'darwin': | 325 if sys.platform == 'darwin': |
340 tar_entries += ['include'] | 326 tar_entries += ['include'] |
341 with tarfile.open(pdir + '.tgz', 'w:gz') as tar: | 327 with tarfile.open(pdir + '.tgz', 'w:gz') as tar: |
342 for entry in tar_entries: | 328 for entry in tar_entries: |
343 tar.add(os.path.join(pdir, entry), arcname=entry, filter=PrintTarProgress) | 329 tar.add(os.path.join(pdir, entry), arcname=entry, filter=PrintTarProgress) |
344 | 330 |
345 MaybeUpload(args, pdir, platform) | 331 MaybeUpload(args, pdir, platform) |
346 | 332 |
347 # Zip up gold plugin on Linux. | |
348 if sys.platform.startswith('linux'): | |
349 shutil.rmtree(golddir, ignore_errors=True) | |
350 os.makedirs(os.path.join(golddir, 'lib')) | |
351 shutil.copy(os.path.join(LLVM_LTO_GOLD_PLUGIN_DIR, 'lib', 'LLVMgold.so'), | |
352 os.path.join(golddir, 'lib')) | |
353 with tarfile.open(golddir + '.tgz', 'w:gz') as tar: | |
354 tar.add(os.path.join(golddir, 'lib'), arcname='lib', | |
355 filter=PrintTarProgress) | |
356 MaybeUpload(args, golddir, platform) | |
357 | |
358 # Zip up llvm-objdump for sanitizer coverage. | 333 # Zip up llvm-objdump for sanitizer coverage. |
359 objdumpdir = 'llvmobjdump-' + stamp | 334 objdumpdir = 'llvmobjdump-' + stamp |
360 shutil.rmtree(objdumpdir, ignore_errors=True) | 335 shutil.rmtree(objdumpdir, ignore_errors=True) |
361 os.makedirs(os.path.join(objdumpdir, 'bin')) | 336 os.makedirs(os.path.join(objdumpdir, 'bin')) |
362 shutil.copy(os.path.join(LLVM_RELEASE_DIR, 'bin', 'llvm-objdump' + exe_ext), | 337 shutil.copy(os.path.join(LLVM_RELEASE_DIR, 'bin', 'llvm-objdump' + exe_ext), |
363 os.path.join(objdumpdir, 'bin')) | 338 os.path.join(objdumpdir, 'bin')) |
364 with tarfile.open(objdumpdir + '.tgz', 'w:gz') as tar: | 339 with tarfile.open(objdumpdir + '.tgz', 'w:gz') as tar: |
365 tar.add(os.path.join(objdumpdir, 'bin'), arcname='bin', | 340 tar.add(os.path.join(objdumpdir, 'bin'), arcname='bin', |
366 filter=PrintTarProgress) | 341 filter=PrintTarProgress) |
367 MaybeUpload(args, objdumpdir, platform) | 342 MaybeUpload(args, objdumpdir, platform) |
(...skipping 11 matching lines...) Expand all Loading... |
379 MaybeUpload(args, translation_unit_dir, platform) | 354 MaybeUpload(args, translation_unit_dir, platform) |
380 | 355 |
381 if sys.platform == 'win32' and args.upload: | 356 if sys.platform == 'win32' and args.upload: |
382 UploadPDBToSymbolServer() | 357 UploadPDBToSymbolServer() |
383 | 358 |
384 # FIXME: Warn if the file already exists on the server. | 359 # FIXME: Warn if the file already exists on the server. |
385 | 360 |
386 | 361 |
387 if __name__ == '__main__': | 362 if __name__ == '__main__': |
388 sys.exit(main()) | 363 sys.exit(main()) |
OLD | NEW |