| OLD | NEW |
| 1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
| 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. | 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. |
| 3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
| 4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
| 5 | 5 |
| 6 """A tool to build chrome, executed by buildbot. | 6 """A tool to build chrome, executed by buildbot. |
| 7 | 7 |
| 8 When this is run, the current directory (cwd) should be the outer build | 8 When this is run, the current directory (cwd) should be the outer build |
| 9 directory (e.g., chrome-release/build/). | 9 directory (e.g., chrome-release/build/). |
| 10 | 10 |
| 11 For a list of command-line options, call this script with '--help'. | 11 For a list of command-line options, call this script with '--help'. |
| 12 """ | 12 """ |
| 13 | 13 |
| 14 import datetime | 14 import datetime |
| 15 import errno | 15 import errno |
| 16 import optparse | 16 import optparse |
| 17 import os | 17 import os |
| 18 import re | 18 import re |
| 19 import shlex | 19 import shlex |
| 20 import shutil | |
| 21 import sys | 20 import sys |
| 22 import time | 21 import time |
| 23 | 22 |
| 24 from common import chromium_utils | 23 from common import chromium_utils |
| 25 from slave import slave_utils | 24 from slave import slave_utils |
| 26 from slave import build_directory | 25 from slave import build_directory |
| 27 | 26 |
| 28 | 27 |
| 29 # Path of the scripts/slave/ checkout on the slave, found by looking at the | 28 # Path of the scripts/slave/ checkout on the slave, found by looking at the |
| 30 # current compile.py script's path's dirname(). | 29 # current compile.py script's path's dirname(). |
| (...skipping 468 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 499 filter_obj=xcodebuild_filter) | 498 filter_obj=xcodebuild_filter) |
| 500 | 499 |
| 501 goma_teardown(options, env) | 500 goma_teardown(options, env) |
| 502 | 501 |
| 503 return result | 502 return result |
| 504 | 503 |
| 505 | 504 |
| 506 def common_make_settings( | 505 def common_make_settings( |
| 507 command, options, env, crosstool=None, compiler=None): | 506 command, options, env, crosstool=None, compiler=None): |
| 508 """ | 507 """ |
| 509 Sets desirable environment variables and command-line options | 508 Sets desirable environment variables and command-line options that are used |
| 510 that are common to the Make and SCons builds. Used on Linux | 509 in the Make build. |
| 511 and for the mac make build. | |
| 512 """ | 510 """ |
| 513 assert compiler in (None, 'clang', 'goma', 'goma-clang', 'jsonclang') | 511 assert compiler in (None, 'clang', 'goma', 'goma-clang', 'jsonclang') |
| 514 maybe_set_official_build_envvars(options, env) | 512 maybe_set_official_build_envvars(options, env) |
| 515 | 513 |
| 516 # Don't stop at the first error. | 514 # Don't stop at the first error. |
| 517 command.append('-k') | 515 command.append('-k') |
| 518 | 516 |
| 519 # Set jobs parallelization based on number of cores. | 517 # Set jobs parallelization based on number of cores. |
| 520 jobs = os.sysconf('SC_NPROCESSORS_ONLN') | 518 jobs = os.sysconf('SC_NPROCESSORS_ONLN') |
| 521 | 519 |
| (...skipping 48 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 570 clang_dir = os.path.join(options.src_dir, | 568 clang_dir = os.path.join(options.src_dir, |
| 571 'third_party', 'llvm-build', 'Release+Asserts', 'bin') | 569 'third_party', 'llvm-build', 'Release+Asserts', 'bin') |
| 572 if options.goma_dir: | 570 if options.goma_dir: |
| 573 env['PATH'] = ':'.join([options.goma_dir, clang_dir, env['PATH']]) | 571 env['PATH'] = ':'.join([options.goma_dir, clang_dir, env['PATH']]) |
| 574 else: | 572 else: |
| 575 env['PATH'] = ':'.join([clang_dir, env['PATH']]) | 573 env['PATH'] = ':'.join([clang_dir, env['PATH']]) |
| 576 | 574 |
| 577 command.append('CC.host=' + env['CC']) | 575 command.append('CC.host=' + env['CC']) |
| 578 command.append('CXX.host=' + env['CXX']) | 576 command.append('CXX.host=' + env['CXX']) |
| 579 | 577 |
| 580 if chromium_utils.IsMac(): | 578 goma_jobs = 50 |
| 581 # The default process limit on 10.6 is 266 (`sysctl kern.maxprocperuid`), | |
| 582 # and about 100 processes are used by the system. The webkit bindings | |
| 583 # generation scripts open a preprocessor child process, so building at | |
| 584 # -j100 runs into the process limit. For now, just build with -j50. | |
| 585 goma_jobs = 50 | |
| 586 if options.clobber: | |
| 587 # Disable compiles on local machine. When the goma server-side object | |
| 588 # file cache is warm, this can speed up clobber builds by up to 30%. | |
| 589 env['GOMA_USE_LOCAL'] = '0' | |
| 590 # Temproary hack to try failing fast when the server looks unhealthy. | |
| 591 # 30 seconds is chosen because a local compile is almost certainly | |
| 592 # faster. | |
| 593 # crbug.com/257467 | |
| 594 env['GOMA_RETRY'] = '0' | |
| 595 env['GOMA_COMPILER_PROXY_RPC_TIMEOUT_SECS'] = '30' | |
| 596 else: | |
| 597 goma_jobs = 50 | |
| 598 if jobs < goma_jobs: | 579 if jobs < goma_jobs: |
| 599 jobs = goma_jobs | 580 jobs = goma_jobs |
| 600 command.append('-j%d' % jobs) | 581 command.append('-j%d' % jobs) |
| 601 return | 582 return |
| 602 | 583 |
| 603 if compiler == 'clang': | 584 if compiler == 'clang': |
| 604 clang_dir = os.path.join(options.src_dir, | 585 clang_dir = os.path.join(options.src_dir, |
| 605 'third_party', 'llvm-build', 'Release+Asserts', 'bin') | 586 'third_party', 'llvm-build', 'Release+Asserts', 'bin') |
| 606 env['CC'] = os.path.join(clang_dir, 'clang') | 587 env['CC'] = os.path.join(clang_dir, 'clang') |
| 607 env['CXX'] = os.path.join(clang_dir, 'clang++') | 588 env['CXX'] = os.path.join(clang_dir, 'clang++') |
| (...skipping 27 matching lines...) Expand all Loading... |
| 635 working_dir = options.build_dir | 616 working_dir = options.build_dir |
| 636 else: | 617 else: |
| 637 # If no solution file (i.e. sub-project *.Makefile) is specified, try to | 618 # If no solution file (i.e. sub-project *.Makefile) is specified, try to |
| 638 # build from <build_dir>/Makefile, or if that doesn't exist, from | 619 # build from <build_dir>/Makefile, or if that doesn't exist, from |
| 639 # the top-level Makefile. | 620 # the top-level Makefile. |
| 640 if os.path.isfile(os.path.join(options.build_dir, 'Makefile')): | 621 if os.path.isfile(os.path.join(options.build_dir, 'Makefile')): |
| 641 working_dir = options.build_dir | 622 working_dir = options.build_dir |
| 642 else: | 623 else: |
| 643 working_dir = options.src_dir | 624 working_dir = options.src_dir |
| 644 | 625 |
| 645 # Lots of test-execution scripts hard-code 'sconsbuild' as the output | |
| 646 # directory. Accomodate them. | |
| 647 # TODO: remove when build_dir is properly parameterized in tests. | |
| 648 sconsbuild = os.path.join(working_dir, 'sconsbuild') | |
| 649 if os.path.islink(sconsbuild): | |
| 650 if os.readlink(sconsbuild) != 'out': | |
| 651 os.remove(sconsbuild) | |
| 652 elif os.path.exists(sconsbuild): | |
| 653 dead = sconsbuild + '.dead' | |
| 654 if os.path.isdir(dead): | |
| 655 shutil.rmtree(dead) | |
| 656 elif os.path.isfile(dead): | |
| 657 os.remove(dead) | |
| 658 os.rename(sconsbuild, sconsbuild+'.dead') | |
| 659 if not os.path.lexists(sconsbuild): | |
| 660 os.symlink('out', sconsbuild) | |
| 661 | |
| 662 os.chdir(working_dir) | 626 os.chdir(working_dir) |
| 663 common_make_settings(command, options, env, options.crosstool, | 627 common_make_settings(command, options, env, options.crosstool, |
| 664 options.compiler) | 628 options.compiler) |
| 665 | 629 |
| 666 # V=1 prints the actual executed command | 630 # V=1 prints the actual executed command |
| 667 if options.verbose: | 631 if options.verbose: |
| 668 command.extend(['V=1']) | 632 command.extend(['V=1']) |
| 669 command.extend(options.build_args + args) | 633 command.extend(options.build_args + args) |
| 670 | 634 |
| 671 # Run the build. | 635 # Run the build. |
| (...skipping 206 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 878 # TODO(maruel): Remove the shell argument as soon as ninja.exe is in PATH. | 842 # TODO(maruel): Remove the shell argument as soon as ninja.exe is in PATH. |
| 879 # At the moment of writing, ninja.bat in depot_tools wraps | 843 # At the moment of writing, ninja.bat in depot_tools wraps |
| 880 # third_party\ninja.exe, which requires shell=True so it is found correctly. | 844 # third_party\ninja.exe, which requires shell=True so it is found correctly. |
| 881 result = chromium_utils.RunCommand( | 845 result = chromium_utils.RunCommand( |
| 882 command, env=env, shell=sys.platform=='win32') | 846 command, env=env, shell=sys.platform=='win32') |
| 883 | 847 |
| 884 goma_teardown(options, env) | 848 goma_teardown(options, env) |
| 885 return result | 849 return result |
| 886 | 850 |
| 887 | 851 |
| 888 def main_scons(options, args): | |
| 889 """Interprets options, clobbers object files, and calls scons. | |
| 890 """ | |
| 891 options.build_dir = os.path.abspath(options.build_dir) | |
| 892 if options.clobber: | |
| 893 print('Removing %s' % options.target_output_dir) | |
| 894 chromium_utils.RemoveDirectory(options.target_output_dir) | |
| 895 | |
| 896 os.chdir(options.build_dir) | |
| 897 | |
| 898 if sys.platform == 'win32': | |
| 899 command = ['hammer.bat'] | |
| 900 else: | |
| 901 command = ['hammer'] | |
| 902 | |
| 903 env = EchoDict(os.environ) | |
| 904 if sys.platform == 'linux2': | |
| 905 common_make_settings(command, options, env) | |
| 906 else: | |
| 907 command.extend(['-k']) | |
| 908 | |
| 909 command.extend([ | |
| 910 # Force scons to always check for dependency changes. | |
| 911 '--implicit-deps-changed', | |
| 912 '--mode=' + options.target, | |
| 913 ]) | |
| 914 | |
| 915 # Here's what you can uncomment if you need to see more info | |
| 916 # about what the build is doing on a slave: | |
| 917 # | |
| 918 # VERBOSE=1 (a setting in our local SCons config) replaces | |
| 919 # the "Compiling ..." and "Linking ..." lines with the | |
| 920 # actual executed command line(s) | |
| 921 # | |
| 922 # --debug=explain (a SCons option) will tell you why SCons | |
| 923 # is deciding to rebuild thing (the target doesn't exist, | |
| 924 # which .h file(s) changed, etc.) | |
| 925 # | |
| 926 #command.extend(['--debug=explain', 'VERBOSE=1']) | |
| 927 command.extend(options.build_args + args) | |
| 928 env.print_overrides() | |
| 929 return chromium_utils.RunCommand(command, env=env) | |
| 930 | |
| 931 | |
| 932 def main_win(options, args): | 852 def main_win(options, args): |
| 933 """Interprets options, clobbers object files, and calls the build tool. | 853 """Interprets options, clobbers object files, and calls the build tool. |
| 934 """ | 854 """ |
| 935 # Prefer the version specified in the .sln. When devenv.com is used at the | 855 # Prefer the version specified in the .sln. When devenv.com is used at the |
| 936 # command line to start a build, it doesn't accept sln file from a different | 856 # command line to start a build, it doesn't accept sln file from a different |
| 937 # version. | 857 # version. |
| 938 if not options.msvs_version: | 858 if not options.msvs_version: |
| 939 sln = open(os.path.join(options.build_dir, options.solution), 'rU') | 859 sln = open(os.path.join(options.build_dir, options.solution), 'rU') |
| 940 header = sln.readline().strip() | 860 header = sln.readline().strip() |
| 941 sln.close() | 861 sln.close() |
| (...skipping 184 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1126 ret = None | 1046 ret = None |
| 1127 if build_tool == 'xcode': | 1047 if build_tool == 'xcode': |
| 1128 ret = os.path.join(src_dir, 'xcodebuild', | 1048 ret = os.path.join(src_dir, 'xcodebuild', |
| 1129 target + ('-iphoneos' if is_iphone else '')) | 1049 target + ('-iphoneos' if is_iphone else '')) |
| 1130 elif build_tool in ['make', 'ninja']: | 1050 elif build_tool in ['make', 'ninja']: |
| 1131 ret = os.path.join(src_dir, 'out', target) | 1051 ret = os.path.join(src_dir, 'out', target) |
| 1132 elif build_tool == 'make-android': | 1052 elif build_tool == 'make-android': |
| 1133 ret = os.path.join(src_dir, 'out') | 1053 ret = os.path.join(src_dir, 'out') |
| 1134 elif build_tool in ['msvs', 'vs', 'ib']: | 1054 elif build_tool in ['msvs', 'vs', 'ib']: |
| 1135 ret = os.path.join(src_dir, 'build', target) | 1055 ret = os.path.join(src_dir, 'build', target) |
| 1136 elif build_tool == 'scons': | |
| 1137 ret = os.path.join(src_dir, 'sconsbuild', target) | |
| 1138 else: | 1056 else: |
| 1139 raise NotImplementedError() | 1057 raise NotImplementedError() |
| 1140 return os.path.abspath(ret) | 1058 return os.path.abspath(ret) |
| 1141 | 1059 |
| 1142 | 1060 |
| 1143 def real_main(): | 1061 def real_main(): |
| 1144 option_parser = optparse.OptionParser() | 1062 option_parser = optparse.OptionParser() |
| 1145 option_parser.add_option('', '--clobber', action='store_true', default=False, | 1063 option_parser.add_option('', '--clobber', action='store_true', default=False, |
| 1146 help='delete the output directory before compiling') | 1064 help='delete the output directory before compiling') |
| 1147 option_parser.add_option('', '--clobber-post-fail', action='store_true', | 1065 option_parser.add_option('', '--clobber-post-fail', action='store_true', |
| (...skipping 15 matching lines...) Expand all Loading... |
| 1163 help='name of project to build') | 1081 help='name of project to build') |
| 1164 option_parser.add_option('', '--build-dir', default='build', | 1082 option_parser.add_option('', '--build-dir', default='build', |
| 1165 help='path to directory containing solution and in ' | 1083 help='path to directory containing solution and in ' |
| 1166 'which the build output will be placed') | 1084 'which the build output will be placed') |
| 1167 option_parser.add_option('', '--src-dir', default=None, | 1085 option_parser.add_option('', '--src-dir', default=None, |
| 1168 help='path to the root of the source tree') | 1086 help='path to the root of the source tree') |
| 1169 option_parser.add_option('', '--mode', default='dev', | 1087 option_parser.add_option('', '--mode', default='dev', |
| 1170 help='build mode (dev or official) controlling ' | 1088 help='build mode (dev or official) controlling ' |
| 1171 'environment variables set during build') | 1089 'environment variables set during build') |
| 1172 option_parser.add_option('', '--build-tool', default=None, | 1090 option_parser.add_option('', '--build-tool', default=None, |
| 1173 help='specify build tool (ib, vs, scons, xcode)') | 1091 help='specify build tool (ib, vs, xcode)') |
| 1174 option_parser.add_option('', '--build-args', action='append', default=[], | 1092 option_parser.add_option('', '--build-args', action='append', default=[], |
| 1175 help='arguments to pass to the build tool') | 1093 help='arguments to pass to the build tool') |
| 1176 option_parser.add_option('', '--compiler', default=None, | 1094 option_parser.add_option('', '--compiler', default=None, |
| 1177 help='specify alternative compiler (e.g. clang)') | 1095 help='specify alternative compiler (e.g. clang)') |
| 1178 if chromium_utils.IsWindows(): | 1096 if chromium_utils.IsWindows(): |
| 1179 # Windows only. | 1097 # Windows only. |
| 1180 option_parser.add_option('', '--no-ib', action='store_true', default=False, | 1098 option_parser.add_option('', '--no-ib', action='store_true', default=False, |
| 1181 help='use Visual Studio instead of IncrediBuild') | 1099 help='use Visual Studio instead of IncrediBuild') |
| 1182 option_parser.add_option('', '--msvs_version', | 1100 option_parser.add_option('', '--msvs_version', |
| 1183 help='VisualStudio version to use') | 1101 help='VisualStudio version to use') |
| (...skipping 45 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1229 else: | 1147 else: |
| 1230 print('Please specify --build-tool.') | 1148 print('Please specify --build-tool.') |
| 1231 return 1 | 1149 return 1 |
| 1232 else: | 1150 else: |
| 1233 build_tool_map = { | 1151 build_tool_map = { |
| 1234 'ib' : main_win, | 1152 'ib' : main_win, |
| 1235 'vs' : main_win, | 1153 'vs' : main_win, |
| 1236 'make' : main_make, | 1154 'make' : main_make, |
| 1237 'make-android' : main_make_android, | 1155 'make-android' : main_make_android, |
| 1238 'ninja' : main_ninja, | 1156 'ninja' : main_ninja, |
| 1239 'scons' : main_scons, | |
| 1240 'xcode' : main_xcode, | 1157 'xcode' : main_xcode, |
| 1241 } | 1158 } |
| 1242 main = build_tool_map.get(options.build_tool) | 1159 main = build_tool_map.get(options.build_tool) |
| 1243 if not main: | 1160 if not main: |
| 1244 sys.stderr.write('Unknown build tool %s.\n' % repr(options.build_tool)) | 1161 sys.stderr.write('Unknown build tool %s.\n' % repr(options.build_tool)) |
| 1245 return 2 | 1162 return 2 |
| 1246 | 1163 |
| 1247 options.target_output_dir = get_target_build_dir(options.build_tool, | 1164 options.target_output_dir = get_target_build_dir(options.build_tool, |
| 1248 options.src_dir, options.target, 'iphoneos' in args) | 1165 options.src_dir, options.target, 'iphoneos' in args) |
| 1249 options.clobber = (options.clobber or | 1166 options.clobber = (options.clobber or |
| 1250 landmines_triggered(options.target_output_dir)) | 1167 landmines_triggered(options.target_output_dir)) |
| 1251 | 1168 |
| 1252 return main(options, args) | 1169 return main(options, args) |
| 1253 | 1170 |
| 1254 | 1171 |
| 1255 if '__main__' == __name__: | 1172 if '__main__' == __name__: |
| 1256 sys.exit(real_main()) | 1173 sys.exit(real_main()) |
| OLD | NEW |