OLD | NEW |
1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. | 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. |
3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
5 | 5 |
6 """A tool to build chrome, executed by buildbot. | 6 """A tool to build chrome, executed by buildbot. |
7 | 7 |
8 When this is run, the current directory (cwd) should be the outer build | 8 When this is run, the current directory (cwd) should be the outer build |
9 directory (e.g., chrome-release/build/). | 9 directory (e.g., chrome-release/build/). |
10 | 10 |
11 For a list of command-line options, call this script with '--help'. | 11 For a list of command-line options, call this script with '--help'. |
12 """ | 12 """ |
13 | 13 |
14 import datetime | 14 import datetime |
15 import errno | 15 import errno |
16 import optparse | 16 import optparse |
17 import os | 17 import os |
18 import re | 18 import re |
19 import shlex | 19 import shlex |
20 import shutil | |
21 import sys | 20 import sys |
22 import time | 21 import time |
23 | 22 |
24 from common import chromium_utils | 23 from common import chromium_utils |
25 from slave import slave_utils | 24 from slave import slave_utils |
26 from slave import build_directory | 25 from slave import build_directory |
27 | 26 |
28 | 27 |
29 # Path of the scripts/slave/ checkout on the slave, found by looking at the | 28 # Path of the scripts/slave/ checkout on the slave, found by looking at the |
30 # current compile.py script's path's dirname(). | 29 # current compile.py script's path's dirname(). |
(...skipping 468 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
499 filter_obj=xcodebuild_filter) | 498 filter_obj=xcodebuild_filter) |
500 | 499 |
501 goma_teardown(options, env) | 500 goma_teardown(options, env) |
502 | 501 |
503 return result | 502 return result |
504 | 503 |
505 | 504 |
506 def common_make_settings( | 505 def common_make_settings( |
507 command, options, env, crosstool=None, compiler=None): | 506 command, options, env, crosstool=None, compiler=None): |
508 """ | 507 """ |
509 Sets desirable environment variables and command-line options | 508 Sets desirable environment variables and command-line options that are used |
510 that are common to the Make and SCons builds. Used on Linux | 509 in the Make build. |
511 and for the mac make build. | |
512 """ | 510 """ |
513 assert compiler in (None, 'clang', 'goma', 'goma-clang', 'jsonclang') | 511 assert compiler in (None, 'clang', 'goma', 'goma-clang', 'jsonclang') |
514 maybe_set_official_build_envvars(options, env) | 512 maybe_set_official_build_envvars(options, env) |
515 | 513 |
516 # Don't stop at the first error. | 514 # Don't stop at the first error. |
517 command.append('-k') | 515 command.append('-k') |
518 | 516 |
519 # Set jobs parallelization based on number of cores. | 517 # Set jobs parallelization based on number of cores. |
520 jobs = os.sysconf('SC_NPROCESSORS_ONLN') | 518 jobs = os.sysconf('SC_NPROCESSORS_ONLN') |
521 | 519 |
(...skipping 48 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
570 clang_dir = os.path.join(options.src_dir, | 568 clang_dir = os.path.join(options.src_dir, |
571 'third_party', 'llvm-build', 'Release+Asserts', 'bin') | 569 'third_party', 'llvm-build', 'Release+Asserts', 'bin') |
572 if options.goma_dir: | 570 if options.goma_dir: |
573 env['PATH'] = ':'.join([options.goma_dir, clang_dir, env['PATH']]) | 571 env['PATH'] = ':'.join([options.goma_dir, clang_dir, env['PATH']]) |
574 else: | 572 else: |
575 env['PATH'] = ':'.join([clang_dir, env['PATH']]) | 573 env['PATH'] = ':'.join([clang_dir, env['PATH']]) |
576 | 574 |
577 command.append('CC.host=' + env['CC']) | 575 command.append('CC.host=' + env['CC']) |
578 command.append('CXX.host=' + env['CXX']) | 576 command.append('CXX.host=' + env['CXX']) |
579 | 577 |
580 if chromium_utils.IsMac(): | 578 goma_jobs = 50 |
581 # The default process limit on 10.6 is 266 (`sysctl kern.maxprocperuid`), | |
582 # and about 100 processes are used by the system. The webkit bindings | |
583 # generation scripts open a preprocessor child process, so building at | |
584 # -j100 runs into the process limit. For now, just build with -j50. | |
585 goma_jobs = 50 | |
586 if options.clobber: | |
587 # Disable compiles on local machine. When the goma server-side object | |
588 # file cache is warm, this can speed up clobber builds by up to 30%. | |
589 env['GOMA_USE_LOCAL'] = '0' | |
590 # Temproary hack to try failing fast when the server looks unhealthy. | |
591 # 30 seconds is chosen because a local compile is almost certainly | |
592 # faster. | |
593 # crbug.com/257467 | |
594 env['GOMA_RETRY'] = '0' | |
595 env['GOMA_COMPILER_PROXY_RPC_TIMEOUT_SECS'] = '30' | |
596 else: | |
597 goma_jobs = 50 | |
598 if jobs < goma_jobs: | 579 if jobs < goma_jobs: |
599 jobs = goma_jobs | 580 jobs = goma_jobs |
600 command.append('-j%d' % jobs) | 581 command.append('-j%d' % jobs) |
601 return | 582 return |
602 | 583 |
603 if compiler == 'clang': | 584 if compiler == 'clang': |
604 clang_dir = os.path.join(options.src_dir, | 585 clang_dir = os.path.join(options.src_dir, |
605 'third_party', 'llvm-build', 'Release+Asserts', 'bin') | 586 'third_party', 'llvm-build', 'Release+Asserts', 'bin') |
606 env['CC'] = os.path.join(clang_dir, 'clang') | 587 env['CC'] = os.path.join(clang_dir, 'clang') |
607 env['CXX'] = os.path.join(clang_dir, 'clang++') | 588 env['CXX'] = os.path.join(clang_dir, 'clang++') |
(...skipping 27 matching lines...) Expand all Loading... |
635 working_dir = options.build_dir | 616 working_dir = options.build_dir |
636 else: | 617 else: |
637 # If no solution file (i.e. sub-project *.Makefile) is specified, try to | 618 # If no solution file (i.e. sub-project *.Makefile) is specified, try to |
638 # build from <build_dir>/Makefile, or if that doesn't exist, from | 619 # build from <build_dir>/Makefile, or if that doesn't exist, from |
639 # the top-level Makefile. | 620 # the top-level Makefile. |
640 if os.path.isfile(os.path.join(options.build_dir, 'Makefile')): | 621 if os.path.isfile(os.path.join(options.build_dir, 'Makefile')): |
641 working_dir = options.build_dir | 622 working_dir = options.build_dir |
642 else: | 623 else: |
643 working_dir = options.src_dir | 624 working_dir = options.src_dir |
644 | 625 |
645 # Lots of test-execution scripts hard-code 'sconsbuild' as the output | |
646 # directory. Accomodate them. | |
647 # TODO: remove when build_dir is properly parameterized in tests. | |
648 sconsbuild = os.path.join(working_dir, 'sconsbuild') | |
649 if os.path.islink(sconsbuild): | |
650 if os.readlink(sconsbuild) != 'out': | |
651 os.remove(sconsbuild) | |
652 elif os.path.exists(sconsbuild): | |
653 dead = sconsbuild + '.dead' | |
654 if os.path.isdir(dead): | |
655 shutil.rmtree(dead) | |
656 elif os.path.isfile(dead): | |
657 os.remove(dead) | |
658 os.rename(sconsbuild, sconsbuild+'.dead') | |
659 if not os.path.lexists(sconsbuild): | |
660 os.symlink('out', sconsbuild) | |
661 | |
662 os.chdir(working_dir) | 626 os.chdir(working_dir) |
663 common_make_settings(command, options, env, options.crosstool, | 627 common_make_settings(command, options, env, options.crosstool, |
664 options.compiler) | 628 options.compiler) |
665 | 629 |
666 # V=1 prints the actual executed command | 630 # V=1 prints the actual executed command |
667 if options.verbose: | 631 if options.verbose: |
668 command.extend(['V=1']) | 632 command.extend(['V=1']) |
669 command.extend(options.build_args + args) | 633 command.extend(options.build_args + args) |
670 | 634 |
671 # Run the build. | 635 # Run the build. |
(...skipping 206 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
878 # TODO(maruel): Remove the shell argument as soon as ninja.exe is in PATH. | 842 # TODO(maruel): Remove the shell argument as soon as ninja.exe is in PATH. |
879 # At the moment of writing, ninja.bat in depot_tools wraps | 843 # At the moment of writing, ninja.bat in depot_tools wraps |
880 # third_party\ninja.exe, which requires shell=True so it is found correctly. | 844 # third_party\ninja.exe, which requires shell=True so it is found correctly. |
881 result = chromium_utils.RunCommand( | 845 result = chromium_utils.RunCommand( |
882 command, env=env, shell=sys.platform=='win32') | 846 command, env=env, shell=sys.platform=='win32') |
883 | 847 |
884 goma_teardown(options, env) | 848 goma_teardown(options, env) |
885 return result | 849 return result |
886 | 850 |
887 | 851 |
888 def main_scons(options, args): | |
889 """Interprets options, clobbers object files, and calls scons. | |
890 """ | |
891 options.build_dir = os.path.abspath(options.build_dir) | |
892 if options.clobber: | |
893 print('Removing %s' % options.target_output_dir) | |
894 chromium_utils.RemoveDirectory(options.target_output_dir) | |
895 | |
896 os.chdir(options.build_dir) | |
897 | |
898 if sys.platform == 'win32': | |
899 command = ['hammer.bat'] | |
900 else: | |
901 command = ['hammer'] | |
902 | |
903 env = EchoDict(os.environ) | |
904 if sys.platform == 'linux2': | |
905 common_make_settings(command, options, env) | |
906 else: | |
907 command.extend(['-k']) | |
908 | |
909 command.extend([ | |
910 # Force scons to always check for dependency changes. | |
911 '--implicit-deps-changed', | |
912 '--mode=' + options.target, | |
913 ]) | |
914 | |
915 # Here's what you can uncomment if you need to see more info | |
916 # about what the build is doing on a slave: | |
917 # | |
918 # VERBOSE=1 (a setting in our local SCons config) replaces | |
919 # the "Compiling ..." and "Linking ..." lines with the | |
920 # actual executed command line(s) | |
921 # | |
922 # --debug=explain (a SCons option) will tell you why SCons | |
923 # is deciding to rebuild thing (the target doesn't exist, | |
924 # which .h file(s) changed, etc.) | |
925 # | |
926 #command.extend(['--debug=explain', 'VERBOSE=1']) | |
927 command.extend(options.build_args + args) | |
928 env.print_overrides() | |
929 return chromium_utils.RunCommand(command, env=env) | |
930 | |
931 | |
932 def main_win(options, args): | 852 def main_win(options, args): |
933 """Interprets options, clobbers object files, and calls the build tool. | 853 """Interprets options, clobbers object files, and calls the build tool. |
934 """ | 854 """ |
935 # Prefer the version specified in the .sln. When devenv.com is used at the | 855 # Prefer the version specified in the .sln. When devenv.com is used at the |
936 # command line to start a build, it doesn't accept sln file from a different | 856 # command line to start a build, it doesn't accept sln file from a different |
937 # version. | 857 # version. |
938 if not options.msvs_version: | 858 if not options.msvs_version: |
939 sln = open(os.path.join(options.build_dir, options.solution), 'rU') | 859 sln = open(os.path.join(options.build_dir, options.solution), 'rU') |
940 header = sln.readline().strip() | 860 header = sln.readline().strip() |
941 sln.close() | 861 sln.close() |
(...skipping 184 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1126 ret = None | 1046 ret = None |
1127 if build_tool == 'xcode': | 1047 if build_tool == 'xcode': |
1128 ret = os.path.join(src_dir, 'xcodebuild', | 1048 ret = os.path.join(src_dir, 'xcodebuild', |
1129 target + ('-iphoneos' if is_iphone else '')) | 1049 target + ('-iphoneos' if is_iphone else '')) |
1130 elif build_tool in ['make', 'ninja']: | 1050 elif build_tool in ['make', 'ninja']: |
1131 ret = os.path.join(src_dir, 'out', target) | 1051 ret = os.path.join(src_dir, 'out', target) |
1132 elif build_tool == 'make-android': | 1052 elif build_tool == 'make-android': |
1133 ret = os.path.join(src_dir, 'out') | 1053 ret = os.path.join(src_dir, 'out') |
1134 elif build_tool in ['msvs', 'vs', 'ib']: | 1054 elif build_tool in ['msvs', 'vs', 'ib']: |
1135 ret = os.path.join(src_dir, 'build', target) | 1055 ret = os.path.join(src_dir, 'build', target) |
1136 elif build_tool == 'scons': | |
1137 ret = os.path.join(src_dir, 'sconsbuild', target) | |
1138 else: | 1056 else: |
1139 raise NotImplementedError() | 1057 raise NotImplementedError() |
1140 return os.path.abspath(ret) | 1058 return os.path.abspath(ret) |
1141 | 1059 |
1142 | 1060 |
1143 def real_main(): | 1061 def real_main(): |
1144 option_parser = optparse.OptionParser() | 1062 option_parser = optparse.OptionParser() |
1145 option_parser.add_option('', '--clobber', action='store_true', default=False, | 1063 option_parser.add_option('', '--clobber', action='store_true', default=False, |
1146 help='delete the output directory before compiling') | 1064 help='delete the output directory before compiling') |
1147 option_parser.add_option('', '--clobber-post-fail', action='store_true', | 1065 option_parser.add_option('', '--clobber-post-fail', action='store_true', |
(...skipping 15 matching lines...) Expand all Loading... |
1163 help='name of project to build') | 1081 help='name of project to build') |
1164 option_parser.add_option('', '--build-dir', default='build', | 1082 option_parser.add_option('', '--build-dir', default='build', |
1165 help='path to directory containing solution and in ' | 1083 help='path to directory containing solution and in ' |
1166 'which the build output will be placed') | 1084 'which the build output will be placed') |
1167 option_parser.add_option('', '--src-dir', default=None, | 1085 option_parser.add_option('', '--src-dir', default=None, |
1168 help='path to the root of the source tree') | 1086 help='path to the root of the source tree') |
1169 option_parser.add_option('', '--mode', default='dev', | 1087 option_parser.add_option('', '--mode', default='dev', |
1170 help='build mode (dev or official) controlling ' | 1088 help='build mode (dev or official) controlling ' |
1171 'environment variables set during build') | 1089 'environment variables set during build') |
1172 option_parser.add_option('', '--build-tool', default=None, | 1090 option_parser.add_option('', '--build-tool', default=None, |
1173 help='specify build tool (ib, vs, scons, xcode)') | 1091 help='specify build tool (ib, vs, xcode)') |
1174 option_parser.add_option('', '--build-args', action='append', default=[], | 1092 option_parser.add_option('', '--build-args', action='append', default=[], |
1175 help='arguments to pass to the build tool') | 1093 help='arguments to pass to the build tool') |
1176 option_parser.add_option('', '--compiler', default=None, | 1094 option_parser.add_option('', '--compiler', default=None, |
1177 help='specify alternative compiler (e.g. clang)') | 1095 help='specify alternative compiler (e.g. clang)') |
1178 if chromium_utils.IsWindows(): | 1096 if chromium_utils.IsWindows(): |
1179 # Windows only. | 1097 # Windows only. |
1180 option_parser.add_option('', '--no-ib', action='store_true', default=False, | 1098 option_parser.add_option('', '--no-ib', action='store_true', default=False, |
1181 help='use Visual Studio instead of IncrediBuild') | 1099 help='use Visual Studio instead of IncrediBuild') |
1182 option_parser.add_option('', '--msvs_version', | 1100 option_parser.add_option('', '--msvs_version', |
1183 help='VisualStudio version to use') | 1101 help='VisualStudio version to use') |
(...skipping 45 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1229 else: | 1147 else: |
1230 print('Please specify --build-tool.') | 1148 print('Please specify --build-tool.') |
1231 return 1 | 1149 return 1 |
1232 else: | 1150 else: |
1233 build_tool_map = { | 1151 build_tool_map = { |
1234 'ib' : main_win, | 1152 'ib' : main_win, |
1235 'vs' : main_win, | 1153 'vs' : main_win, |
1236 'make' : main_make, | 1154 'make' : main_make, |
1237 'make-android' : main_make_android, | 1155 'make-android' : main_make_android, |
1238 'ninja' : main_ninja, | 1156 'ninja' : main_ninja, |
1239 'scons' : main_scons, | |
1240 'xcode' : main_xcode, | 1157 'xcode' : main_xcode, |
1241 } | 1158 } |
1242 main = build_tool_map.get(options.build_tool) | 1159 main = build_tool_map.get(options.build_tool) |
1243 if not main: | 1160 if not main: |
1244 sys.stderr.write('Unknown build tool %s.\n' % repr(options.build_tool)) | 1161 sys.stderr.write('Unknown build tool %s.\n' % repr(options.build_tool)) |
1245 return 2 | 1162 return 2 |
1246 | 1163 |
1247 options.target_output_dir = get_target_build_dir(options.build_tool, | 1164 options.target_output_dir = get_target_build_dir(options.build_tool, |
1248 options.src_dir, options.target, 'iphoneos' in args) | 1165 options.src_dir, options.target, 'iphoneos' in args) |
1249 options.clobber = (options.clobber or | 1166 options.clobber = (options.clobber or |
1250 landmines_triggered(options.target_output_dir)) | 1167 landmines_triggered(options.target_output_dir)) |
1251 | 1168 |
1252 return main(options, args) | 1169 return main(options, args) |
1253 | 1170 |
1254 | 1171 |
1255 if '__main__' == __name__: | 1172 if '__main__' == __name__: |
1256 sys.exit(real_main()) | 1173 sys.exit(real_main()) |
OLD | NEW |