Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(532)

Side by Side Diff: scripts/slave/compile.py

Issue 2191483002: Remove dead code from compile.py. (Closed) Base URL: https://chromium.googlesource.com/chromium/tools/build.git@master
Patch Set: Created 4 years, 4 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
« no previous file with comments | « no previous file | no next file » | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 #!/usr/bin/env python 1 #!/usr/bin/env python
2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
3 # Use of this source code is governed by a BSD-style license that can be 3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file. 4 # found in the LICENSE file.
5 5
6 """A tool to build chrome, executed by buildbot. 6 """A tool to build chrome, executed by buildbot.
7 7
8 When this is run, the current directory (cwd) should be the outer build 8 When this is run, the current directory (cwd) should be the outer build
9 directory (e.g., chrome-release/build/). 9 directory (e.g., chrome-release/build/).
10 10
(...skipping 58 matching lines...) Expand 10 before | Expand all | Expand 10 after
69 fh.write(' %s=%s\n' % (k, self[k])) 69 fh.write(' %s=%s\n' % (k, self[k]))
70 else: 70 else:
71 fh.write(' %s (removed)\n' % k) 71 fh.write(' %s (removed)\n' % k)
72 fh.write('\n') 72 fh.write('\n')
73 73
74 74
75 # TODO(tikuta): move to goma_utils.py 75 # TODO(tikuta): move to goma_utils.py
76 def goma_setup(options, env): 76 def goma_setup(options, env):
77 """Sets up goma if necessary. 77 """Sets up goma if necessary.
78 78
79 If using the Goma compiler, first call goma_ctl to ensure the proxy is 79 If using the Goma compiler, first call goma_ctl to ensure the proxy is
80 available, and returns (True, instance of cloudtail subprocess). 80 available, and returns (True, instance of cloudtail subprocess).
81 If it failed to start up compiler_proxy, modify options.compiler and 81 If it failed to start up compiler_proxy, modify options.compiler
82 options.goma_dir and returns (False, None) 82 and options.goma_dir, modify env to GOMA_DISABLED=true,
83 83 and returns (False, None).
84 """ 84 """
85 if options.compiler not in ('goma', 'goma-clang'): 85 if options.compiler not in ('goma', 'goma-clang'):
86 # Unset goma_dir to make sure we'll not use goma. 86 # Unset goma_dir to make sure we'll not use goma.
87 options.goma_dir = None 87 options.goma_dir = None
88 return False, None 88 return False, None
89 89
90 hostname = goma_utils.GetShortHostname()
91 # HACK(shinyak, yyanagisawa, goma): Windows NO_NACL_GOMA (crbug.com/390764)
92 # Building NaCl untrusted code using goma brings large performance
93 # improvement but it sometimes cause build failure by race condition.
94 # Let me enable goma build on goma canary buildslaves to confirm the issue
95 # has been fixed by a workaround.
96 # vm*-m4 are trybots. build*-m1 and vm*-m1 are all goma canary bots.
97 if hostname in ['build28-m1', 'build58-m1', 'vm191-m1', 'vm480-m1',
98 'vm820-m1', 'vm821-m1', 'vm848-m1']:
99 env['NO_NACL_GOMA'] = 'false'
100
101 if options.goma_fail_fast: 90 if options.goma_fail_fast:
102 # startup fails when initial ping failed. 91 # startup fails when initial ping failed.
103 env['GOMA_FAIL_FAST'] = 'true' 92 env['GOMA_FAIL_FAST'] = 'true'
104 else: 93 else:
105 # If a network error continues 30 minutes, compiler_proxy make the compile 94 # If a network error continues 30 minutes, compiler_proxy make the compile
106 # failed. When people use goma, they expect using goma is faster than 95 # failed. When people use goma, they expect using goma is faster than
107 # compile locally. If goma cannot guarantee that, let it make compile 96 # compile locally. If goma cannot guarantee that, let it make compile
108 # as error. 97 # as error.
109 env['GOMA_ALLOWED_NETWORK_ERROR_DURATION'] = '1800' 98 env['GOMA_ALLOWED_NETWORK_ERROR_DURATION'] = '1800'
110 99
111 # HACK(yyanagisawa): reduce GOMA_BURST_MAX_PROCS crbug.com/592306
112 # Recently, I sometimes see buildbot slave time out, one possibility I come
113 # up with is burst mode use up resource.
114 # Let me temporary set small values to GOMA_BURST_MAX_PROCS to confirm
115 # the possibility is true or false.
116 max_subprocs = '3'
117 max_heavy_subprocs = '1'
118 number_of_processors = 0
119 try:
120 number_of_processors = multiprocessing.cpu_count()
121 except NotImplementedError:
122 print 'cpu_count() is not implemented, using default value.'
123 number_of_processors = 1
124 if number_of_processors > 3:
125 max_subprocs = str(number_of_processors - 1)
126 max_heavy_subprocs = str(number_of_processors / 2)
127 env['GOMA_BURST_MAX_SUBPROCS'] = max_subprocs
128 env['GOMA_BURST_MAX_SUBPROCS_LOW'] = max_subprocs
129 env['GOMA_BURST_MAX_SUBPROCS_HEAVY'] = max_heavy_subprocs
130
131 # Caches CRLs in GOMA_CACHE_DIR. 100 # Caches CRLs in GOMA_CACHE_DIR.
132 # Since downloading CRLs is usually slow, caching them may improves 101 # Since downloading CRLs is usually slow, caching them may improves
133 # compiler_proxy start time. 102 # compiler_proxy start time.
134 if not os.path.exists(options.goma_cache_dir): 103 if not os.path.exists(options.goma_cache_dir):
135 os.mkdir(options.goma_cache_dir, 0700) 104 os.mkdir(options.goma_cache_dir, 0700)
136 env['GOMA_CACHE_DIR'] = options.goma_cache_dir 105 env['GOMA_CACHE_DIR'] = options.goma_cache_dir
137 106
138 # Enable DepsCache. DepsCache caches the list of files to send goma server. 107 # Enable DepsCache. DepsCache caches the list of files to send goma server.
139 # This will greatly improve build speed when cache is warmed. 108 # This will greatly improve build speed when cache is warmed.
140 # The cache file is stored in the target output directory. 109 # The cache file is stored in the target output directory.
141 env['GOMA_DEPS_CACHE_DIR'] = ( 110 env['GOMA_DEPS_CACHE_DIR'] = (
142 options.goma_deps_cache_dir or options.target_output_dir) 111 options.goma_deps_cache_dir or options.target_output_dir)
143 112
144 if not env.get('GOMA_HERMETIC'): 113 if options.goma_hermetic:
145 env['GOMA_HERMETIC'] = options.goma_hermetic 114 env['GOMA_HERMETIC'] = options.goma_hermetic
146 if options.goma_enable_remote_link: 115 if options.goma_enable_remote_link:
147 env['GOMA_ENABLE_REMOTE_LINK'] = options.goma_enable_remote_link 116 env['GOMA_ENABLE_REMOTE_LINK'] = options.goma_enable_remote_link
148 if options.goma_store_local_run_output: 117 if options.goma_store_local_run_output:
149 env['GOMA_STORE_LOCAL_RUN_OUTPUT'] = options.goma_store_local_run_output 118 env['GOMA_STORE_LOCAL_RUN_OUTPUT'] = options.goma_store_local_run_output
150 if options.goma_enable_compiler_info_cache: 119 if options.goma_enable_compiler_info_cache:
151 # Will be stored in GOMA_CACHE_DIR. 120 # Will be stored in GOMA_CACHE_DIR.
152 env['GOMA_COMPILER_INFO_CACHE_FILE'] = 'goma-compiler-info.cache' 121 env['GOMA_COMPILER_INFO_CACHE_FILE'] = 'goma-compiler-info.cache'
153 122
154 if options.build_data_dir: 123 if options.build_data_dir:
155 env['GOMA_DUMP_STATS_FILE'] = os.path.join(options.build_data_dir, 124 env['GOMA_DUMP_STATS_FILE'] = os.path.join(options.build_data_dir,
156 'goma_stats_proto') 125 'goma_stats_proto')
157 126
158 # goma is requested.
159 goma_key = os.path.join(options.goma_dir, 'goma.key')
160 if os.path.exists(goma_key):
161 env['GOMA_API_KEY_FILE'] = goma_key
162 if options.goma_service_account_json_file: 127 if options.goma_service_account_json_file:
163 env['GOMA_SERVICE_ACCOUNT_JSON_FILE'] = \ 128 env['GOMA_SERVICE_ACCOUNT_JSON_FILE'] = \
164 options.goma_service_account_json_file 129 options.goma_service_account_json_file
165 if chromium_utils.IsWindows():
166 env['GOMA_RPC_EXTRA_PARAMS'] = '?win'
167 goma_start_command = ['restart'] 130 goma_start_command = ['restart']
168 goma_ctl_cmd = [sys.executable, 131 goma_ctl_cmd = [sys.executable,
169 os.path.join(options.goma_dir, 'goma_ctl.py')] 132 os.path.join(options.goma_dir, 'goma_ctl.py')]
170 result = chromium_utils.RunCommand(goma_ctl_cmd + goma_start_command, env=env) 133 result = chromium_utils.RunCommand(goma_ctl_cmd + goma_start_command, env=env)
171 if not result: 134 if not result:
172 # goma started sucessfully. 135 # goma started sucessfully.
173 # Making cloudtail to upload the latest log. 136 # Making cloudtail to upload the latest log.
174 # TODO(yyanagisawa): install cloudtail from CIPD. 137 # TODO(yyanagisawa): install cloudtail from CIPD.
175 cloudtail_path = '/opt/infra-tools/cloudtail' 138 cloudtail_path = '/opt/infra-tools/cloudtail'
176 if chromium_utils.IsWindows(): 139 if chromium_utils.IsWindows():
(...skipping 43 matching lines...) Expand 10 before | Expand all | Expand 10 after
220 options.compiler = None 183 options.compiler = None
221 # Reset options.goma_dir. 184 # Reset options.goma_dir.
222 options.goma_dir = None 185 options.goma_dir = None
223 env['GOMA_DISABLED'] = '1' 186 env['GOMA_DISABLED'] = '1'
224 return False, None 187 return False, None
225 188
226 189
227 # TODO(tikuta): move to goma_utils.py 190 # TODO(tikuta): move to goma_utils.py
228 def goma_teardown(options, env, exit_status, cloudtail_proc): 191 def goma_teardown(options, env, exit_status, cloudtail_proc):
229 """Tears down goma if necessary. """ 192 """Tears down goma if necessary. """
230 if (options.compiler in ('goma', 'goma-clang') and 193 if options.goma_dir:
231 options.goma_dir):
232 override_gsutil = None 194 override_gsutil = None
233 if options.gsutil_py_path: 195 if options.gsutil_py_path:
234 override_gsutil = [sys.executable, options.gsutil_py_path] 196 override_gsutil = [sys.executable, options.gsutil_py_path]
235 197
236 # If goma compiler_proxy crashes during the build, there could be crash 198 # If goma compiler_proxy crashes during the build, there could be crash
237 # dump. 199 # dump.
238 if options.build_data_dir: 200 if options.build_data_dir:
239 env['GOMACTL_CRASH_REPORT_ID_FILE'] = os.path.join(options.build_data_dir, 201 env['GOMACTL_CRASH_REPORT_ID_FILE'] = os.path.join(options.build_data_dir,
240 'crash_report_id_file') 202 'crash_report_id_file')
241 goma_ctl_cmd = [sys.executable, 203 goma_ctl_cmd = [sys.executable,
242 os.path.join(options.goma_dir, 'goma_ctl.py')] 204 os.path.join(options.goma_dir, 'goma_ctl.py')]
243 if options.goma_jsonstatus: 205 if options.goma_jsonstatus:
244 chromium_utils.RunCommand( 206 chromium_utils.RunCommand(
245 goma_ctl_cmd + ['jsonstatus', options.goma_jsonstatus], env=env) 207 goma_ctl_cmd + ['jsonstatus', options.goma_jsonstatus], env=env)
246 goma_utils.SendGomaTsMon(options.goma_jsonstatus, exit_status) 208 goma_utils.SendGomaTsMon(options.goma_jsonstatus, exit_status)
247 # Always stop the proxy for now to allow in-place update. 209 # Always stop the proxy to dump GomaStats.
248 chromium_utils.RunCommand(goma_ctl_cmd + ['stop'], env=env) 210 chromium_utils.RunCommand(goma_ctl_cmd + ['stop'], env=env)
249 goma_utils.UploadGomaCompilerProxyInfo(override_gsutil=override_gsutil) 211 goma_utils.UploadGomaCompilerProxyInfo(override_gsutil=override_gsutil)
250 if env.get('GOMA_DUMP_STATS_FILE'): 212 if env.get('GOMA_DUMP_STATS_FILE'):
251 goma_utils.SendGomaStats(env['GOMA_DUMP_STATS_FILE'], 213 goma_utils.SendGomaStats(env['GOMA_DUMP_STATS_FILE'],
252 env.get('GOMACTL_CRASH_REPORT_ID_FILE'), 214 env.get('GOMACTL_CRASH_REPORT_ID_FILE'),
253 options.build_data_dir) 215 options.build_data_dir)
254 if cloudtail_proc: 216 if cloudtail_proc:
255 cloudtail_proc.terminate() 217 cloudtail_proc.terminate()
256 cloudtail_proc.wait() 218 cloudtail_proc.wait()
257 219
(...skipping 203 matching lines...) Expand 10 before | Expand all | Expand 10 after
461 finally: 423 finally:
462 override_gsutil = None 424 override_gsutil = None
463 if options.gsutil_py_path: 425 if options.gsutil_py_path:
464 override_gsutil = [sys.executable, options.gsutil_py_path] 426 override_gsutil = [sys.executable, options.gsutil_py_path]
465 427
466 goma_utils.UploadNinjaLog( 428 goma_utils.UploadNinjaLog(
467 options.target_output_dir, options.compiler, command, exit_status, 429 options.target_output_dir, options.compiler, command, exit_status,
468 override_gsutil=override_gsutil) 430 override_gsutil=override_gsutil)
469 431
470 432
471 def get_target_build_dir(args, options): 433 def get_target_build_dir(options):
472 """Keep this function in sync with src/build/landmines.py""" 434 """Keep this function in sync with src/build/landmines.py"""
473 if chromium_utils.IsLinux() and options.cros_board: 435 if chromium_utils.IsLinux() and options.cros_board:
474 # When building ChromeOS's Simple Chrome workflow, the output directory 436 # When building ChromeOS's Simple Chrome workflow, the output directory
475 # has a CROS board name suffix. 437 # has a CROS board name suffix.
476 outdir = 'out_%s' % (options.cros_board,) 438 outdir = 'out_%s' % (options.cros_board,)
477 elif options.out_dir: 439 elif options.out_dir:
478 outdir = options.out_dir 440 outdir = options.out_dir
479 else: 441 else:
480 outdir = 'out' 442 outdir = 'out'
481 return os.path.abspath(os.path.join(options.src_dir, outdir, options.target)) 443 return os.path.abspath(os.path.join(options.src_dir, outdir, options.target))
(...skipping 58 matching lines...) Expand 10 before | Expand all | Expand 10 after
540 help='Checks the output of the ninja builder to ' 502 help='Checks the output of the ninja builder to '
541 'confirm that a second compile immediately ' 503 'confirm that a second compile immediately '
542 'the first is a no-op.') 504 'the first is a no-op.')
543 505
544 options, args = option_parser.parse_args() 506 options, args = option_parser.parse_args()
545 507
546 if not options.src_dir: 508 if not options.src_dir:
547 options.src_dir = 'src' 509 options.src_dir = 'src'
548 options.src_dir = os.path.abspath(options.src_dir) 510 options.src_dir = os.path.abspath(options.src_dir)
549 511
550 options.target_output_dir = get_target_build_dir(args, options) 512 options.target_output_dir = get_target_build_dir(options)
551 513
552 assert options.build_tool in (None, 'ninja') 514 assert options.build_tool in (None, 'ninja')
553 return options, args 515 return options, args
554 516
555 517
556 def real_main(): 518 def real_main():
557 options, args = get_parsed_options() 519 options, args = get_parsed_options()
558 520
559 # Prepare environment. 521 # Prepare environment.
560 env = EchoDict(os.environ) 522 env = EchoDict(os.environ)
(...skipping 11 matching lines...) Expand all
572 exit_status = main_ninja(options, args, env) 534 exit_status = main_ninja(options, args, env)
573 535
574 # stop goma 536 # stop goma
575 goma_teardown(options, env, exit_status, goma_cloudtail) 537 goma_teardown(options, env, exit_status, goma_cloudtail)
576 538
577 return exit_status 539 return exit_status
578 540
579 541
580 if '__main__' == __name__: 542 if '__main__' == __name__:
581 sys.exit(real_main()) 543 sys.exit(real_main())
OLDNEW
« no previous file with comments | « no previous file | no next file » | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698