| OLD | NEW |
| 1 # Copyright 2015 The Chromium Authors. All rights reserved. | 1 # Copyright 2015 The Chromium Authors. All rights reserved. |
| 2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
| 3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
| 4 | 4 |
| 5 import re | 5 import re |
| 6 from contextlib import contextmanager | |
| 7 | 6 |
| 8 from recipe_engine import recipe_api | 7 from recipe_engine import recipe_api |
| 9 | 8 |
| 10 class GomaApi(recipe_api.RecipeApi): | 9 class GomaApi(recipe_api.RecipeApi): |
| 11 """GomaApi contains helper functions for using goma.""" | 10 """GomaApi contains helper functions for using goma.""" |
| 12 | 11 |
| 13 def __init__(self, **kwargs): | 12 def __init__(self, **kwargs): |
| 14 super(GomaApi, self).__init__(**kwargs) | 13 super(GomaApi, self).__init__(**kwargs) |
| 15 self._goma_dir = None | 14 self._goma_dir = None |
| 16 self._goma_started = False | 15 self._goma_started = False |
| (...skipping 70 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 87 | 86 |
| 88 return self._goma_jobs | 87 return self._goma_jobs |
| 89 | 88 |
| 90 def remove_j_flag(self, command): | 89 def remove_j_flag(self, command): |
| 91 """Remove -j flag from command. | 90 """Remove -j flag from command. |
| 92 This function elimitnates -j flag from command for | 91 This function elimitnates -j flag from command for |
| 93 '-j', '80' style and '-j80' style. | 92 '-j', '80' style and '-j80' style. |
| 94 | 93 |
| 95 Args: | 94 Args: |
| 96 command: List of command line arg. | 95 command: List of command line arg. |
| 96 |
| 97 Returns: |
| 98 list(string): Command line args parallel option removed. |
| 97 """ | 99 """ |
| 100 command = command[:] |
| 98 parallel_flag_regexp = re.compile('-j\d*') | 101 parallel_flag_regexp = re.compile('-j\d*') |
| 99 for i in range(len(command)): | 102 for i in range(len(command)): |
| 100 if (isinstance(command[i], str) and | 103 if (isinstance(command[i], str) and |
| 101 parallel_flag_regexp.match(command[i])): | 104 parallel_flag_regexp.match(command[i])): |
| 102 if command[i] == '-j': | 105 if command[i] == '-j': |
| 103 command.pop(i + 1) | 106 command.pop(i + 1) |
| 104 command.pop(i) | 107 command.pop(i) |
| 105 return | 108 break |
| 109 return command |
| 106 | 110 |
| 107 def ensure_goma(self, canary=False): | 111 def ensure_goma(self, canary=False): |
| 108 with self.m.step.nest('ensure_goma'): | 112 with self.m.step.nest('ensure_goma'): |
| 109 with self.m.step.context({'infra_step': True}): | 113 with self.m.step.context({'infra_step': True}): |
| 110 try: | 114 try: |
| 111 self.m.cipd.set_service_account_credentials( | 115 self.m.cipd.set_service_account_credentials( |
| 112 self.service_account_json_path) | 116 self.service_account_json_path) |
| 113 | 117 |
| 114 self.m.cipd.install_client() | 118 self.m.cipd.install_client() |
| 115 goma_package = ('infra_internal/goma/client/%s' % | 119 goma_package = ('infra_internal/goma/client/%s' % |
| (...skipping 204 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 320 | 324 |
| 321 | 325 |
| 322 self.m.python( | 326 self.m.python( |
| 323 name=name or 'upload_log', | 327 name=name or 'upload_log', |
| 324 script=self.package_repo_resource( | 328 script=self.package_repo_resource( |
| 325 'scripts', 'slave', 'upload_goma_logs.py'), | 329 'scripts', 'slave', 'upload_goma_logs.py'), |
| 326 args=args, | 330 args=args, |
| 327 env=self._goma_ctl_env | 331 env=self._goma_ctl_env |
| 328 ) | 332 ) |
| 329 | 333 |
| 330 @contextmanager | 334 def build_with_goma(self, ninja_command, name=None, ninja_log_outdir=None, |
| 331 def build_with_goma(self, ninja_log_outdir=None, ninja_log_compiler=None, | 335 ninja_log_compiler=None, goma_env=None, ninja_env=None, |
| 332 ninja_log_command=None, env=None, | 336 allow_build_without_goma=False, **kwargs): |
| 333 allow_build_without_goma=False): | 337 """Build with ninja_command using goma |
| 334 """Make context wrapping goma start/stop. | |
| 335 | |
| 336 Args ninja_log_* are NOT used to run actual build in this context. | |
| 337 These args are only used to collect log and upload them. | |
| 338 | 338 |
| 339 Args: | 339 Args: |
| 340 ninja_command: Command used for build. |
| 341 This is sent as part of log. |
| 342 (e.g. ['ninja', '-C', 'out/Release']) |
| 343 name: Name of compile step. |
| 340 ninja_log_outdir: Directory of ninja log. (e.g. "out/Release") | 344 ninja_log_outdir: Directory of ninja log. (e.g. "out/Release") |
| 341 ninja_log_compiler: Compiler used in ninja. (e.g. "clang") | 345 ninja_log_compiler: Compiler used in ninja. (e.g. "clang") |
| 342 ninja_log_command: Command used for build. | 346 goma_env: Environment controlling goma behavior. |
| 343 This is used only to be sent as part of log, | 347 ninja_env: Environment for ninja. |
| 344 NOT used to run actual build. | |
| 345 (e.g. ['ninja', '-C', 'out/Release', '-j', '100']) | |
| 346 env: Environment controlling goma behavior. | |
| 347 This should be used for ninja if allow_build_without_goma is True. | |
| 348 Otherwise, only used for goma behavior control. | |
| 349 allow_build_without_goma (bool): | 348 allow_build_without_goma (bool): |
| 350 If this is not True, goma starting failure is ignored. | 349 If this is True, goma starting failure is ignored and |
| 351 This argument should be used with env. | 350 build without goma. |
| 352 If starting goma fails, env and ninja_log_command are modified. | 351 |
| 352 Returns: |
| 353 TODO(tikuta): return step_result |
| 353 | 354 |
| 354 Raises: | 355 Raises: |
| 355 StepFailure or InfraFailure if it fails to build. | 356 StepFailure or InfraFailure if it fails to build or |
| 357 occurs something failure on goma steps. |
| 356 """ | 358 """ |
| 357 ninja_log_exit_status = 0 | 359 ninja_log_exit_status = 0 |
| 358 | 360 |
| 361 if ninja_env is None: |
| 362 ninja_env = {} |
| 363 if goma_env is None: |
| 364 goma_env = {} |
| 365 |
| 366 # TODO(tikuta): Remove -j flag from ninja_command and set appropriate value. |
| 367 |
| 359 if allow_build_without_goma: | 368 if allow_build_without_goma: |
| 360 assert(env is not None) | 369 assert ninja_log_outdir is not None |
| 370 |
| 361 try: | 371 try: |
| 362 self.start(env) | 372 self.start(goma_env) |
| 363 except: | 373 except: |
| 364 env['GOMA_DISABLED'] = 'true' | |
| 365 self.remove_j_flag(ninja_log_command) | |
| 366 try: | 374 try: |
| 367 yield | 375 if self.m.platform.is_win: |
| 376 goma_env = goma_env.copy() |
| 377 goma_env['GOMA_DISABLED'] = 'true' |
| 378 self.m.python('update windows env', |
| 379 script=self.package_repo_resource( |
| 380 'scripts', 'slave', 'update_windows_env.py'), |
| 381 args=['--envfile-dir', str(ninja_log_outdir)], |
| 382 env=goma_env) |
| 383 ninja_env = ninja_env.copy() |
| 384 ninja_env['GOMA_DISABLED'] = 'true' |
| 385 ninja_command = self.remove_j_flag(ninja_command) |
| 386 self.m.step(name or 'compile', ninja_command, |
| 387 env=ninja_env, **kwargs) |
| 388 |
| 368 except self.m.step.StepFailure as e: # pragma: no cover | 389 except self.m.step.StepFailure as e: # pragma: no cover |
| 369 ninja_log_exit_status = e.retcode | 390 ninja_log_exit_status = e.retcode |
| 370 raise e | 391 raise e |
| 371 except self.m.step.InfraFailure as e: # pragma: no cover | 392 except self.m.step.InfraFailure as e: # pragma: no cover |
| 372 ninja_log_exit_status = -1 | 393 ninja_log_exit_status = -1 |
| 373 raise e | 394 raise e |
| 374 finally: | 395 finally: |
| 375 self._upload_logs(ninja_log_outdir=ninja_log_outdir, | 396 self._upload_logs(ninja_log_outdir=ninja_log_outdir, |
| 376 ninja_log_compiler=ninja_log_compiler, | 397 ninja_log_compiler=ninja_log_compiler, |
| 377 ninja_log_command=ninja_log_command, | 398 ninja_log_command=ninja_command, |
| 378 ninja_log_exit_status=ninja_log_exit_status, | 399 ninja_log_exit_status=ninja_log_exit_status, |
| 379 name='upload log goma start failed') | 400 name='upload log goma start failed') |
| 380 return | 401 return |
| 381 else: | 402 else: |
| 382 self.start(env) | 403 self.start(goma_env) |
| 383 | 404 |
| 384 try: | 405 try: |
| 385 yield | 406 self.m.step(name or 'compile', ninja_command, |
| 407 env=ninja_env, **kwargs) |
| 408 |
| 386 except self.m.step.StepFailure as e: # pragma: no cover | 409 except self.m.step.StepFailure as e: # pragma: no cover |
| 387 ninja_log_exit_status = e.retcode | 410 ninja_log_exit_status = e.retcode |
| 388 raise e | 411 raise e |
| 389 except self.m.step.InfraFailure as e: # pragma: no cover | 412 except self.m.step.InfraFailure as e: # pragma: no cover |
| 390 ninja_log_exit_status = -1 | 413 ninja_log_exit_status = -1 |
| 391 raise e | 414 raise e |
| 392 finally: | 415 finally: |
| 393 self.stop(ninja_log_outdir=ninja_log_outdir, | 416 self.stop(ninja_log_outdir=ninja_log_outdir, |
| 394 ninja_log_compiler=ninja_log_compiler, | 417 ninja_log_compiler=ninja_log_compiler, |
| 395 ninja_log_command=ninja_log_command, | 418 ninja_log_command=ninja_command, |
| 396 ninja_log_exit_status=ninja_log_exit_status) | 419 ninja_log_exit_status=ninja_log_exit_status) |
| OLD | NEW |