| OLD | NEW |
| 1 # Copyright 2015 The Chromium Authors. All rights reserved. | 1 # Copyright 2015 The Chromium Authors. All rights reserved. |
| 2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
| 3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
| 4 | 4 |
| 5 from recipe_engine import recipe_api | 5 from recipe_engine import recipe_api |
| 6 | 6 |
| 7 class GomaApi(recipe_api.RecipeApi): | 7 class GomaApi(recipe_api.RecipeApi): |
| 8 """GomaApi contains helper functions for using goma.""" | 8 """GomaApi contains helper functions for using goma.""" |
| 9 | 9 |
| 10 def __init__(self, **kwargs): | 10 def __init__(self, **kwargs): |
| 11 super(GomaApi, self).__init__(**kwargs) | 11 super(GomaApi, self).__init__(**kwargs) |
| 12 self._goma_dir = None | 12 self._goma_dir = None |
| 13 self._goma_started = False | 13 self._goma_started = False |
| 14 | 14 |
| 15 self._goma_ctl_env = {} | 15 self._goma_ctl_env = {} |
| 16 self._cloudtail_pid = None | 16 self._cloudtail_pid = None |
| 17 self._goma_jobs = None |
| 17 | 18 |
| 18 @property | 19 @property |
| 19 def service_account_json_path(self): | 20 def service_account_json_path(self): |
| 20 if self.m.platform.is_win: | 21 if self.m.platform.is_win: |
| 21 return 'C:\\creds\\service_accounts\\service-account-goma-client.json' | 22 return 'C:\\creds\\service_accounts\\service-account-goma-client.json' |
| 22 return '/creds/service_accounts/service-account-goma-client.json' | 23 return '/creds/service_accounts/service-account-goma-client.json' |
| 23 | 24 |
| 24 @property | 25 @property |
| 25 def cloudtail_path(self): | 26 def cloudtail_path(self): |
| 26 assert self._goma_dir | 27 assert self._goma_dir |
| 27 return self.m.path.join(self._goma_dir, 'cloudtail') | 28 return self.m.path.join(self._goma_dir, 'cloudtail') |
| 28 | 29 |
| 29 @property | 30 @property |
| 30 def json_path(self): | 31 def json_path(self): |
| 31 assert self._goma_dir | 32 assert self._goma_dir |
| 32 return self.m.path.join(self._goma_dir, 'jsonstatus') | 33 return self.m.path.join(self._goma_dir, 'jsonstatus') |
| 33 | 34 |
| 35 @property |
| 36 def recommended_goma_jobs(self): |
| 37 """ |
| 38 Return the recommended number of jobs for parallel build using Goma. |
| 39 |
| 40 This function caches the _goma_jobs. |
| 41 """ |
| 42 if self._goma_jobs: |
| 43 return self._goma_jobs |
| 44 |
| 45 # In presubmit, a buildbot generates recipe json file and |
| 46 # another buildbot checks generated recipe, |
| 47 # so we need to use python.inline not to change |
| 48 # behavior of recipes. |
| 49 step_result = self.m.python.inline( |
| 50 'calculate the number of recommended jobs', |
| 51 """ |
| 52 import multiprocessing |
| 53 import sys |
| 54 |
| 55 job_limit = 200 |
| 56 if sys.platform.startswith('linux'): |
| 57 # Use 80 for linux not to load goma backend. |
| 58 job_limit = 80 |
| 59 |
| 60 try: |
| 61 jobs = min(job_limit, multiprocessing.cpu_count() * 10) |
| 62 except NotImplementedError: |
| 63 jobs = 50 |
| 64 |
| 65 print jobs |
| 66 """, |
| 67 stdout=self.m.raw_io.output(), |
| 68 step_test_data=( |
| 69 lambda: self.m.raw_io.test_api.stream_output('50\n')) |
| 70 ) |
| 71 self._goma_jobs = int(step_result.stdout) |
| 72 |
| 73 return self._goma_jobs |
| 74 |
| 34 def ensure_goma(self, canary=False): | 75 def ensure_goma(self, canary=False): |
| 35 with self.m.step.nest('ensure_goma'): | 76 with self.m.step.nest('ensure_goma'): |
| 36 with self.m.step.context({'infra_step': True}): | 77 with self.m.step.context({'infra_step': True}): |
| 37 try: | 78 try: |
| 38 self.m.cipd.set_service_account_credentials( | 79 self.m.cipd.set_service_account_credentials( |
| 39 self.service_account_json_path) | 80 self.service_account_json_path) |
| 40 | 81 |
| 41 self.m.cipd.install_client() | 82 self.m.cipd.install_client() |
| 42 goma_package = ('infra_internal/goma/client/%s' % | 83 goma_package = ('infra_internal/goma/client/%s' % |
| 43 self.m.cipd.platform_suffix()) | 84 self.m.cipd.platform_suffix()) |
| (...skipping 196 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 240 ]) | 281 ]) |
| 241 | 282 |
| 242 | 283 |
| 243 self.m.python( | 284 self.m.python( |
| 244 name=name or 'upload_log', | 285 name=name or 'upload_log', |
| 245 script=self.package_repo_resource( | 286 script=self.package_repo_resource( |
| 246 'scripts', 'slave', 'upload_goma_logs.py'), | 287 'scripts', 'slave', 'upload_goma_logs.py'), |
| 247 args=args, | 288 args=args, |
| 248 env=self._goma_ctl_env | 289 env=self._goma_ctl_env |
| 249 ) | 290 ) |
| OLD | NEW |