OLD | NEW |
---|---|
1 # Copyright 2015 The Chromium Authors. All rights reserved. | 1 # Copyright 2015 The Chromium Authors. All rights reserved. |
2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
4 | 4 |
5 from recipe_engine import recipe_api | 5 from recipe_engine import recipe_api |
6 | 6 |
7 class GomaApi(recipe_api.RecipeApi): | 7 class GomaApi(recipe_api.RecipeApi): |
8 """GomaApi contains helper functions for using goma.""" | 8 """GomaApi contains helper functions for using goma.""" |
9 | 9 |
10 def __init__(self, **kwargs): | 10 def __init__(self, **kwargs): |
11 super(GomaApi, self).__init__(**kwargs) | 11 super(GomaApi, self).__init__(**kwargs) |
12 self._goma_dir = None | 12 self._goma_dir = None |
13 self._goma_started = False | 13 self._goma_started = False |
14 | 14 |
15 self._goma_ctl_env = {} | 15 self._goma_ctl_env = {} |
16 self._cloudtail_pid = None | 16 self._cloudtail_pid = None |
17 self._goma_jobs = 50 | |
ukai
2016/09/02 05:02:14
can recipe_module user configure specific number f
tikuta
2016/09/05 03:07:27
User can set jobs by themselves where they run bui
| |
17 | 18 |
18 @property | 19 @property |
19 def service_account_json_path(self): | 20 def service_account_json_path(self): |
20 if self.m.platform.is_win: | 21 if self.m.platform.is_win: |
21 return 'C:\\creds\\service_accounts\\service-account-goma-client.json' | 22 return 'C:\\creds\\service_accounts\\service-account-goma-client.json' |
22 return '/creds/service_accounts/service-account-goma-client.json' | 23 return '/creds/service_accounts/service-account-goma-client.json' |
23 | 24 |
24 @property | 25 @property |
25 def cloudtail_path(self): | 26 def cloudtail_path(self): |
26 assert self._goma_dir | 27 assert self._goma_dir |
27 return self.m.path.join(self._goma_dir, 'cloudtail') | 28 return self.m.path.join(self._goma_dir, 'cloudtail') |
28 | 29 |
29 @property | 30 @property |
30 def json_path(self): | 31 def json_path(self): |
31 assert self._goma_dir | 32 assert self._goma_dir |
32 return self.m.path.join(self._goma_dir, 'jsonstatus') | 33 return self.m.path.join(self._goma_dir, 'jsonstatus') |
33 | 34 |
35 @property | |
36 def goma_jobs(self): | |
37 return self._goma_jobs | |
38 | |
34 def ensure_goma(self, canary=False): | 39 def ensure_goma(self, canary=False): |
35 with self.m.step.nest('ensure_goma'): | 40 with self.m.step.nest('ensure_goma'): |
36 with self.m.step.context({'infra_step': True}): | 41 with self.m.step.context({'infra_step': True}): |
37 try: | 42 try: |
38 self.m.cipd.set_service_account_credentials( | 43 self.m.cipd.set_service_account_credentials( |
39 self.service_account_json_path) | 44 self.service_account_json_path) |
40 | 45 |
41 self.m.cipd.install_client() | 46 self.m.cipd.install_client() |
42 goma_package = ('infra_internal/goma/client/%s' % | 47 goma_package = ('infra_internal/goma/client/%s' % |
43 self.m.cipd.platform_suffix()) | 48 self.m.cipd.platform_suffix()) |
(...skipping 14 matching lines...) Expand all Loading... | |
58 # ./cipd describe infra/tools/cloudtail/linux-amd64 \ | 63 # ./cipd describe infra/tools/cloudtail/linux-amd64 \ |
59 # -version goma_recipe_module | 64 # -version goma_recipe_module |
60 cloudtail_package = ( | 65 cloudtail_package = ( |
61 'infra/tools/cloudtail/%s' % self.m.cipd.platform_suffix()) | 66 'infra/tools/cloudtail/%s' % self.m.cipd.platform_suffix()) |
62 cloudtail_version = 'goma_recipe_module' | 67 cloudtail_version = 'goma_recipe_module' |
63 | 68 |
64 self.m.cipd.ensure(self._goma_dir, | 69 self.m.cipd.ensure(self._goma_dir, |
65 {goma_package: ref, | 70 {goma_package: ref, |
66 cloudtail_package: cloudtail_version}) | 71 cloudtail_package: cloudtail_version}) |
67 | 72 |
73 step_result = self.m.python.inline( | |
74 'get the number of cpus', | |
75 """ | |
76 import multiprocessing | |
77 print min(200, multiprocessing.cpu_count() * 10) | |
shinyak
2016/09/01 09:38:28
I didn't set 200 for all platforms? Only Mac & Win
ukai
2016/09/02 05:02:14
https://docs.python.org/2.7/library/multiprocessin
tikuta
2016/09/05 03:01:37
Done.
tikuta
2016/09/05 03:01:37
For linux, I set 80 for the number of jobs.
| |
78 """, | |
79 stdout=self.m.raw_io.output(), | |
80 step_test_data=( | |
81 lambda: self.m.raw_io.test_api.stream_output('40')) | |
82 ) | |
83 self._goma_jobs = int(step_result.stdout) | |
84 | |
68 return self._goma_dir | 85 return self._goma_dir |
69 except self.m.step.StepFailure: | 86 except self.m.step.StepFailure: |
70 # TODO(phajdan.jr): make failures fatal after experiment. | 87 # TODO(phajdan.jr): make failures fatal after experiment. |
71 return None | 88 return None |
72 | 89 |
73 @property | 90 @property |
74 def goma_ctl(self): | 91 def goma_ctl(self): |
75 return self.m.path.join(self._goma_dir, 'goma_ctl.py') | 92 return self.m.path.join(self._goma_dir, 'goma_ctl.py') |
76 | 93 |
77 @property | 94 @property |
(...skipping 159 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
237 ]) | 254 ]) |
238 | 255 |
239 | 256 |
240 self.m.python( | 257 self.m.python( |
241 name=name or 'upload_log', | 258 name=name or 'upload_log', |
242 script=self.package_repo_resource( | 259 script=self.package_repo_resource( |
243 'scripts', 'slave', 'upload_goma_logs.py'), | 260 'scripts', 'slave', 'upload_goma_logs.py'), |
244 args=args, | 261 args=args, |
245 env=self._goma_ctl_env | 262 env=self._goma_ctl_env |
246 ) | 263 ) |
OLD | NEW |