Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(715)

Side by Side Diff: scripts/slave/recipe_modules/goma/api.py

Issue 2291273005: Add counting the number of CPU's in goma module (Closed)
Patch Set: add comment why I use python.inline Created 4 years, 3 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
« no previous file with comments | « no previous file | scripts/slave/recipe_modules/goma/example.py » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 # Copyright 2015 The Chromium Authors. All rights reserved. 1 # Copyright 2015 The Chromium Authors. All rights reserved.
2 # Use of this source code is governed by a BSD-style license that can be 2 # Use of this source code is governed by a BSD-style license that can be
3 # found in the LICENSE file. 3 # found in the LICENSE file.
4 4
5 from recipe_engine import recipe_api 5 from recipe_engine import recipe_api
6 6
7 class GomaApi(recipe_api.RecipeApi): 7 class GomaApi(recipe_api.RecipeApi):
8 """GomaApi contains helper functions for using goma.""" 8 """GomaApi contains helper functions for using goma."""
9 9
10 def __init__(self, **kwargs): 10 def __init__(self, **kwargs):
11 super(GomaApi, self).__init__(**kwargs) 11 super(GomaApi, self).__init__(**kwargs)
12 self._goma_dir = None 12 self._goma_dir = None
13 self._goma_started = False 13 self._goma_started = False
14 14
15 self._goma_ctl_env = {} 15 self._goma_ctl_env = {}
16 self._cloudtail_pid = None 16 self._cloudtail_pid = None
17 self._goma_jobs = None
17 18
18 @property 19 @property
19 def service_account_json_path(self): 20 def service_account_json_path(self):
20 if self.m.platform.is_win: 21 if self.m.platform.is_win:
21 return 'C:\\creds\\service_accounts\\service-account-goma-client.json' 22 return 'C:\\creds\\service_accounts\\service-account-goma-client.json'
22 return '/creds/service_accounts/service-account-goma-client.json' 23 return '/creds/service_accounts/service-account-goma-client.json'
23 24
24 @property 25 @property
25 def cloudtail_path(self): 26 def cloudtail_path(self):
26 assert self._goma_dir 27 assert self._goma_dir
27 return self.m.path.join(self._goma_dir, 'cloudtail') 28 return self.m.path.join(self._goma_dir, 'cloudtail')
28 29
29 @property 30 @property
30 def json_path(self): 31 def json_path(self):
31 assert self._goma_dir 32 assert self._goma_dir
32 return self.m.path.join(self._goma_dir, 'jsonstatus') 33 return self.m.path.join(self._goma_dir, 'jsonstatus')
33 34
35 @property
36 def recommended_goma_jobs(self):
37 """
38 Return the recommended number of jobs for parallel build using Goma.
39
40 This function caches the _goma_jobs.
41 """
42 if self._goma_jobs:
43 return self._goma_jobs
44
45 # In presubmit, a buildbot generates recipe json file and
46 # another buildbot checks generated recipe,
47 # so we need to use python.inline not to change
48 # behavior of recipes.
49 step_result = self.m.python.inline(
50 'calculate the number of recommended jobs',
51 """
52 import multiprocessing
53 import sys
54
55 job_limit = 200
56 if sys.platform.startswith('linux'):
57 # Use 80 for linux not to load goma backend.
58 job_limit = 80
59
60 try:
61 jobs = min(job_limit, multiprocessing.cpu_count() * 10)
62 except NotImplementedError:
63 jobs = 50
64
65 print jobs
66 """,
67 stdout=self.m.raw_io.output(),
68 step_test_data=(
69 lambda: self.m.raw_io.test_api.stream_output('50\n'))
70 )
71 self._goma_jobs = int(step_result.stdout)
72
73 return self._goma_jobs
74
34 def ensure_goma(self, canary=False): 75 def ensure_goma(self, canary=False):
35 with self.m.step.nest('ensure_goma'): 76 with self.m.step.nest('ensure_goma'):
36 with self.m.step.context({'infra_step': True}): 77 with self.m.step.context({'infra_step': True}):
37 try: 78 try:
38 self.m.cipd.set_service_account_credentials( 79 self.m.cipd.set_service_account_credentials(
39 self.service_account_json_path) 80 self.service_account_json_path)
40 81
41 self.m.cipd.install_client() 82 self.m.cipd.install_client()
42 goma_package = ('infra_internal/goma/client/%s' % 83 goma_package = ('infra_internal/goma/client/%s' %
43 self.m.cipd.platform_suffix()) 84 self.m.cipd.platform_suffix())
(...skipping 196 matching lines...) Expand 10 before | Expand all | Expand 10 after
240 ]) 281 ])
241 282
242 283
243 self.m.python( 284 self.m.python(
244 name=name or 'upload_log', 285 name=name or 'upload_log',
245 script=self.package_repo_resource( 286 script=self.package_repo_resource(
246 'scripts', 'slave', 'upload_goma_logs.py'), 287 'scripts', 'slave', 'upload_goma_logs.py'),
247 args=args, 288 args=args,
248 env=self._goma_ctl_env 289 env=self._goma_ctl_env
249 ) 290 )
OLDNEW
« no previous file with comments | « no previous file | scripts/slave/recipe_modules/goma/example.py » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698