Add counting the number of CPU's in goma module

scripts/slave/recipe_modules/goma/api.py

 # Copyright 2015 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 from recipe_engine import recipe_api
 
 class GomaApi(recipe_api.RecipeApi):
   """GomaApi contains helper functions for using goma."""
 
   def __init__(self, **kwargs):
     super(GomaApi, self).__init__(**kwargs)
     self._goma_dir = None
     self._goma_started = False
 
     self._goma_ctl_env = {}
     self._cloudtail_pid = None
+    self._goma_jobs = 50

[ukai, 2016/09/02 05:02:14]

can recipe_module user configure specific number for goma_jobs?

[tikuta, 2016/09/05 03:07:27]

User can set jobs by themselves where they run build.
I propose goma_jobs as recommended the number of jobs.

 
   @property
   def service_account_json_path(self):
     if self.m.platform.is_win:
       return 'C:\\creds\\service_accounts\\service-account-goma-client.json'
     return '/creds/service_accounts/service-account-goma-client.json'
 
   @property
   def cloudtail_path(self):
     assert self._goma_dir
     return self.m.path.join(self._goma_dir, 'cloudtail')
 
   @property
   def json_path(self):
     assert self._goma_dir
     return self.m.path.join(self._goma_dir, 'jsonstatus')
 
+  @property
+  def goma_jobs(self):
+    return self._goma_jobs
+
   def ensure_goma(self, canary=False):
     with self.m.step.nest('ensure_goma'):
       with self.m.step.context({'infra_step': True}):
         try:
           self.m.cipd.set_service_account_credentials(
               self.service_account_json_path)
 
           self.m.cipd.install_client()
           goma_package = ('infra_internal/goma/client/%s' %
               self.m.cipd.platform_suffix())
@@ skipping 14 matching lines @@
           # ./cipd describe infra/tools/cloudtail/linux-amd64 \
           #     -version goma_recipe_module
           cloudtail_package = (
               'infra/tools/cloudtail/%s' % self.m.cipd.platform_suffix())
           cloudtail_version = 'goma_recipe_module'
 
           self.m.cipd.ensure(self._goma_dir,
                              {goma_package: ref,
                               cloudtail_package: cloudtail_version})
 
+          step_result = self.m.python.inline(
+            'get the number of cpus',
+            """
+            import multiprocessing
+            print min(200, multiprocessing.cpu_count() * 10)

[shinyak, 2016/09/01 09:38:28]

I didn't set 200 for all platforms? Only Mac & Windows.
Not sure backend resource is enough. (See compile.py comment)

[ukai, 2016/09/02 05:02:14]

https://docs.python.org/2.7/library/multiprocessing.html#multiprocessing.cpu_...

may raise NotImplementedError?

[tikuta, 2016/09/05 03:01:37]

Done.

[tikuta, 2016/09/05 03:01:37]

For linux, I set 80 for the number of jobs.

+            """,
+            stdout=self.m.raw_io.output(),
+            step_test_data=(
+                lambda: self.m.raw_io.test_api.stream_output('40'))
+          )
+          self._goma_jobs = int(step_result.stdout)
+
           return self._goma_dir
         except self.m.step.StepFailure:
           # TODO(phajdan.jr): make failures fatal after experiment.
           return None
 
   @property
   def goma_ctl(self):
     return self.m.path.join(self._goma_dir, 'goma_ctl.py')
 
   @property
@@ skipping 159 matching lines @@
         ])
 
 
     self.m.python(
       name=name or 'upload_log',
       script=self.package_repo_resource(
           'scripts', 'slave', 'upload_goma_logs.py'),
       args=args,
       env=self._goma_ctl_env
     )