Implement upload_logs() in recipe_modules/goma

scripts/slave/recipe_modules/goma/api.py

 # Copyright 2015 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 from recipe_engine import recipe_api
 
 class GomaApi(recipe_api.RecipeApi):
   """GomaApi contains helper functions for using goma."""
 
   def __init__(self, **kwargs):
     super(GomaApi, self).__init__(**kwargs)
     self._goma_dir = None
     self._goma_started = False
 
+    self._env = {}

[Yoshisato Yanagisawa, 2016/08/09 04:01:49]

might be better to say goma_ctl_env or so?

[tikuta, 2016/08/09 05:30:39]

Done.

+
+    # options for upload_goma_logs.py
+    # TODO(tikuta) set these variables
+    self._ninja_log_outdir = None
+    self._ninja_log_compiler = None
+    self._ninja_log_command = None
+    self._ninja_log_exit_status = None
+
+    self._build_data_dir = None
+
+
   @property
   def service_account_json_path(self):
     if self.m.platform.is_win:
       return 'C:\\creds\\service_accounts\\service-account-goma-client.json'
     return '/creds/service_accounts/service-account-goma-client.json'
 
   def ensure_goma(self, canary=False):
     with self.m.step.nest('ensure_goma'):
       with self.m.step.context({'infra_step': True}):
         try:
           self.m.cipd.set_service_account_credentials(
               self.service_account_json_path)
 
           self.m.cipd.install_client()
           goma_package = ('infra_internal/goma/client/%s' %
               self.m.cipd.platform_suffix())
           # For Windows there's only 64-bit goma client.
           if self.m.platform.is_win:
             goma_package = goma_package.replace('386', 'amd64')
           ref='release'
           if canary:
             ref='candidate'
           self._goma_dir = self.m.path['cache'].join('cipd', 'goma')
           self.m.cipd.ensure(self._goma_dir, {goma_package: ref})
           return self._goma_dir
         except self.m.step.StepFailure:
           # TODO(phajdan.jr): make failures fatal after experiment.
           return None
 
-  def start(self, env=None, **kwargs):
+  @property
+  def goma_ctl(self):
+    return self.m.path.join(self._goma_dir, 'goma_ctl.py')
+
+  def start(self, build_data_dir, env=None, **kwargs):

[Yoshisato Yanagisawa, 2016/08/09 04:01:49]

might be better to make build_data_dir optional?
or, always enable this if m.properties have it? (disables it if properties does
not have it?)

[tikuta, 2016/08/09 05:30:40]

Done.

     """Start goma compiler_proxy.
 
     A user MUST execute ensure_goma beforehand.
     It is user's responsibility to handle failure of starting compiler_proxy.
     """
     assert self._goma_dir
     assert not self._goma_started
-    if not env:
-      env = {}
-    env.update({'GOMA_SERVICE_ACCOUNT_JSON_FILE': self.service_account_json_path})
+
+    self._build_data_dir = build_data_dir
+    self._env['GOMA_DUMP_STATS_FILE'] = (
+        self.m.path.join(build_data_dir, 'goma_stats_proto'))
+    self._env['GOMACTL_CRASH_REPORT_ID_FILE'] = (
+        self.m.path.join(build_data_dir, 'crash_report_id_file'))
+
+    self._env['GOMA_SERVICE_ACCOUNT_JSON_FILE'] = self.service_account_json_path
+
+    if env:
+      self._env.update(env)
+
     self.m.python(
         name='start_goma',
-        script=self.m.path.join(self._goma_dir, 'goma_ctl.py'),
-        args=['restart'], env=env, **kwargs)
+        script=self.goma_ctl,
+        args=['restart'], env=self._env, **kwargs)

[Yoshisato Yanagisawa, 2016/08/09 04:01:49]

can you upload stats on goma_start failure?

[tikuta, 2016/08/09 05:30:39]

Done.

     self._goma_started = True
 
-  def stop(self, goma_dir=None, **kwargs):
+  def stop(self, **kwargs):
     """Stop goma compiler_proxy.
 
     A user MUST execute start beforehand.
     It is user's responsibility to handle failure of stopping compiler_proxy.
     """
     assert self._goma_dir
     assert self._goma_started
     self.m.python(
         name='stop_goma',
-        script=self.m.path.join(self._goma_dir, 'goma_ctl.py'),
-        args=['stop'], **kwargs)
+        script=self.goma_ctl,
+        args=['stop'], env=self._env, **kwargs)
     self._goma_started = False

[Yoshisato Yanagisawa, 2016/08/09 04:01:49]

should clear self._goma_ctl_env?

[tikuta, 2016/08/09 05:30:39]

Done.

+    self.upload_logs()
+
+  def upload_logs(self):
+    args = [
+        '--upload-compiler-proxy-info'
+    ]
+
+    if self._ninja_log_outdir:
+      args.extend([
+        '--ninja-log-outdir', self._ninja_log_outdir,
+        '--ninja-log-compiler', self._ninja_log_compiler,
+        '--ninja-log-command', self._ninja_log_command,
+        '--ninja-log-exit-status', self._ninja_log_exit_status,
+      ])
+
+    args.extend([
+        '--goma-stats-file', self._env['GOMA_DUMP_STATS_FILE'],
+        '--goma-crash-report-id-file',
+        self._env['GOMACTL_CRASH_REPORT_ID_FILE'],
+        '--build-data-dir', self._build_data_dir,
+    ])
+
+    self.m.python(
+      name='upload_log',
+      script='upload_goma_logs.py',
+      args=args
+    )