Add buildbot metadata to compiler_proxy log

scripts/slave/goma_utils.py

 # Copyright (c) 2015 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Functions specific to handle goma related info.
 """
 
 import base64
 import datetime
 import getpass
@@ skipping 60 matching lines @@
 def GetLatestGomaCompilerProxyInfo():
   """Get a filename of the latest goma comiler_proxy.INFO."""
   return GetLatestGlogInfoFile('compiler_proxy')
 
 
 def GetLatestGomaCompilerProxySubprocInfo():
   """Get a filename of the latest goma comiler_proxy-subproc.INFO."""
   return GetLatestGlogInfoFile('compiler_proxy-subproc')
 
 
-def UploadToGomaLogGS(
-    file_path, gs_filename, text_to_append=None, override_gsutil=None):
+def UploadToGomaLogGS(file_path, gs_filename,
+                      text_to_append=None,
+                      metadata=None,
+                      override_gsutil=None):
   """Upload a file to Google Cloud Storage (gs://chrome-goma-log).
 
   Note that the uploaded file would automatically be gzip compressed.
 
   Args:
     file_path: a path of a file to be uploaded.
     gs_filename: a name of a file in Google Storage.
+    metadata: (dict) A dictionary of string key/value metadata entries.
     text_to_append: an addtional text to be added to a file in GS.
 
   Returns:
     a stored path name without the bucket name in GS.
   """
   hostname = GetShortHostname()
   today = datetime.datetime.utcnow().date()
   log_path = '%s/%s/%s.gz' % (
     today.strftime('%Y/%m/%d'), hostname, gs_filename)
   gs_path = 'gs://%s/%s' % (GOMA_LOG_GS_BUCKET, log_path)
   temp = tempfile.NamedTemporaryFile(delete=False)
   try:
     with temp as f_out:
       with gzip.GzipFile(fileobj=f_out) as gzipf_out:
         with open(file_path) as f_in:
           shutil.copyfileobj(f_in, gzipf_out)
         if text_to_append:
           gzipf_out.write(text_to_append)
-    slave_utils.GSUtilCopy(temp.name, gs_path, override_gsutil=override_gsutil)
+    slave_utils.GSUtilCopy(temp.name, gs_path,
+                           metadata=metadata, override_gsutil=override_gsutil)
     print "Copied log file to %s" % gs_path
   finally:
     os.remove(temp.name)
   return log_path
 
 
-def UploadGomaCompilerProxyInfo(override_gsutil=None):
+def UploadGomaCompilerProxyInfo(override_gsutil=None, builder='unknown', master='unknown',
+                                slave='unknown', clobber=''):
   """Upload goma compiler_proxy.INFO to Google Storage."""
   latest_subproc_info = GetLatestGomaCompilerProxySubprocInfo()
+
+  metadata = {

[Yoshisato Yanagisawa, 2016/09/05 08:01:39]

I thought you encode metadata with gzip+base64 and subscriber just read one
field and decode, but maybe ok with this.

[shinyak, 2016/09/06 02:30:33]

Hmm, I'm OK with gzip. these name could be too general for metadata?

[shinyak, 2016/09/06 02:36:36]

Since this is small, I feel it's ok to save as json (without gzip), though...

[Yoshisato Yanagisawa, 2016/09/06 02:50:50]

Other reviewers may think the other things but I feel metadata keys a bit
confusing because there is no hint in keys to mention who set them, or for
guessing the purpose of them.

I am afraid something bad happens with Windows command line but no need to
compress or base64 encode. (maybe not encoding help us to read the config
easily)

[shinyak, 2016/09/06 02:59:22]

Since this runs with python subprocess.Popen (finally), it should be ok (I
believe.)

+    'builder': builder,
+    'master': master,
+    'slave': slave,
+    'clobber': '1' if clobber else '0',
+    'os': chromium_utils.PlatformName(),
+  }
+
   if latest_subproc_info:
     UploadToGomaLogGS(latest_subproc_info,
                       os.path.basename(latest_subproc_info),
+                      metadata=metadata,
                       override_gsutil=override_gsutil)
   else:
     print 'No compiler_proxy-subproc.INFO to upload'
   latest_info = GetLatestGomaCompilerProxyInfo()
   if not latest_info:
     print 'No compiler_proxy.INFO to upload'
     return
   # Since a filename of compiler_proxy.INFO is fairly unique,
   # we might be able to upload it as-is.
   log_path = UploadToGomaLogGS(
       latest_info, os.path.basename(latest_info),
+      metadata=metadata,
       override_gsutil=override_gsutil)
   viewer_url = ('http://chromium-build-stats.appspot.com/compiler_proxy_log/'
                 + log_path)
   print 'Visualization at %s' % viewer_url
 
 
 def UploadNinjaLog(
     outdir, compiler, command, exit_status, override_gsutil=None):
   """Upload .ninja_log to Google Cloud Storage (gs://chrome-goma-log),
   in the same folder with goma's compiler_proxy.INFO.
@@ skipping 29 matching lines @@
   if compiler_proxy_info:
     info['compiler_proxy_info'] = compiler_proxy_info
 
   username = getpass.getuser()
   hostname = GetShortHostname()
   pid = os.getpid()
   ninja_log_filename = 'ninja_log.%s.%s.%s.%d' % (
       hostname, username, mtime.strftime('%Y%m%d-%H%M%S'), pid)
   additional_text = '# end of ninja log\n' + json.dumps(info)
   log_path = UploadToGomaLogGS(
-      ninja_log_path, ninja_log_filename, additional_text,
+      ninja_log_path, ninja_log_filename, text_to_append=additional_text,
       override_gsutil=override_gsutil)
   viewer_url = 'http://chromium-build-stats.appspot.com/ninja_log/' + log_path
   print 'Visualization at %s' % viewer_url
 
 
 def IsCompilerProxyKilledByFatalError():
   """Returns true if goma compiler_proxy is killed by CHECK or LOG(FATAL)."""
   info_file = GetLatestGomaCompilerProxyInfo()
   if not info_file:
     return False
@@ skipping 154 matching lines @@
     retcode = chromium_utils.RunCommand(
       cmd, filter_obj=cmd_filter,
       max_time=30)
     if retcode:
       print('Execution of send_ts_mon_values failed with code %s'
             % retcode)
       print '\n'.join(cmd_filter.text)
 
   except Exception as ex:
     print('error while sending ts mon json_file=%s: %s' % (json_file, ex))