| OLD | NEW |
| 1 # Copyright (c) 2015 The Chromium Authors. All rights reserved. | 1 # Copyright (c) 2015 The Chromium Authors. All rights reserved. |
| 2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
| 3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
| 4 | 4 |
| 5 """Functions specific to handle goma related info. | 5 """Functions specific to handle goma related info. |
| 6 """ | 6 """ |
| 7 | 7 |
| 8 import base64 | 8 import base64 |
| 9 import datetime | 9 import datetime |
| 10 import getpass | 10 import getpass |
| (...skipping 60 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 71 def GetLatestGomaCompilerProxyInfo(): | 71 def GetLatestGomaCompilerProxyInfo(): |
| 72 """Get a filename of the latest goma comiler_proxy.INFO.""" | 72 """Get a filename of the latest goma comiler_proxy.INFO.""" |
| 73 return GetLatestGlogInfoFile('compiler_proxy') | 73 return GetLatestGlogInfoFile('compiler_proxy') |
| 74 | 74 |
| 75 | 75 |
| 76 def GetLatestGomaCompilerProxySubprocInfo(): | 76 def GetLatestGomaCompilerProxySubprocInfo(): |
| 77 """Get a filename of the latest goma comiler_proxy-subproc.INFO.""" | 77 """Get a filename of the latest goma comiler_proxy-subproc.INFO.""" |
| 78 return GetLatestGlogInfoFile('compiler_proxy-subproc') | 78 return GetLatestGlogInfoFile('compiler_proxy-subproc') |
| 79 | 79 |
| 80 | 80 |
| 81 def UploadToGomaLogGS(file_path, gs_filename, | 81 def UploadToGomaLogGS( |
| 82 text_to_append=None, | 82 file_path, gs_filename, text_to_append=None, override_gsutil=None): |
| 83 metadata=None, | |
| 84 override_gsutil=None): | |
| 85 """Upload a file to Google Cloud Storage (gs://chrome-goma-log). | 83 """Upload a file to Google Cloud Storage (gs://chrome-goma-log). |
| 86 | 84 |
| 87 Note that the uploaded file would automatically be gzip compressed. | 85 Note that the uploaded file would automatically be gzip compressed. |
| 88 | 86 |
| 89 Args: | 87 Args: |
| 90 file_path: a path of a file to be uploaded. | 88 file_path: a path of a file to be uploaded. |
| 91 gs_filename: a name of a file in Google Storage. | 89 gs_filename: a name of a file in Google Storage. |
| 92 metadata: (dict) A dictionary of string key/value metadata entries. | |
| 93 text_to_append: an addtional text to be added to a file in GS. | 90 text_to_append: an addtional text to be added to a file in GS. |
| 94 | 91 |
| 95 Returns: | 92 Returns: |
| 96 a stored path name without the bucket name in GS. | 93 a stored path name without the bucket name in GS. |
| 97 """ | 94 """ |
| 98 hostname = GetShortHostname() | 95 hostname = GetShortHostname() |
| 99 today = datetime.datetime.utcnow().date() | 96 today = datetime.datetime.utcnow().date() |
| 100 log_path = '%s/%s/%s.gz' % ( | 97 log_path = '%s/%s/%s.gz' % ( |
| 101 today.strftime('%Y/%m/%d'), hostname, gs_filename) | 98 today.strftime('%Y/%m/%d'), hostname, gs_filename) |
| 102 gs_path = 'gs://%s/%s' % (GOMA_LOG_GS_BUCKET, log_path) | 99 gs_path = 'gs://%s/%s' % (GOMA_LOG_GS_BUCKET, log_path) |
| 103 temp = tempfile.NamedTemporaryFile(delete=False) | 100 temp = tempfile.NamedTemporaryFile(delete=False) |
| 104 try: | 101 try: |
| 105 with temp as f_out: | 102 with temp as f_out: |
| 106 with gzip.GzipFile(fileobj=f_out) as gzipf_out: | 103 with gzip.GzipFile(fileobj=f_out) as gzipf_out: |
| 107 with open(file_path) as f_in: | 104 with open(file_path) as f_in: |
| 108 shutil.copyfileobj(f_in, gzipf_out) | 105 shutil.copyfileobj(f_in, gzipf_out) |
| 109 if text_to_append: | 106 if text_to_append: |
| 110 gzipf_out.write(text_to_append) | 107 gzipf_out.write(text_to_append) |
| 111 slave_utils.GSUtilCopy(temp.name, gs_path, | 108 slave_utils.GSUtilCopy(temp.name, gs_path, override_gsutil=override_gsutil) |
| 112 metadata=metadata, override_gsutil=override_gsutil) | |
| 113 print "Copied log file to %s" % gs_path | 109 print "Copied log file to %s" % gs_path |
| 114 finally: | 110 finally: |
| 115 os.remove(temp.name) | 111 os.remove(temp.name) |
| 116 return log_path | 112 return log_path |
| 117 | 113 |
| 118 | 114 |
| 119 def UploadGomaCompilerProxyInfo(override_gsutil=None, | 115 def UploadGomaCompilerProxyInfo(override_gsutil=None): |
| 120 builder='unknown', master='unknown', | |
| 121 slave='unknown', clobber=''): | |
| 122 """Upload goma compiler_proxy.INFO to Google Storage.""" | 116 """Upload goma compiler_proxy.INFO to Google Storage.""" |
| 123 latest_subproc_info = GetLatestGomaCompilerProxySubprocInfo() | 117 latest_subproc_info = GetLatestGomaCompilerProxySubprocInfo() |
| 124 | |
| 125 builderinfo = { | |
| 126 'builder': builder, | |
| 127 'master': master, | |
| 128 'slave': slave, | |
| 129 'clobber': True if clobber else False, | |
| 130 'os': chromium_utils.PlatformName(), | |
| 131 } | |
| 132 # Append 'x-' to indicate this is custom metadata. | |
| 133 metadata = { | |
| 134 'x-builderinfo': json.dumps(builderinfo) | |
| 135 } | |
| 136 | |
| 137 if latest_subproc_info: | 118 if latest_subproc_info: |
| 138 UploadToGomaLogGS(latest_subproc_info, | 119 UploadToGomaLogGS(latest_subproc_info, |
| 139 os.path.basename(latest_subproc_info), | 120 os.path.basename(latest_subproc_info), |
| 140 metadata=metadata, | |
| 141 override_gsutil=override_gsutil) | 121 override_gsutil=override_gsutil) |
| 142 else: | 122 else: |
| 143 print 'No compiler_proxy-subproc.INFO to upload' | 123 print 'No compiler_proxy-subproc.INFO to upload' |
| 144 latest_info = GetLatestGomaCompilerProxyInfo() | 124 latest_info = GetLatestGomaCompilerProxyInfo() |
| 145 if not latest_info: | 125 if not latest_info: |
| 146 print 'No compiler_proxy.INFO to upload' | 126 print 'No compiler_proxy.INFO to upload' |
| 147 return | 127 return |
| 148 # Since a filename of compiler_proxy.INFO is fairly unique, | 128 # Since a filename of compiler_proxy.INFO is fairly unique, |
| 149 # we might be able to upload it as-is. | 129 # we might be able to upload it as-is. |
| 150 log_path = UploadToGomaLogGS( | 130 log_path = UploadToGomaLogGS( |
| 151 latest_info, os.path.basename(latest_info), | 131 latest_info, os.path.basename(latest_info), |
| 152 metadata=metadata, | |
| 153 override_gsutil=override_gsutil) | 132 override_gsutil=override_gsutil) |
| 154 viewer_url = ('http://chromium-build-stats.appspot.com/compiler_proxy_log/' | 133 viewer_url = ('http://chromium-build-stats.appspot.com/compiler_proxy_log/' |
| 155 + log_path) | 134 + log_path) |
| 156 print 'Visualization at %s' % viewer_url | 135 print 'Visualization at %s' % viewer_url |
| 157 | 136 |
| 158 | 137 |
| 159 def UploadNinjaLog( | 138 def UploadNinjaLog( |
| 160 outdir, compiler, command, exit_status, override_gsutil=None): | 139 outdir, compiler, command, exit_status, override_gsutil=None): |
| 161 """Upload .ninja_log to Google Cloud Storage (gs://chrome-goma-log), | 140 """Upload .ninja_log to Google Cloud Storage (gs://chrome-goma-log), |
| 162 in the same folder with goma's compiler_proxy.INFO. | 141 in the same folder with goma's compiler_proxy.INFO. |
| (...skipping 29 matching lines...) Expand all Loading... |
| 192 if compiler_proxy_info: | 171 if compiler_proxy_info: |
| 193 info['compiler_proxy_info'] = compiler_proxy_info | 172 info['compiler_proxy_info'] = compiler_proxy_info |
| 194 | 173 |
| 195 username = getpass.getuser() | 174 username = getpass.getuser() |
| 196 hostname = GetShortHostname() | 175 hostname = GetShortHostname() |
| 197 pid = os.getpid() | 176 pid = os.getpid() |
| 198 ninja_log_filename = 'ninja_log.%s.%s.%s.%d' % ( | 177 ninja_log_filename = 'ninja_log.%s.%s.%s.%d' % ( |
| 199 hostname, username, mtime.strftime('%Y%m%d-%H%M%S'), pid) | 178 hostname, username, mtime.strftime('%Y%m%d-%H%M%S'), pid) |
| 200 additional_text = '# end of ninja log\n' + json.dumps(info) | 179 additional_text = '# end of ninja log\n' + json.dumps(info) |
| 201 log_path = UploadToGomaLogGS( | 180 log_path = UploadToGomaLogGS( |
| 202 ninja_log_path, ninja_log_filename, text_to_append=additional_text, | 181 ninja_log_path, ninja_log_filename, additional_text, |
| 203 override_gsutil=override_gsutil) | 182 override_gsutil=override_gsutil) |
| 204 viewer_url = 'http://chromium-build-stats.appspot.com/ninja_log/' + log_path | 183 viewer_url = 'http://chromium-build-stats.appspot.com/ninja_log/' + log_path |
| 205 print 'Visualization at %s' % viewer_url | 184 print 'Visualization at %s' % viewer_url |
| 206 | 185 |
| 207 | 186 |
| 208 def IsCompilerProxyKilledByFatalError(): | 187 def IsCompilerProxyKilledByFatalError(): |
| 209 """Returns true if goma compiler_proxy is killed by CHECK or LOG(FATAL).""" | 188 """Returns true if goma compiler_proxy is killed by CHECK or LOG(FATAL).""" |
| 210 info_file = GetLatestGomaCompilerProxyInfo() | 189 info_file = GetLatestGomaCompilerProxyInfo() |
| 211 if not info_file: | 190 if not info_file: |
| 212 return False | 191 return False |
| (...skipping 154 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 367 retcode = chromium_utils.RunCommand( | 346 retcode = chromium_utils.RunCommand( |
| 368 cmd, filter_obj=cmd_filter, | 347 cmd, filter_obj=cmd_filter, |
| 369 max_time=30) | 348 max_time=30) |
| 370 if retcode: | 349 if retcode: |
| 371 print('Execution of send_ts_mon_values failed with code %s' | 350 print('Execution of send_ts_mon_values failed with code %s' |
| 372 % retcode) | 351 % retcode) |
| 373 print '\n'.join(cmd_filter.text) | 352 print '\n'.join(cmd_filter.text) |
| 374 | 353 |
| 375 except Exception as ex: | 354 except Exception as ex: |
| 376 print('error while sending ts mon json_file=%s: %s' % (json_file, ex)) | 355 print('error while sending ts mon json_file=%s: %s' % (json_file, ex)) |
| OLD | NEW |