Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(77)

Side by Side Diff: scripts/slave/goma_utils.py

Issue 2313783003: Revert of Reland 'Add buildbot metadata to compiler_proxy log' (Closed)
Patch Set: Created 4 years, 3 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
« no previous file with comments | « scripts/slave/compile.py ('k') | no next file » | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 # Copyright (c) 2015 The Chromium Authors. All rights reserved. 1 # Copyright (c) 2015 The Chromium Authors. All rights reserved.
2 # Use of this source code is governed by a BSD-style license that can be 2 # Use of this source code is governed by a BSD-style license that can be
3 # found in the LICENSE file. 3 # found in the LICENSE file.
4 4
5 """Functions specific to handle goma related info. 5 """Functions specific to handle goma related info.
6 """ 6 """
7 7
8 import base64 8 import base64
9 import datetime 9 import datetime
10 import getpass 10 import getpass
(...skipping 60 matching lines...) Expand 10 before | Expand all | Expand 10 after
71 def GetLatestGomaCompilerProxyInfo(): 71 def GetLatestGomaCompilerProxyInfo():
72 """Get a filename of the latest goma comiler_proxy.INFO.""" 72 """Get a filename of the latest goma comiler_proxy.INFO."""
73 return GetLatestGlogInfoFile('compiler_proxy') 73 return GetLatestGlogInfoFile('compiler_proxy')
74 74
75 75
76 def GetLatestGomaCompilerProxySubprocInfo(): 76 def GetLatestGomaCompilerProxySubprocInfo():
77 """Get a filename of the latest goma comiler_proxy-subproc.INFO.""" 77 """Get a filename of the latest goma comiler_proxy-subproc.INFO."""
78 return GetLatestGlogInfoFile('compiler_proxy-subproc') 78 return GetLatestGlogInfoFile('compiler_proxy-subproc')
79 79
80 80
81 def UploadToGomaLogGS(file_path, gs_filename, 81 def UploadToGomaLogGS(
82 text_to_append=None, 82 file_path, gs_filename, text_to_append=None, override_gsutil=None):
83 metadata=None,
84 override_gsutil=None):
85 """Upload a file to Google Cloud Storage (gs://chrome-goma-log). 83 """Upload a file to Google Cloud Storage (gs://chrome-goma-log).
86 84
87 Note that the uploaded file would automatically be gzip compressed. 85 Note that the uploaded file would automatically be gzip compressed.
88 86
89 Args: 87 Args:
90 file_path: a path of a file to be uploaded. 88 file_path: a path of a file to be uploaded.
91 gs_filename: a name of a file in Google Storage. 89 gs_filename: a name of a file in Google Storage.
92 metadata: (dict) A dictionary of string key/value metadata entries.
93 text_to_append: an addtional text to be added to a file in GS. 90 text_to_append: an addtional text to be added to a file in GS.
94 91
95 Returns: 92 Returns:
96 a stored path name without the bucket name in GS. 93 a stored path name without the bucket name in GS.
97 """ 94 """
98 hostname = GetShortHostname() 95 hostname = GetShortHostname()
99 today = datetime.datetime.utcnow().date() 96 today = datetime.datetime.utcnow().date()
100 log_path = '%s/%s/%s.gz' % ( 97 log_path = '%s/%s/%s.gz' % (
101 today.strftime('%Y/%m/%d'), hostname, gs_filename) 98 today.strftime('%Y/%m/%d'), hostname, gs_filename)
102 gs_path = 'gs://%s/%s' % (GOMA_LOG_GS_BUCKET, log_path) 99 gs_path = 'gs://%s/%s' % (GOMA_LOG_GS_BUCKET, log_path)
103 temp = tempfile.NamedTemporaryFile(delete=False) 100 temp = tempfile.NamedTemporaryFile(delete=False)
104 try: 101 try:
105 with temp as f_out: 102 with temp as f_out:
106 with gzip.GzipFile(fileobj=f_out) as gzipf_out: 103 with gzip.GzipFile(fileobj=f_out) as gzipf_out:
107 with open(file_path) as f_in: 104 with open(file_path) as f_in:
108 shutil.copyfileobj(f_in, gzipf_out) 105 shutil.copyfileobj(f_in, gzipf_out)
109 if text_to_append: 106 if text_to_append:
110 gzipf_out.write(text_to_append) 107 gzipf_out.write(text_to_append)
111 slave_utils.GSUtilCopy(temp.name, gs_path, 108 slave_utils.GSUtilCopy(temp.name, gs_path, override_gsutil=override_gsutil)
112 metadata=metadata, override_gsutil=override_gsutil)
113 print "Copied log file to %s" % gs_path 109 print "Copied log file to %s" % gs_path
114 finally: 110 finally:
115 os.remove(temp.name) 111 os.remove(temp.name)
116 return log_path 112 return log_path
117 113
118 114
119 def UploadGomaCompilerProxyInfo(override_gsutil=None, 115 def UploadGomaCompilerProxyInfo(override_gsutil=None):
120 builder='unknown', master='unknown',
121 slave='unknown', clobber=''):
122 """Upload goma compiler_proxy.INFO to Google Storage.""" 116 """Upload goma compiler_proxy.INFO to Google Storage."""
123 latest_subproc_info = GetLatestGomaCompilerProxySubprocInfo() 117 latest_subproc_info = GetLatestGomaCompilerProxySubprocInfo()
124
125 builderinfo = {
126 'builder': builder,
127 'master': master,
128 'slave': slave,
129 'clobber': True if clobber else False,
130 'os': chromium_utils.PlatformName(),
131 }
132 # Append 'x-' to indicate this is custom metadata.
133 metadata = {
134 'x-builderinfo': json.dumps(builderinfo)
135 }
136
137 if latest_subproc_info: 118 if latest_subproc_info:
138 UploadToGomaLogGS(latest_subproc_info, 119 UploadToGomaLogGS(latest_subproc_info,
139 os.path.basename(latest_subproc_info), 120 os.path.basename(latest_subproc_info),
140 metadata=metadata,
141 override_gsutil=override_gsutil) 121 override_gsutil=override_gsutil)
142 else: 122 else:
143 print 'No compiler_proxy-subproc.INFO to upload' 123 print 'No compiler_proxy-subproc.INFO to upload'
144 latest_info = GetLatestGomaCompilerProxyInfo() 124 latest_info = GetLatestGomaCompilerProxyInfo()
145 if not latest_info: 125 if not latest_info:
146 print 'No compiler_proxy.INFO to upload' 126 print 'No compiler_proxy.INFO to upload'
147 return 127 return
148 # Since a filename of compiler_proxy.INFO is fairly unique, 128 # Since a filename of compiler_proxy.INFO is fairly unique,
149 # we might be able to upload it as-is. 129 # we might be able to upload it as-is.
150 log_path = UploadToGomaLogGS( 130 log_path = UploadToGomaLogGS(
151 latest_info, os.path.basename(latest_info), 131 latest_info, os.path.basename(latest_info),
152 metadata=metadata,
153 override_gsutil=override_gsutil) 132 override_gsutil=override_gsutil)
154 viewer_url = ('http://chromium-build-stats.appspot.com/compiler_proxy_log/' 133 viewer_url = ('http://chromium-build-stats.appspot.com/compiler_proxy_log/'
155 + log_path) 134 + log_path)
156 print 'Visualization at %s' % viewer_url 135 print 'Visualization at %s' % viewer_url
157 136
158 137
159 def UploadNinjaLog( 138 def UploadNinjaLog(
160 outdir, compiler, command, exit_status, override_gsutil=None): 139 outdir, compiler, command, exit_status, override_gsutil=None):
161 """Upload .ninja_log to Google Cloud Storage (gs://chrome-goma-log), 140 """Upload .ninja_log to Google Cloud Storage (gs://chrome-goma-log),
162 in the same folder with goma's compiler_proxy.INFO. 141 in the same folder with goma's compiler_proxy.INFO.
(...skipping 29 matching lines...) Expand all
192 if compiler_proxy_info: 171 if compiler_proxy_info:
193 info['compiler_proxy_info'] = compiler_proxy_info 172 info['compiler_proxy_info'] = compiler_proxy_info
194 173
195 username = getpass.getuser() 174 username = getpass.getuser()
196 hostname = GetShortHostname() 175 hostname = GetShortHostname()
197 pid = os.getpid() 176 pid = os.getpid()
198 ninja_log_filename = 'ninja_log.%s.%s.%s.%d' % ( 177 ninja_log_filename = 'ninja_log.%s.%s.%s.%d' % (
199 hostname, username, mtime.strftime('%Y%m%d-%H%M%S'), pid) 178 hostname, username, mtime.strftime('%Y%m%d-%H%M%S'), pid)
200 additional_text = '# end of ninja log\n' + json.dumps(info) 179 additional_text = '# end of ninja log\n' + json.dumps(info)
201 log_path = UploadToGomaLogGS( 180 log_path = UploadToGomaLogGS(
202 ninja_log_path, ninja_log_filename, text_to_append=additional_text, 181 ninja_log_path, ninja_log_filename, additional_text,
203 override_gsutil=override_gsutil) 182 override_gsutil=override_gsutil)
204 viewer_url = 'http://chromium-build-stats.appspot.com/ninja_log/' + log_path 183 viewer_url = 'http://chromium-build-stats.appspot.com/ninja_log/' + log_path
205 print 'Visualization at %s' % viewer_url 184 print 'Visualization at %s' % viewer_url
206 185
207 186
208 def IsCompilerProxyKilledByFatalError(): 187 def IsCompilerProxyKilledByFatalError():
209 """Returns true if goma compiler_proxy is killed by CHECK or LOG(FATAL).""" 188 """Returns true if goma compiler_proxy is killed by CHECK or LOG(FATAL)."""
210 info_file = GetLatestGomaCompilerProxyInfo() 189 info_file = GetLatestGomaCompilerProxyInfo()
211 if not info_file: 190 if not info_file:
212 return False 191 return False
(...skipping 154 matching lines...) Expand 10 before | Expand all | Expand 10 after
367 retcode = chromium_utils.RunCommand( 346 retcode = chromium_utils.RunCommand(
368 cmd, filter_obj=cmd_filter, 347 cmd, filter_obj=cmd_filter,
369 max_time=30) 348 max_time=30)
370 if retcode: 349 if retcode:
371 print('Execution of send_ts_mon_values failed with code %s' 350 print('Execution of send_ts_mon_values failed with code %s'
372 % retcode) 351 % retcode)
373 print '\n'.join(cmd_filter.text) 352 print '\n'.join(cmd_filter.text)
374 353
375 except Exception as ex: 354 except Exception as ex:
376 print('error while sending ts mon json_file=%s: %s' % (json_file, ex)) 355 print('error while sending ts mon json_file=%s: %s' % (json_file, ex))
OLDNEW
« no previous file with comments | « scripts/slave/compile.py ('k') | no next file » | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698