Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(140)

Side by Side Diff: buildbot/prebuilt.py

Issue 6840064: Restart codereview issue 6792042 (Closed) Base URL: http://git.chromium.org/git/chromite.git@master
Patch Set: Simplify the addition of the emptyboard flag. Created 9 years, 8 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch | Annotate | Revision Log
OLDNEW
1 #!/usr/bin/python 1 #!/usr/bin/python
2 # Copyright (c) 2010 The Chromium OS Authors. All rights reserved. 2 # Copyright (c) 2010 The Chromium OS Authors. All rights reserved.
3 # Use of this source code is governed by a BSD-style license that can be 3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file. 4 # found in the LICENSE file.
5 5
6 import datetime 6 import datetime
7 import multiprocessing 7 import multiprocessing
8 import optparse 8 import optparse
9 import os 9 import os
10 import re 10 import re
(...skipping 124 matching lines...) Expand 10 before | Expand all | Expand 10 after
135 """Repo sync and then push git changes in flight. 135 """Repo sync and then push git changes in flight.
136 136
137 Args: 137 Args:
138 retries: The number of times to retry before giving up, default: 5 138 retries: The number of times to retry before giving up, default: 5
139 139
140 Raises: 140 Raises:
141 GitPushFailed if push was unsuccessful after retries 141 GitPushFailed if push was unsuccessful after retries
142 """ 142 """
143 for retry in range(1, retries + 1): 143 for retry in range(1, retries + 1):
144 try: 144 try:
145 cros_build_lib.RunCommand('repo sync .', shell=True) 145 cros_build_lib.RunCommand(['repo', 'sync', '.'])
146 cros_build_lib.RunCommand('git push', shell=True) 146 cros_build_lib.RunCommand(['git', 'push'])
147 break 147 break
148 except cros_build_lib.RunCommandError: 148 except cros_build_lib.RunCommandError:
149 if retry < retries: 149 if retry < retries:
150 print 'Error pushing changes trying again (%s/%s)' % (retry, retries) 150 print 'Error pushing changes trying again (%s/%s)' % (retry, retries)
151 time.sleep(5 * retry) 151 time.sleep(5 * retry)
152 else: 152 else:
153 raise GitPushFailed('Failed to push change after %s retries' % retries) 153 raise GitPushFailed('Failed to push change after %s retries' % retries)
154 154
155 155
156 def RevGitFile(filename, value, retries=5, key='PORTAGE_BINHOST'): 156 def RevGitFile(filename, value, retries=5, key='PORTAGE_BINHOST'):
157 """Update and push the git file. 157 """Update and push the git file.
158 158
159 Args: 159 Args:
160 filename: file to modify that is in a git repo already 160 filename: file to modify that is in a git repo already
161 value: string representing the version of the prebuilt that has been 161 value: string representing the version of the prebuilt that has been
162 uploaded. 162 uploaded.
163 retries: The number of times to retry before giving up, default: 5 163 retries: The number of times to retry before giving up, default: 5
164 key: The variable key to update in the git file. 164 key: The variable key to update in the git file.
165 (Default: PORTAGE_BINHOST) 165 (Default: PORTAGE_BINHOST)
166 """ 166 """
167 prebuilt_branch = 'prebuilt_branch' 167 prebuilt_branch = 'prebuilt_branch'
168 old_cwd = os.getcwd() 168 old_cwd = os.getcwd()
169 os.chdir(os.path.dirname(filename)) 169 os.chdir(os.path.dirname(filename))
170 170
171 commit = cros_build_lib.RunCommand('git rev-parse HEAD', shell=True, 171 commit = cros_build_lib.RunCommand(['git', 'rev-parse', 'HEAD'],
172 redirect_stdout=True).output 172 redirect_stdout=True).output
173 cros_build_lib.RunCommand('git remote update', shell=True) 173 cros_build_lib.RunCommand(['git', 'remote', 'update'])
174 cros_build_lib.RunCommand('repo start %s .' % prebuilt_branch, shell=True) 174 cros_build_lib.RunCommand(['repo', 'start', prebuilt_branch, '.'])
175 git_ssh_config_cmd = ( 175 git_ssh_config_cmd = [
176 'git config url.ssh://git@gitrw.chromium.org:9222.pushinsteadof ' 176 'git',
177 'http://git.chromium.org/git') 177 'config',
178 cros_build_lib.RunCommand(git_ssh_config_cmd, shell=True) 178 'url.ssh://git@gitrw.chromium.org:9222.pushinsteadof',
179 'http://git.chromium.org/git' ]
180 cros_build_lib.RunCommand(git_ssh_config_cmd)
179 description = 'Update %s="%s" in %s' % (key, value, filename) 181 description = 'Update %s="%s" in %s' % (key, value, filename)
180 print description 182 print description
181 try: 183 try:
182 UpdateLocalFile(filename, value, key) 184 UpdateLocalFile(filename, value, key)
183 cros_build_lib.RunCommand('git config push.default tracking', shell=True) 185 cros_build_lib.RunCommand(['git', 'config', 'push.default', 'tracking'])
184 cros_build_lib.RunCommand('git commit -am "%s"' % description, shell=True) 186 cros_build_lib.RunCommand(['git', 'commit', '-am', description])
185 RevGitPushWithRetry(retries) 187 RevGitPushWithRetry(retries)
186 finally: 188 finally:
187 cros_build_lib.RunCommand('repo abandon %s .' % prebuilt_branch, shell=True) 189 cros_build_lib.RunCommand(['repo', 'abandon', 'prebuilt_branch', '.'])
188 cros_build_lib.RunCommand('git checkout %s' % commit, shell=True) 190 cros_build_lib.RunCommand(['git', 'checkout', commit])
189 os.chdir(old_cwd) 191 os.chdir(old_cwd)
190 192
191 193
192 def GetVersion(): 194 def GetVersion():
193 """Get the version to put in LATEST and update the git version with.""" 195 """Get the version to put in LATEST and update the git version with."""
194 return datetime.datetime.now().strftime('%d.%m.%y.%H%M%S') 196 return datetime.datetime.now().strftime('%d.%m.%y.%H%M%S')
195 197
196 198
197 def LoadPrivateFilters(build_path): 199 def LoadPrivateFilters(build_path):
198 """Load private filters based on ebuilds found under _PRIVATE_OVERLAY_DIR. 200 """Load private filters based on ebuilds found under _PRIVATE_OVERLAY_DIR.
(...skipping 34 matching lines...) Expand 10 before | Expand all | Expand 10 after
233 False otherwise. 235 False otherwise.
234 """ 236 """
235 for name in _FILTER_PACKAGES: 237 for name in _FILTER_PACKAGES:
236 if name in file_path: 238 if name in file_path:
237 print 'FILTERING %s' % file_path 239 print 'FILTERING %s' % file_path
238 return True 240 return True
239 241
240 return False 242 return False
241 243
242 244
243 def _RetryRun(cmd, print_cmd=True, shell=False, cwd=None): 245 def _RetryRun(cmd, print_cmd=True, cwd=None):
244 """Run the specified command, retrying if necessary. 246 """Run the specified command, retrying if necessary.
245 247
246 Args: 248 Args:
247 cmd: The command to run. 249 cmd: The command to run.
248 print_cmd: Whether to print out the cmd. 250 print_cmd: Whether to print out the cmd.
249 shell: Whether to treat the command as a shell. 251 shell: Whether to treat the command as a shell.
250 cwd: Working directory to run command in. 252 cwd: Working directory to run command in.
251 253
252 Returns: 254 Returns:
253 True if the command succeeded. Otherwise, returns False. 255 True if the command succeeded. Otherwise, returns False.
254 """ 256 """
255 257
256 # TODO(scottz): port to use _Run or similar when it is available in 258 # TODO(scottz): port to use _Run or similar when it is available in
257 # cros_build_lib. 259 # cros_build_lib.
258 for attempt in range(_RETRIES): 260 for attempt in range(_RETRIES):
259 try: 261 try:
260 output = cros_build_lib.RunCommand(cmd, print_cmd=print_cmd, shell=shell, 262 output = cros_build_lib.RunCommand(cmd, print_cmd=print_cmd,
261 cwd=cwd) 263 cwd=cwd)
262 return True 264 return True
263 except cros_build_lib.RunCommandError: 265 except cros_build_lib.RunCommandError:
264 print 'Failed to run %s' % cmd 266 print 'Failed to run %r' % cmd
265 else: 267 else:
266 print 'Retry failed run %s, giving up' % cmd 268 print 'Retry failed run %r, giving up' % cmd
267 return False 269 return False
268 270
269 271
270 def _GsUpload(args): 272 def _GsUpload(args):
271 """Upload to GS bucket. 273 """Upload to GS bucket.
272 274
273 Args: 275 Args:
274 args: a tuple of three arguments that contains local_file, remote_file, and 276 args: a tuple of three arguments that contains local_file, remote_file, and
275 the acl used for uploading the file. 277 the acl used for uploading the file.
276 278
(...skipping 13 matching lines...) Expand all
290 cmd = '%s cp -a private %s %s' % (_GSUTIL_BIN, local_file, remote_file) 292 cmd = '%s cp -a private %s %s' % (_GSUTIL_BIN, local_file, remote_file)
291 if not os.path.exists(acl): 293 if not os.path.exists(acl):
292 print >> sys.stderr, ('You are specifying either a file that does not ' 294 print >> sys.stderr, ('You are specifying either a file that does not '
293 'exist or an unknown canned acl: %s. Aborting ' 295 'exist or an unknown canned acl: %s. Aborting '
294 'upload') % acl 296 'upload') % acl
295 # emulate the failing of an upload since we are not uploading the file 297 # emulate the failing of an upload since we are not uploading the file
296 return (local_file, remote_file) 298 return (local_file, remote_file)
297 299
298 acl_cmd = '%s setacl %s %s' % (_GSUTIL_BIN, acl, remote_file) 300 acl_cmd = '%s setacl %s %s' % (_GSUTIL_BIN, acl, remote_file)
299 301
300 if not _RetryRun(cmd, print_cmd=False, shell=True): 302 if not _RetryRun(cmd, print_cmd=False):
301 return (local_file, remote_file) 303 return (local_file, remote_file)
302 304
303 if acl_cmd: 305 if acl_cmd:
304 # Apply the passed in ACL xml file to the uploaded object. 306 # Apply the passed in ACL xml file to the uploaded object.
305 _RetryRun(acl_cmd, print_cmd=False, shell=True) 307 _RetryRun(acl_cmd, print_cmd=False)
306 308
307 309
308 def RemoteUpload(acl, files, pool=10): 310 def RemoteUpload(acl, files, pool=10):
309 """Upload to google storage. 311 """Upload to google storage.
310 312
311 Create a pool of process and call _GsUpload with the proper arguments. 313 Create a pool of process and call _GsUpload with the proper arguments.
312 314
313 Args: 315 Args:
314 acl: The canned acl used for uploading. acl can be one of: "public-read", 316 acl: The canned acl used for uploading. acl can be one of: "public-read",
315 "public-read-write", "authenticated-read", "bucket-owner-read", 317 "public-read-write", "authenticated-read", "bucket-owner-read",
(...skipping 99 matching lines...) Expand 10 before | Expand all | Expand 10 after
415 value: New value for key. 417 value: New value for key.
416 """ 418 """
417 cwd = os.path.dirname(os.path.abspath(path)) 419 cwd = os.path.dirname(os.path.abspath(path))
418 filename = os.path.basename(path) 420 filename = os.path.basename(path)
419 if not os.path.isdir(cwd): 421 if not os.path.isdir(cwd):
420 os.makedirs(cwd) 422 os.makedirs(cwd)
421 if not os.path.isfile(path): 423 if not os.path.isfile(path):
422 config_file = file(path, 'w') 424 config_file = file(path, 'w')
423 config_file.close() 425 config_file.close()
424 UpdateLocalFile(path, value, key) 426 UpdateLocalFile(path, value, key)
425 cros_build_lib.RunCommand('git add %s' % filename, cwd=cwd, shell=True) 427 cros_build_lib.RunCommand(['git', 'add', filename], cwd=cwd)
426 description = 'Update %s=%s in %s' % (key, value, filename) 428 description = 'Update %s=%s in %s' % (key, value, filename)
427 cros_build_lib.RunCommand('git commit -m "%s"' % description, cwd=cwd, 429 cros_build_lib.RunCommand(['git', 'commit', '-m', description], cwd=cwd)
428 shell=True)
429 430
430 431
431 def _GrabAllRemotePackageIndexes(binhost_urls): 432 def _GrabAllRemotePackageIndexes(binhost_urls):
432 """Grab all of the packages files associated with a list of binhost_urls. 433 """Grab all of the packages files associated with a list of binhost_urls.
433 434
434 Args: 435 Args:
435 binhost_urls: The URLs for the directories containing the Packages files we 436 binhost_urls: The URLs for the directories containing the Packages files we
436 want to grab. 437 want to grab.
437 438
438 Returns: 439 Returns:
(...skipping 66 matching lines...) Expand 10 before | Expand all | Expand 10 after
505 'pkgs': pkgs, 506 'pkgs': pkgs,
506 'remote_packages': '%s/Packages' % remote_location.rstrip('/'), 507 'remote_packages': '%s/Packages' % remote_location.rstrip('/'),
507 'remote_path': remote_path.rstrip('/'), 508 'remote_path': remote_path.rstrip('/'),
508 'remote_location': remote_location.rstrip('/'), 509 'remote_location': remote_location.rstrip('/'),
509 'ssh_server': ssh_server } 510 'ssh_server': ssh_server }
510 cmds = ['ssh %(ssh_server)s mkdir -p %(remote_path)s' % d, 511 cmds = ['ssh %(ssh_server)s mkdir -p %(remote_path)s' % d,
511 'rsync -av --chmod=a+r %(pkg_index)s %(remote_packages)s' % d] 512 'rsync -av --chmod=a+r %(pkg_index)s %(remote_packages)s' % d]
512 if pkgs: 513 if pkgs:
513 cmds.append('rsync -Rav %(pkgs)s %(remote_location)s/' % d) 514 cmds.append('rsync -Rav %(pkgs)s %(remote_location)s/' % d)
514 for cmd in cmds: 515 for cmd in cmds:
515 if not _RetryRun(cmd, shell=True, cwd=package_path): 516 if not _RetryRun(cmd, cwd=package_path):
516 raise UploadFailed('Could not run %s' % cmd) 517 raise UploadFailed('Could not run %s' % cmd)
517 518
518 def _UploadBoardTarball(self, board_path, url_suffix): 519 def _UploadBoardTarball(self, board_path, url_suffix):
519 """Upload a tarball of the board at the specified path to Google Storage. 520 """Upload a tarball of the board at the specified path to Google Storage.
520 521
521 Args: 522 Args:
522 board_path: The path to the board dir. 523 board_path: The path to the board dir.
523 url_suffix: The remote subdirectory where we should upload the packages. 524 url_suffix: The remote subdirectory where we should upload the packages.
524 """ 525 """
525 remote_location = '%s/%s' % (self._upload_location.rstrip('/'), url_suffix) 526 remote_location = '%s/%s' % (self._upload_location.rstrip('/'), url_suffix)
(...skipping 205 matching lines...) Expand 10 before | Expand all | Expand 10 after
731 options.git_sync, options.sync_binhost_conf) 732 options.git_sync, options.sync_binhost_conf)
732 733
733 if options.board: 734 if options.board:
734 uploader._SyncBoardPrebuilts(options.board, options.build_path, version, 735 uploader._SyncBoardPrebuilts(options.board, options.build_path, version,
735 options.key, options.git_sync, 736 options.key, options.git_sync,
736 options.sync_binhost_conf, 737 options.sync_binhost_conf,
737 options.upload_board_tarball) 738 options.upload_board_tarball)
738 739
739 if __name__ == '__main__': 740 if __name__ == '__main__':
740 main() 741 main()
OLDNEW

Powered by Google App Engine
This is Rietveld 408576698