Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(461)

Side by Side Diff: cros_mark_as_stable.py

Issue 6286040: Remove buildbot code that has been added to chromite. (Closed) Base URL: http://git.chromium.org/git/crosutils.git@master
Patch Set: Verify Created 9 years, 10 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch | Annotate | Revision Log
« no previous file with comments | « cros_mark_as_stable ('k') | cros_mark_as_stable_blacklist » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
(Empty)
1 #!/usr/bin/python
2
3 # Copyright (c) 2010 The Chromium OS Authors. All rights reserved.
4 # Use of this source code is governed by a BSD-style license that can be
5 # found in the LICENSE file.
6
7 """This module uprevs a given package's ebuild to the next revision."""
8
9
10 import fileinput
11 import gflags
12 import os
13 import re
14 import shutil
15 import subprocess
16 import sys
17
18 sys.path.append(os.path.join(os.path.dirname(__file__), 'lib'))
19 from cros_build_lib import Info, RunCommand, Warning, Die
20
21 gflags.DEFINE_boolean('all', False,
22 'Mark all packages as stable.')
23 gflags.DEFINE_string('board', '',
24 'Board for which the package belongs.', short_name='b')
25 gflags.DEFINE_string('drop_file', None,
26 'File to list packages that were revved.')
27 gflags.DEFINE_boolean('dryrun', False,
28 'Passes dry-run to git push if pushing a change.')
29 gflags.DEFINE_string('overlays', '',
30 'Colon-separated list of overlays to modify.',
31 short_name='o')
32 gflags.DEFINE_string('packages', '',
33 'Colon-separated list of packages to mark as stable.',
34 short_name='p')
35 gflags.DEFINE_string('srcroot', '%s/trunk/src' % os.environ['HOME'],
36 'Path to root src directory.',
37 short_name='r')
38 gflags.DEFINE_string('tracking_branch', 'cros/master',
39 'Used with commit to specify branch to track against.',
40 short_name='t')
41 gflags.DEFINE_boolean('verbose', False,
42 'Prints out verbose information about what is going on.',
43 short_name='v')
44
45
46 # Takes two strings, package_name and commit_id.
47 _GIT_COMMIT_MESSAGE = 'Marking 9999 ebuild for %s with commit %s as stable.'
48
49 # Dictionary of valid commands with usage information.
50 COMMAND_DICTIONARY = {
51 'clean':
52 'Cleans up previous calls to either commit or push',
53 'commit':
54 'Marks given ebuilds as stable locally',
55 'push':
56 'Pushes previous marking of ebuilds to remote repo',
57 }
58
59 # Name used for stabilizing branch.
60 STABLE_BRANCH_NAME = 'stabilizing_branch'
61
62
63 def BestEBuild(ebuilds):
64 """Returns the newest EBuild from a list of EBuild objects."""
65 from portage.versions import vercmp
66 winner = ebuilds[0]
67 for ebuild in ebuilds[1:]:
68 if vercmp(winner.version, ebuild.version) < 0:
69 winner = ebuild
70 return winner
71
72 # ======================= Global Helper Functions ========================
73
74
75 def _Print(message):
76 """Verbose print function."""
77 if gflags.FLAGS.verbose:
78 Info(message)
79
80
81 def _CleanStalePackages(board, package_atoms):
82 """Cleans up stale package info from a previous build."""
83 Info('Cleaning up stale packages %s.' % package_atoms)
84 unmerge_board_cmd = ['emerge-%s' % board, '--unmerge']
85 unmerge_board_cmd.extend(package_atoms)
86 RunCommand(unmerge_board_cmd)
87
88 unmerge_host_cmd = ['sudo', 'emerge', '--unmerge']
89 unmerge_host_cmd.extend(package_atoms)
90 RunCommand(unmerge_host_cmd)
91
92 RunCommand(['eclean-%s' % board, '-d', 'packages'], redirect_stderr=True)
93 RunCommand(['sudo', 'eclean', '-d', 'packages'], redirect_stderr=True)
94
95
96 def _FindUprevCandidates(files):
97 """Return a list of uprev candidates from specified list of files.
98
99 Usually an uprev candidate is a the stable ebuild in a cros_workon directory.
100 However, if no such stable ebuild exists (someone just checked in the 9999
101 ebuild), this is the unstable ebuild.
102
103 Args:
104 files: List of files.
105 """
106 workon_dir = False
107 stable_ebuilds = []
108 unstable_ebuilds = []
109 for path in files:
110 if path.endswith('.ebuild') and not os.path.islink(path):
111 ebuild = EBuild(path)
112 if ebuild.is_workon:
113 workon_dir = True
114 if ebuild.is_stable:
115 stable_ebuilds.append(ebuild)
116 else:
117 unstable_ebuilds.append(ebuild)
118
119 # If we found a workon ebuild in this directory, apply some sanity checks.
120 if workon_dir:
121 if len(unstable_ebuilds) > 1:
122 Die('Found multiple unstable ebuilds in %s' % os.path.dirname(path))
123 if len(stable_ebuilds) > 1:
124 stable_ebuilds = [BestEBuild(stable_ebuilds)]
125
126 # Print a warning if multiple stable ebuilds are found in the same
127 # directory. Storing multiple stable ebuilds is error-prone because
128 # the older ebuilds will not get rev'd.
129 #
130 # We make a special exception for x11-drivers/xf86-video-msm for legacy
131 # reasons.
132 if stable_ebuilds[0].package != 'x11-drivers/xf86-video-msm':
133 Warning('Found multiple stable ebuilds in %s' % os.path.dirname(path))
134
135 if not unstable_ebuilds:
136 Die('Missing 9999 ebuild in %s' % os.path.dirname(path))
137 if not stable_ebuilds:
138 Warning('Missing stable ebuild in %s' % os.path.dirname(path))
139 return unstable_ebuilds[0]
140
141 if stable_ebuilds:
142 return stable_ebuilds[0]
143 else:
144 return None
145
146
147 def _BuildEBuildDictionary(overlays, all, packages):
148 """Build a dictionary of the ebuilds in the specified overlays.
149
150 overlays: A map which maps overlay directories to arrays of stable EBuilds
151 inside said directories.
152 all: Whether to include all ebuilds in the specified directories. If true,
153 then we gather all packages in the directories regardless of whether
154 they are in our set of packages.
155 packages: A set of the packages we want to gather.
156 """
157 for overlay in overlays:
158 for package_dir, dirs, files in os.walk(overlay):
159 # Add stable ebuilds to overlays[overlay].
160 paths = [os.path.join(package_dir, path) for path in files]
161 ebuild = _FindUprevCandidates(paths)
162
163 # If the --all option isn't used, we only want to update packages that
164 # are in packages.
165 if ebuild and (all or ebuild.package in packages):
166 overlays[overlay].append(ebuild)
167
168
169 def _DoWeHaveLocalCommits(stable_branch, tracking_branch):
170 """Returns true if there are local commits."""
171 current_branch = _SimpleRunCommand('git branch | grep \*').split()[1]
172 if current_branch == stable_branch:
173 current_commit_id = _SimpleRunCommand('git rev-parse HEAD')
174 tracking_commit_id = _SimpleRunCommand('git rev-parse %s' % tracking_branch)
175 return current_commit_id != tracking_commit_id
176 else:
177 return False
178
179
180 def _CheckSaneArguments(package_list, command):
181 """Checks to make sure the flags are sane. Dies if arguments are not sane."""
182 if not command in COMMAND_DICTIONARY.keys():
183 _PrintUsageAndDie('%s is not a valid command' % command)
184 if not gflags.FLAGS.packages and command == 'commit' and not gflags.FLAGS.all:
185 _PrintUsageAndDie('Please specify at least one package')
186 if not gflags.FLAGS.board and command == 'commit':
187 _PrintUsageAndDie('Please specify a board')
188 if not os.path.isdir(gflags.FLAGS.srcroot):
189 _PrintUsageAndDie('srcroot is not a valid path')
190 gflags.FLAGS.srcroot = os.path.abspath(gflags.FLAGS.srcroot)
191
192
193 def _PrintUsageAndDie(error_message=''):
194 """Prints optional error_message the usage and returns an error exit code."""
195 command_usage = 'Commands: \n'
196 # Add keys and usage information from dictionary.
197 commands = sorted(COMMAND_DICTIONARY.keys())
198 for command in commands:
199 command_usage += ' %s: %s\n' % (command, COMMAND_DICTIONARY[command])
200 commands_str = '|'.join(commands)
201 Warning('Usage: %s FLAGS [%s]\n\n%s\nFlags:%s' % (sys.argv[0], commands_str,
202 command_usage, gflags.FLAGS))
203 if error_message:
204 Die(error_message)
205 else:
206 sys.exit(1)
207
208
209 def _SimpleRunCommand(command):
210 """Runs a shell command and returns stdout back to caller."""
211 _Print(' + %s' % command)
212 proc_handle = subprocess.Popen(command, stdout=subprocess.PIPE, shell=True)
213 stdout = proc_handle.communicate()[0]
214 retcode = proc_handle.wait()
215 if retcode != 0:
216 _Print(stdout)
217 raise subprocess.CalledProcessError(retcode, command)
218 return stdout
219
220
221 # ======================= End Global Helper Functions ========================
222
223
224 def Clean(tracking_branch):
225 """Cleans up uncommitted changes.
226
227 Args:
228 tracking_branch: The tracking branch we want to return to after the call.
229 """
230 # Safety case in case we got into a bad state with a previous build.
231 try:
232 _SimpleRunCommand('git rebase --abort')
233 except:
234 pass
235
236 _SimpleRunCommand('git reset HEAD --hard')
237 branch = GitBranch(STABLE_BRANCH_NAME, tracking_branch)
238 if branch.Exists():
239 GitBranch.Checkout(branch)
240 branch.Delete()
241
242
243 def PushChange(stable_branch, tracking_branch):
244 """Pushes commits in the stable_branch to the remote git repository.
245
246 Pushes locals commits from calls to CommitChange to the remote git
247 repository specified by current working directory.
248
249 Args:
250 stable_branch: The local branch with commits we want to push.
251 tracking_branch: The tracking branch of the local branch.
252 Raises:
253 OSError: Error occurred while pushing.
254 """
255 num_retries = 5
256
257 # Sanity check to make sure we're on a stabilizing branch before pushing.
258 if not _DoWeHaveLocalCommits(stable_branch, tracking_branch):
259 Info('Not work found to push. Exiting')
260 return
261
262 description = _SimpleRunCommand('git log --format=format:%s%n%n%b ' +
263 tracking_branch + '..')
264 description = 'Marking set of ebuilds as stable\n\n%s' % description
265 Info('Using description %s' % description)
266 merge_branch_name = 'merge_branch'
267 for push_try in range(num_retries + 1):
268 try:
269 merge_branch = GitBranch(merge_branch_name, tracking_branch)
270 if merge_branch.Exists():
271 merge_branch.Delete()
272 _SimpleRunCommand('repo sync .')
273 merge_branch.CreateBranch()
274 if not merge_branch.Exists():
275 Die('Unable to create merge branch.')
276 _SimpleRunCommand('git merge --squash %s' % stable_branch)
277 _SimpleRunCommand('git commit -m "%s"' % description)
278 _SimpleRunCommand('git config push.default tracking')
279 if gflags.FLAGS.dryrun:
280 _SimpleRunCommand('git push --dry-run')
281 else:
282 _SimpleRunCommand('git push')
283
284 break
285 except:
286 if push_try < num_retries:
287 Warning('Failed to push change, performing retry (%s/%s)' % (
288 push_try + 1, num_retries))
289 else:
290 raise
291
292
293 class GitBranch(object):
294 """Wrapper class for a git branch."""
295
296 def __init__(self, branch_name, tracking_branch):
297 """Sets up variables but does not create the branch."""
298 self.branch_name = branch_name
299 self.tracking_branch = tracking_branch
300
301 def CreateBranch(self):
302 GitBranch.Checkout(self)
303
304 @classmethod
305 def Checkout(cls, target):
306 """Function used to check out to another GitBranch."""
307 if target.branch_name == target.tracking_branch or target.Exists():
308 git_cmd = 'git checkout %s -f' % target.branch_name
309 else:
310 git_cmd = 'git checkout -b %s %s -f' % (target.branch_name,
311 target.tracking_branch)
312 _SimpleRunCommand(git_cmd)
313
314 def Exists(self):
315 """Returns True if the branch exists."""
316 branch_cmd = 'git branch'
317 branches = _SimpleRunCommand(branch_cmd)
318 return self.branch_name in branches.split()
319
320 def Delete(self):
321 """Deletes the branch and returns the user to the master branch.
322
323 Returns True on success.
324 """
325 tracking_branch = GitBranch(self.tracking_branch, self.tracking_branch)
326 GitBranch.Checkout(tracking_branch)
327 delete_cmd = 'git branch -D %s' % self.branch_name
328 _SimpleRunCommand(delete_cmd)
329
330
331 class EBuild(object):
332 """Wrapper class for information about an ebuild."""
333
334 def __init__(self, path):
335 """Sets up data about an ebuild from its path."""
336 from portage.versions import pkgsplit
337 unused_path, self.category, self.pkgname, filename = path.rsplit('/', 3)
338 unused_pkgname, self.version_no_rev, rev = pkgsplit(
339 filename.replace('.ebuild', ''))
340
341 self.ebuild_path_no_version = os.path.join(
342 os.path.dirname(path), self.pkgname)
343 self.ebuild_path_no_revision = '%s-%s' % (self.ebuild_path_no_version,
344 self.version_no_rev)
345 self.current_revision = int(rev.replace('r', ''))
346 self.version = '%s-%s' % (self.version_no_rev, rev)
347 self.package = '%s/%s' % (self.category, self.pkgname)
348 self.ebuild_path = path
349
350 self.is_workon = False
351 self.is_stable = False
352
353 for line in fileinput.input(path):
354 if line.startswith('inherit ') and 'cros-workon' in line:
355 self.is_workon = True
356 elif (line.startswith('KEYWORDS=') and '~' not in line and
357 ('amd64' in line or 'x86' in line or 'arm' in line)):
358 self.is_stable = True
359 fileinput.close()
360
361 def GetCommitId(self):
362 """Get the commit id for this ebuild."""
363 # Grab and evaluate CROS_WORKON variables from this ebuild.
364 unstable_ebuild = '%s-9999.ebuild' % self.ebuild_path_no_version
365 cmd = ('export CROS_WORKON_LOCALNAME="%s" CROS_WORKON_PROJECT="%s"; '
366 'eval $(grep -E "^CROS_WORKON" %s) && '
367 'echo $CROS_WORKON_PROJECT '
368 '$CROS_WORKON_LOCALNAME/$CROS_WORKON_SUBDIR'
369 % (self.pkgname, self.pkgname, unstable_ebuild))
370 project, subdir = _SimpleRunCommand(cmd).split()
371
372 # Calculate srcdir.
373 srcroot = gflags.FLAGS.srcroot
374 if self.category == 'chromeos-base':
375 dir = 'platform'
376 else:
377 dir = 'third_party'
378 srcdir = os.path.join(srcroot, dir, subdir)
379
380 if not os.path.isdir(srcdir):
381 Die('Cannot find commit id for %s' % self.ebuild_path)
382
383 # Verify that we're grabbing the commit id from the right project name.
384 # NOTE: chromeos-kernel has the wrong project name, so it fails this
385 # check.
386 # TODO(davidjames): Fix the project name in the chromeos-kernel ebuild.
387 cmd = 'cd %s && git config --get remote.cros.projectname' % srcdir
388 actual_project = _SimpleRunCommand(cmd).rstrip()
389 if project not in (actual_project, 'chromeos-kernel'):
390 Die('Project name mismatch for %s (%s != %s)' % (unstable_ebuild, project,
391 actual_project))
392
393 # Get commit id.
394 output = _SimpleRunCommand('cd %s && git rev-parse HEAD' % srcdir)
395 if not output:
396 Die('Missing commit id for %s' % self.ebuild_path)
397 return output.rstrip()
398
399
400 class EBuildStableMarker(object):
401 """Class that revs the ebuild and commits locally or pushes the change."""
402
403 def __init__(self, ebuild):
404 assert ebuild
405 self._ebuild = ebuild
406
407 @classmethod
408 def MarkAsStable(cls, unstable_ebuild_path, new_stable_ebuild_path,
409 commit_keyword, commit_value, redirect_file=None,
410 make_stable=True):
411 """Static function that creates a revved stable ebuild.
412
413 This function assumes you have already figured out the name of the new
414 stable ebuild path and then creates that file from the given unstable
415 ebuild and marks it as stable. If the commit_value is set, it also
416 set the commit_keyword=commit_value pair in the ebuild.
417
418 Args:
419 unstable_ebuild_path: The path to the unstable ebuild.
420 new_stable_ebuild_path: The path you want to use for the new stable
421 ebuild.
422 commit_keyword: Optional keyword to set in the ebuild to mark it as
423 stable.
424 commit_value: Value to set the above keyword to.
425 redirect_file: Optionally redirect output of new ebuild somewhere else.
426 make_stable: Actually make the ebuild stable.
427 """
428 shutil.copyfile(unstable_ebuild_path, new_stable_ebuild_path)
429 for line in fileinput.input(new_stable_ebuild_path, inplace=1):
430 # Has to be done here to get changes to sys.stdout from fileinput.input.
431 if not redirect_file:
432 redirect_file = sys.stdout
433 if line.startswith('KEYWORDS'):
434 # Actually mark this file as stable by removing ~'s.
435 if make_stable:
436 redirect_file.write(line.replace('~', ''))
437 else:
438 redirect_file.write(line)
439 elif line.startswith('EAPI'):
440 # Always add new commit_id after EAPI definition.
441 redirect_file.write(line)
442 if commit_keyword and commit_value:
443 redirect_file.write('%s="%s"\n' % (commit_keyword, commit_value))
444 elif not line.startswith(commit_keyword):
445 # Skip old commit_keyword definition.
446 redirect_file.write(line)
447 fileinput.close()
448
449 def RevWorkOnEBuild(self, commit_id, redirect_file=None):
450 """Revs a workon ebuild given the git commit hash.
451
452 By default this class overwrites a new ebuild given the normal
453 ebuild rev'ing logic. However, a user can specify a redirect_file
454 to redirect the new stable ebuild to another file.
455
456 Args:
457 commit_id: String corresponding to the commit hash of the developer
458 package to rev.
459 redirect_file: Optional file to write the new ebuild. By default
460 it is written using the standard rev'ing logic. This file must be
461 opened and closed by the caller.
462
463 Raises:
464 OSError: Error occurred while creating a new ebuild.
465 IOError: Error occurred while writing to the new revved ebuild file.
466 Returns:
467 If the revved package is different than the old ebuild, return the full
468 revved package name, including the version number. Otherwise, return None.
469 """
470 if self._ebuild.is_stable:
471 stable_version_no_rev = self._ebuild.version_no_rev
472 else:
473 # If given unstable ebuild, use 0.0.1 rather than 9999.
474 stable_version_no_rev = '0.0.1'
475
476 new_version = '%s-r%d' % (stable_version_no_rev,
477 self._ebuild.current_revision + 1)
478 new_stable_ebuild_path = '%s-%s.ebuild' % (
479 self._ebuild.ebuild_path_no_version, new_version)
480
481 _Print('Creating new stable ebuild %s' % new_stable_ebuild_path)
482 unstable_ebuild_path = ('%s-9999.ebuild' %
483 self._ebuild.ebuild_path_no_version)
484 if not os.path.exists(unstable_ebuild_path):
485 Die('Missing unstable ebuild: %s' % unstable_ebuild_path)
486
487 self.MarkAsStable(unstable_ebuild_path, new_stable_ebuild_path,
488 'CROS_WORKON_COMMIT', commit_id, redirect_file)
489
490 old_ebuild_path = self._ebuild.ebuild_path
491 diff_cmd = ['diff', '-Bu', old_ebuild_path, new_stable_ebuild_path]
492 if 0 == RunCommand(diff_cmd, exit_code=True, redirect_stdout=True,
493 redirect_stderr=True, print_cmd=gflags.FLAGS.verbose):
494 os.unlink(new_stable_ebuild_path)
495 return None
496 else:
497 _Print('Adding new stable ebuild to git')
498 _SimpleRunCommand('git add %s' % new_stable_ebuild_path)
499
500 if self._ebuild.is_stable:
501 _Print('Removing old ebuild from git')
502 _SimpleRunCommand('git rm %s' % old_ebuild_path)
503
504 return '%s-%s' % (self._ebuild.package, new_version)
505
506 @classmethod
507 def CommitChange(cls, message):
508 """Commits current changes in git locally with given commit message.
509
510 Args:
511 message: the commit string to write when committing to git.
512
513 Raises:
514 OSError: Error occurred while committing.
515 """
516 Info('Committing changes with commit message: %s' % message)
517 git_commit_cmd = 'git commit -am "%s"' % message
518 _SimpleRunCommand(git_commit_cmd)
519
520
521 def main(argv):
522 try:
523 argv = gflags.FLAGS(argv)
524 if len(argv) != 2:
525 _PrintUsageAndDie('Must specify a valid command')
526 else:
527 command = argv[1]
528 except gflags.FlagsError, e :
529 _PrintUsageAndDie(str(e))
530
531 package_list = gflags.FLAGS.packages.split(':')
532 _CheckSaneArguments(package_list, command)
533 if gflags.FLAGS.overlays:
534 overlays = {}
535 for path in gflags.FLAGS.overlays.split(':'):
536 if command != 'clean' and not os.path.isdir(path):
537 Die('Cannot find overlay: %s' % path)
538 overlays[path] = []
539 else:
540 Warning('Missing --overlays argument')
541 overlays = {
542 '%s/private-overlays/chromeos-overlay' % gflags.FLAGS.srcroot: [],
543 '%s/third_party/chromiumos-overlay' % gflags.FLAGS.srcroot: []
544 }
545
546 if command == 'commit':
547 _BuildEBuildDictionary(overlays, gflags.FLAGS.all, package_list)
548
549 for overlay, ebuilds in overlays.items():
550 if not os.path.isdir(overlay):
551 Warning("Skipping %s" % overlay)
552 continue
553
554 # TODO(davidjames): Currently, all code that interacts with git depends on
555 # the cwd being set to the overlay directory. We should instead pass in
556 # this parameter so that we don't need to modify the cwd globally.
557 os.chdir(overlay)
558
559 if command == 'clean':
560 Clean(gflags.FLAGS.tracking_branch)
561 elif command == 'push':
562 PushChange(STABLE_BRANCH_NAME, gflags.FLAGS.tracking_branch)
563 elif command == 'commit' and ebuilds:
564 work_branch = GitBranch(STABLE_BRANCH_NAME, gflags.FLAGS.tracking_branch)
565 work_branch.CreateBranch()
566 if not work_branch.Exists():
567 Die('Unable to create stabilizing branch in %s' % overlay)
568
569 # Contains the array of packages we actually revved.
570 revved_packages = []
571 new_package_atoms = []
572 for ebuild in ebuilds:
573 try:
574 _Print('Working on %s' % ebuild.package)
575 worker = EBuildStableMarker(ebuild)
576 commit_id = ebuild.GetCommitId()
577 new_package = worker.RevWorkOnEBuild(commit_id)
578 if new_package:
579 message = _GIT_COMMIT_MESSAGE % (ebuild.package, commit_id)
580 worker.CommitChange(message)
581 revved_packages.append(ebuild.package)
582 new_package_atoms.append('=%s' % new_package)
583 except (OSError, IOError):
584 Warning('Cannot rev %s\n' % ebuild.package,
585 'Note you will have to go into %s '
586 'and reset the git repo yourself.' % overlay)
587 raise
588
589 _CleanStalePackages(gflags.FLAGS.board, new_package_atoms)
590 if gflags.FLAGS.drop_file:
591 fh = open(gflags.FLAGS.drop_file, 'w')
592 fh.write(' '.join(revved_packages))
593 fh.close()
594
595
596 if __name__ == '__main__':
597 main(sys.argv)
OLDNEW
« no previous file with comments | « cros_mark_as_stable ('k') | cros_mark_as_stable_blacklist » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698