| OLD | NEW |
|---|
| (Empty) |
| 1 #!/usr/bin/env python | |
| 2 # Copyright 2014 The Chromium Authors. All rights reserved. | |
| 3 # Use of this source code is governed by a BSD-style license that can be | |
| 4 # found in the LICENSE file. | |
| 5 | |
| 6 | |
| 7 """Automates creation and management of DEPS roll CLs. | |
| 8 | |
| 9 This script is designed to be run in a loop (eg. with auto_roll_wrapper.sh) or | |
| 10 on a timer. It may take one of several actions, depending on the state of | |
| 11 in-progress DEPS roll CLs and the state of the repository: | |
| 12 | |
| 13 - If there is already a DEPS roll CL in the Commit Queue, just exit. | |
| 14 - If there is an open DEPS roll CL which is not in the Commit Queue: | |
| 15 - If there's a comment containing the "STOP" keyword, just exit. | |
| 16 - Otherwise, close the issue and continue. | |
| 17 - If there is no open DEPS roll CL, create one using the | |
| 18 src/tools/safely-roll-deps.py script. | |
| 19 """ | |
| 20 | |
| 21 | |
| 22 import datetime | |
| 23 import json | |
| 24 import optparse | |
| 25 import os.path | |
| 26 import re | |
| 27 import sys | |
| 28 import textwrap | |
| 29 import urllib2 | |
| 30 | |
| 31 | |
| 32 SCRIPTS_DIR = os.path.realpath(os.path.join(os.path.dirname(__file__), | |
| 33 os.pardir, os.pardir)) | |
| 34 sys.path.insert(0, SCRIPTS_DIR) | |
| 35 | |
| 36 # pylint: disable=W0611 | |
| 37 from common import find_depot_tools | |
| 38 | |
| 39 import auth | |
| 40 import rietveld | |
| 41 import scm | |
| 42 import subprocess2 | |
| 43 | |
| 44 | |
| 45 ROLL_DESCRIPTION_STR = ( | |
| 46 '''Roll %(dep_path)s %(before_rev)s:%(after_rev)s%(svn_range)s | |
| 47 | |
| 48 Summary of changes available at: | |
| 49 %(revlog_url)s | |
| 50 ''') | |
| 51 | |
| 52 NO_TRY_STR = ( | |
| 53 ''' | |
| 54 Sheriffs: In case of breakage, do NOT revert this roll, revert the | |
| 55 offending commit in the %(project)s repository instead. | |
| 56 | |
| 57 NOTRY=true''') | |
| 58 | |
| 59 BLINK_SHERIFF_URL = ( | |
| 60 'http://build.chromium.org/p/chromium.webkit/sheriff_webkit.js') | |
| 61 CHROMIUM_SHERIFF_URL = ( | |
| 62 'http://build.chromium.org/p/chromium.webkit/sheriff.js') | |
| 63 | |
| 64 CQ_INCLUDE_TRYBOTS = 'CQ_INCLUDE_TRYBOTS=' | |
| 65 | |
| 66 # Does not support unicode or special characters. | |
| 67 VALID_EMAIL_REGEXP = re.compile(r'^[A-Za-z0-9\.&\'\+-/=_]+@' | |
| 68 r'[A-Za-z0-9\.\-]+$') | |
| 69 | |
| 70 | |
| 71 def _get_skia_sheriff(): | |
| 72 """Finds and returns the current Skia sheriff.""" | |
| 73 skia_sheriff_url = 'https://skia-tree-status.appspot.com/current-sheriff' | |
| 74 return json.load(urllib2.urlopen(skia_sheriff_url))['username'] | |
| 75 | |
| 76 | |
| 77 def _complete_email(name): | |
| 78 """If the name does not include '@', append '@chromium.org'.""" | |
| 79 if '@' not in name: | |
| 80 return name + '@chromium.org' | |
| 81 return name | |
| 82 | |
| 83 | |
| 84 def _names_from_sheriff_js(sheriff_js): | |
| 85 match = re.match(r'document.write\(\'(.*)\'\)', sheriff_js) | |
| 86 emails_string = match.group(1) | |
| 87 # Detect 'none (channel is sheriff)' text and ignore it. | |
| 88 if 'channel is sheriff' in emails_string.lower(): | |
| 89 return [] | |
| 90 return map(str.strip, emails_string.split(',')) | |
| 91 | |
| 92 | |
| 93 def _email_is_valid(email): | |
| 94 """Determines whether the given email address is valid.""" | |
| 95 return VALID_EMAIL_REGEXP.match(email) is not None | |
| 96 | |
| 97 | |
| 98 def _filter_emails(emails): | |
| 99 """Returns the given list with any invalid email addresses removed.""" | |
| 100 rv = [] | |
| 101 for email in emails: | |
| 102 if _email_is_valid(email): | |
| 103 rv.append(email) | |
| 104 else: | |
| 105 print 'WARNING: Not including %s (invalid email address)' % email | |
| 106 return rv | |
| 107 | |
| 108 | |
| 109 def _emails_from_url(sheriff_url): | |
| 110 sheriff_js = urllib2.urlopen(sheriff_url).read() | |
| 111 return map(_complete_email, _names_from_sheriff_js(sheriff_js)) | |
| 112 | |
| 113 | |
| 114 def _current_gardener_emails(): | |
| 115 return _emails_from_url(BLINK_SHERIFF_URL) | |
| 116 | |
| 117 | |
| 118 def _current_sheriff_emails(): | |
| 119 return _emails_from_url(CHROMIUM_SHERIFF_URL) | |
| 120 | |
| 121 | |
| 122 def _do_git_fetch(git_dir): | |
| 123 subprocess2.check_call(['git', '--git-dir', git_dir, 'fetch']) | |
| 124 | |
| 125 | |
| 126 PROJECT_CONFIGS = { | |
| 127 'blink': { | |
| 128 'extra_emails_fn': _current_gardener_emails, | |
| 129 'path_to_project': os.path.join('third_party', 'WebKit'), | |
| 130 'project_alias': 'webkit', | |
| 131 }, | |
| 132 'skia': { | |
| 133 'cq_extra_trybots': ['tryserver.blink:linux_blink_rel'], | |
| 134 'extra_emails_fn': lambda: [_get_skia_sheriff()], | |
| 135 'path_to_project': os.path.join('third_party', 'skia'), | |
| 136 'include_commit_log': True, | |
| 137 # 'dry_run': True, | |
| 138 }, | |
| 139 } | |
| 140 | |
| 141 | |
| 142 class AutoRollException(Exception): | |
| 143 pass | |
| 144 | |
| 145 | |
| 146 class AutoRoller(object): | |
| 147 RIETVELD_URL = 'https://codereview.chromium.org' | |
| 148 RIETVELD_TIME_FORMAT = '%Y-%m-%d %H:%M:%S.%f' | |
| 149 ROLL_TIME_LIMIT = datetime.timedelta(hours=24) | |
| 150 STOP_NAG_TIME_LIMIT = datetime.timedelta(hours=12) | |
| 151 ADMIN_EMAIL = 'dpranke@chromium.org' | |
| 152 | |
| 153 ROLL_DESCRIPTION_REGEXP = ROLL_DESCRIPTION_STR % { | |
| 154 'dep_path': '%(project)s', | |
| 155 'before_rev': '(?P<from_revision>[0-9a-fA-F]{2,40})', | |
| 156 'after_rev': '(?P<to_revision>[0-9a-fA-F]{2,40})', | |
| 157 'svn_range': '.*', | |
| 158 'revlog_url': '.+', | |
| 159 } | |
| 160 | |
| 161 # FIXME: These are taken from gardeningserver.py and should be shared. | |
| 162 CHROMIUM_SVN_DEPS_URL = 'http://src.chromium.org/chrome/trunk/src/DEPS' | |
| 163 | |
| 164 ROLL_BOT_INSTRUCTIONS = textwrap.dedent( | |
| 165 '''This roll was created by the Blink AutoRollBot. | |
| 166 http://www.chromium.org/blink/blinkrollbot''') | |
| 167 | |
| 168 PLEASE_RESUME_NAG = textwrap.dedent(''' | |
| 169 Rollbot was stopped by the presence of 'STOP' in an earlier comment. | |
| 170 The last update to this issue was over %(stop_nag_timeout)s hours ago. | |
| 171 Please close this issue as soon as possible to allow the bot to continue. | |
| 172 | |
| 173 Please email (%(admin)s) if the Rollbot is causing trouble. | |
| 174 ''' % {'admin': ADMIN_EMAIL, 'stop_nag_timeout': STOP_NAG_TIME_LIMIT}) | |
| 175 | |
| 176 def __init__(self, project, author, path_to_chrome, auth_config=None, | |
| 177 options=None): | |
| 178 self._author = author | |
| 179 self._project = project | |
| 180 self._path_to_chrome = path_to_chrome | |
| 181 self._rietveld = rietveld.Rietveld( | |
| 182 self.RIETVELD_URL, auth_config, self._author) | |
| 183 self._notry = options and options.notry | |
| 184 self._cached_last_roll_revision = None | |
| 185 | |
| 186 project_config = PROJECT_CONFIGS.get(self._project, { | |
| 187 'path_to_project': os.path.join('third_party', self._project), | |
| 188 }) | |
| 189 self._project_alias = project_config.get('project_alias', self._project) | |
| 190 self._path_to_project = project_config['path_to_project'] | |
| 191 self._get_extra_emails = project_config.get('extra_emails_fn', lambda: []) | |
| 192 self._cq_extra_trybots = project_config.get('cq_extra_trybots', []) | |
| 193 self._include_commit_log = project_config.get('include_commit_log', False) | |
| 194 self._cq_dry_run = project_config.get('dry_run', False) | |
| 195 | |
| 196 self._chromium_git_dir = self._path_from_chromium_root('.git') | |
| 197 self._project_git_dir = self._path_from_chromium_root( | |
| 198 self._path_to_project, '.git') | |
| 199 self._roll_description_regexp = (self.ROLL_DESCRIPTION_REGEXP % { | |
| 200 'project': 'src/' + self._path_to_project | |
| 201 }).splitlines()[0] | |
| 202 | |
| 203 def _parse_time(self, time_string): | |
| 204 return datetime.datetime.strptime(time_string, self.RIETVELD_TIME_FORMAT) | |
| 205 | |
| 206 def _url_for_issue(self, issue_number): | |
| 207 return '%s/%d/' % (self.RIETVELD_URL, issue_number) | |
| 208 | |
| 209 def _search_for_active_roll(self): | |
| 210 # FIXME: Rietveld.search is broken, we should use closed=False | |
| 211 # but that sends closed=1, we want closed=3. Using closed=2 | |
| 212 # to that search translates it correctly to closed=3 internally. | |
| 213 # https://code.google.com/p/chromium/issues/detail?id=242628 | |
| 214 for result in self._rietveld.search(owner=self._author, closed=2): | |
| 215 if re.search(self._roll_description_regexp, result['subject']): | |
| 216 return result | |
| 217 return None | |
| 218 | |
| 219 def _rollbot_should_stop(self, issue): | |
| 220 issue_number = issue['issue'] | |
| 221 for message in issue['messages']: | |
| 222 if 'STOP' in message['text']: | |
| 223 last_modified = self._parse_time(issue['modified']) | |
| 224 time_since_last_comment = datetime.datetime.utcnow() - last_modified | |
| 225 if time_since_last_comment > self.STOP_NAG_TIME_LIMIT: | |
| 226 self._rietveld.add_comment(issue_number, self.PLEASE_RESUME_NAG) | |
| 227 | |
| 228 print '%s: Rollbot was stopped by %s on at %s, waiting.' % ( | |
| 229 self._url_for_issue(issue_number), message['sender'], message['date']) | |
| 230 return True | |
| 231 return False | |
| 232 | |
| 233 def _close_issue(self, issue_number, message=None): | |
| 234 print 'Closing %s with message: \'%s\'' % ( | |
| 235 self._url_for_issue(issue_number), message) | |
| 236 if message: | |
| 237 self._rietveld.add_comment(issue_number, message) | |
| 238 self._rietveld.close_issue(issue_number) | |
| 239 | |
| 240 def _path_from_chromium_root(self, *components): | |
| 241 assert os.pardir not in components | |
| 242 return os.path.join(self._path_to_chrome, *components) | |
| 243 | |
| 244 def _last_roll_revision(self): | |
| 245 """Returns the revision of the last roll. | |
| 246 | |
| 247 Returns: | |
| 248 revision of the last roll; either a 40-character Git commit hash or an | |
| 249 SVN revision number. | |
| 250 """ | |
| 251 if not self._cached_last_roll_revision: | |
| 252 revinfo = subprocess2.check_output(['gclient', 'revinfo'], | |
| 253 cwd=self._path_to_chrome) | |
| 254 project_path = 'src/' + self._path_to_project | |
| 255 for line in revinfo.splitlines(): | |
| 256 dep_path, source = line.split(': ', 1) | |
| 257 if dep_path == project_path: | |
| 258 self._cached_last_roll_revision = source.split('@')[-1] | |
| 259 break | |
| 260 assert len(self._cached_last_roll_revision) == 40 | |
| 261 return self._cached_last_roll_revision | |
| 262 | |
| 263 def _current_revision(self): | |
| 264 git_revparse_cmd = ['git', '--git-dir', self._project_git_dir, | |
| 265 'rev-parse', 'origin/master'] | |
| 266 return subprocess2.check_output(git_revparse_cmd).rstrip() | |
| 267 | |
| 268 def _emails_to_cc_on_rolls(self): | |
| 269 return _filter_emails(self._get_extra_emails()) | |
| 270 | |
| 271 def _start_roll(self, last_roll_revision, new_roll_revision): | |
| 272 roll_branch = '%s_roll' % self._project | |
| 273 cwd_kwargs = {'cwd': self._path_to_chrome} | |
| 274 subprocess2.check_call(['git', 'clean', '-d', '-f'], **cwd_kwargs) | |
| 275 subprocess2.call(['git', 'rebase', '--abort'], **cwd_kwargs) | |
| 276 subprocess2.call(['git', 'branch', '-D', roll_branch], **cwd_kwargs) | |
| 277 subprocess2.check_call(['git', 'checkout', 'origin/master', '-f'], | |
| 278 **cwd_kwargs) | |
| 279 subprocess2.check_call(['git', 'checkout', '-b', roll_branch, | |
| 280 '-t', 'origin/master', '-f'], **cwd_kwargs) | |
| 281 try: | |
| 282 subprocess2.check_call(['roll-dep-svn', self._path_to_project, | |
| 283 new_roll_revision], **cwd_kwargs) | |
| 284 subprocess2.check_call(['git', 'add', 'DEPS'], **cwd_kwargs) | |
| 285 subprocess2.check_call(['git', 'commit', '--no-edit'], **cwd_kwargs) | |
| 286 commit_msg = subprocess2.check_output( | |
| 287 ['git', 'log', '-n1', '--format=%B', 'HEAD'], | |
| 288 **cwd_kwargs).decode('utf-8') | |
| 289 | |
| 290 if self._notry: | |
| 291 commit_msg += NO_TRY_STR % { 'project': self._project } | |
| 292 | |
| 293 upload_cmd = ['git', 'cl', 'upload', '--bypass-hooks', '-f'] | |
| 294 if self._cq_dry_run: | |
| 295 upload_cmd.append('--cq-dry-run') | |
| 296 else: | |
| 297 upload_cmd.append('--use-commit-queue') | |
| 298 if self._cq_extra_trybots: | |
| 299 commit_msg += ('\n' + CQ_INCLUDE_TRYBOTS + | |
| 300 ','.join(self._cq_extra_trybots)) | |
| 301 tbr = '\nTBR=' | |
| 302 emails = self._emails_to_cc_on_rolls() | |
| 303 if emails: | |
| 304 emails_str = ','.join(emails) | |
| 305 tbr += emails_str | |
| 306 upload_cmd.extend(['--cc', emails_str, '--send-mail']) | |
| 307 commit_msg += tbr | |
| 308 if self._include_commit_log: | |
| 309 log_cmd = ['git', 'log', '--format=%h %ae %s', | |
| 310 '%s..%s' % (last_roll_revision, new_roll_revision)] | |
| 311 git_log = subprocess2.check_output(log_cmd, cwd=self._project_git_dir) | |
| 312 commit_msg += '\n\nCommits in this roll:\n' + git_log.decode('utf-8') | |
| 313 upload_cmd.extend(['-m', commit_msg]) | |
| 314 subprocess2.check_call(upload_cmd, **cwd_kwargs) | |
| 315 finally: | |
| 316 subprocess2.check_call(['git', 'checkout', 'origin/master', '-f'], | |
| 317 **cwd_kwargs) | |
| 318 subprocess2.check_call( | |
| 319 ['git', 'branch', '-D', roll_branch], **cwd_kwargs) | |
| 320 | |
| 321 # FIXME: It's easier to pull the issue id from rietveld rather than | |
| 322 # parse it from the safely-roll-deps output. Once we inline | |
| 323 # safely-roll-deps into this script this can go away. | |
| 324 search_result = self._search_for_active_roll() | |
| 325 if search_result: | |
| 326 self._rietveld.add_comment(search_result['issue'], | |
| 327 self.ROLL_BOT_INSTRUCTIONS) | |
| 328 | |
| 329 def _maybe_close_active_roll(self, issue): | |
| 330 issue_number = issue['issue'] | |
| 331 | |
| 332 # If the CQ failed, this roll is DOA. | |
| 333 if not issue['commit']: | |
| 334 self._close_issue( | |
| 335 issue_number, | |
| 336 'No longer marked for the CQ. Closing, will open a new roll.') | |
| 337 return True | |
| 338 | |
| 339 create_time = self._parse_time(issue['created']) | |
| 340 time_since_roll = datetime.datetime.utcnow() - create_time | |
| 341 print '%s started %s ago' % ( | |
| 342 self._url_for_issue(issue_number), time_since_roll) | |
| 343 if time_since_roll > self.ROLL_TIME_LIMIT: | |
| 344 self._close_issue( | |
| 345 issue_number, | |
| 346 'Giving up on this roll after %s. Closing, will open a new roll.' % | |
| 347 self.ROLL_TIME_LIMIT) | |
| 348 return True | |
| 349 | |
| 350 last_roll_revision = self._short_rev(self._last_roll_revision()) | |
| 351 match = re.match(self._roll_description_regexp, issue['subject']) | |
| 352 if match.group('from_revision') != last_roll_revision: | |
| 353 self._close_issue( | |
| 354 issue_number, | |
| 355 'DEPS has already rolled to %s. Closing, will open a new roll.' % | |
| 356 last_roll_revision) | |
| 357 return True | |
| 358 | |
| 359 return False | |
| 360 | |
| 361 def _compare_revisions(self, last_roll_revision, new_roll_revision): | |
| 362 """Ensure that new_roll_revision is newer than last_roll_revision.""" | |
| 363 # Ensure that new_roll_revision is not an ancestor of old_roll_revision. | |
| 364 try: | |
| 365 subprocess2.check_call(['git', '--git-dir', self._project_git_dir, | |
| 366 'merge-base', '--is-ancestor', | |
| 367 new_roll_revision, last_roll_revision]) | |
| 368 print ('Already at %s refusing to roll backwards to %s.' % ( | |
| 369 last_roll_revision, new_roll_revision)) | |
| 370 return False | |
| 371 except subprocess2.CalledProcessError: | |
| 372 pass | |
| 373 return True | |
| 374 | |
| 375 def _short_rev(self, revision): | |
| 376 """Shorten a Git commit hash.""" | |
| 377 return subprocess2.check_output(['git', '--git-dir', self._project_git_dir, | |
| 378 'rev-parse', '--short', revision] | |
| 379 ).rstrip() | |
| 380 | |
| 381 def main(self): | |
| 382 _do_git_fetch(self._chromium_git_dir) | |
| 383 _do_git_fetch(self._project_git_dir) | |
| 384 | |
| 385 search_result = self._search_for_active_roll() | |
| 386 issue_number = search_result['issue'] if search_result else None | |
| 387 if issue_number: | |
| 388 issue = self._rietveld.get_issue_properties(issue_number, messages=True) | |
| 389 else: | |
| 390 issue = None | |
| 391 | |
| 392 if issue: | |
| 393 if self._rollbot_should_stop(issue): | |
| 394 return 1 | |
| 395 if not self._maybe_close_active_roll(issue): | |
| 396 print '%s is still active, nothing to do.' % \ | |
| 397 self._url_for_issue(issue_number) | |
| 398 return 0 | |
| 399 | |
| 400 last_roll_revision = self._last_roll_revision() | |
| 401 new_roll_revision = self._current_revision() | |
| 402 | |
| 403 if not new_roll_revision: | |
| 404 raise AutoRollException( | |
| 405 'Could not determine the current revision.') | |
| 406 | |
| 407 if not self._compare_revisions(last_roll_revision, new_roll_revision): | |
| 408 return 0 | |
| 409 | |
| 410 self._start_roll(last_roll_revision, new_roll_revision) | |
| 411 return 0 | |
| 412 | |
| 413 | |
| 414 def main(): | |
| 415 usage = 'Usage: %prog project_name author path_to_chromium' | |
| 416 | |
| 417 # The default HelpFormatter causes the docstring to display improperly. | |
| 418 class VanillaHelpFormatter(optparse.IndentedHelpFormatter): | |
| 419 def format_description(self, description): | |
| 420 if description: | |
| 421 return description | |
| 422 else: | |
| 423 return '' | |
| 424 | |
| 425 parser = optparse.OptionParser(usage=usage, | |
| 426 description=sys.modules[__name__].__doc__, | |
| 427 formatter=VanillaHelpFormatter()) | |
| 428 | |
| 429 parser.add_option('--no-try', action='store_true', dest='notry', | |
| 430 help='Create the CL with NOTRY=true') | |
| 431 | |
| 432 auth.add_auth_options(parser) | |
| 433 options, args = parser.parse_args() | |
| 434 auth_config = auth.extract_auth_config_from_options(options) | |
| 435 if len(args) != 3: | |
| 436 parser.print_usage() | |
| 437 return 1 | |
| 438 | |
| 439 AutoRoller(*args, auth_config=auth_config, options=options).main() | |
| 440 | |
| 441 | |
| 442 if __name__ == '__main__': | |
| 443 sys.exit(main()) | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 2205383002:
Remove Blink auto-roll script (scripts/tools/blink_roller/). (Closed)
Base URL: https://chromium.googlesource.com/chromium/tools/build.git@master