Auto-roll script for WebRTC.

tools/roll_webrtc.py

+#!/usr/bin/env python
+# Copyright 2015 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import argparse
+import collections
+import logging
+import os
+import re
+import subprocess
+import sys
+import time
+
+
+SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
+SRC_DIR = os.path.abspath(os.path.join(SCRIPT_DIR, os.pardir))
+import find_depot_tools
+find_depot_tools.add_depot_tools_to_path()
+import rietveld
+import roll_dep
+from gclient import GClientKeywords
+from third_party import upload
+
+# Avoid depot_tools/third_party/upload.py print verbose messages.
+upload.verbosity = 0  # Errors only.
+
+CHROMIUM_GIT_URL = 'https://chromium.googlesource.com/chromium/src.git'
+COMMIT_POSITION_RE = re.compile('^Cr-Original-Commit-Position: .*#([0-9]+).*$')
+CL_ISSUE_RE = re.compile('^Issue number: ([0-9]+) \((.*)\)$')
+RIETVELD_URL_RE = re.compile('^https?://(.*)/(.*)')
+ROLL_BRANCH_NAME = 'special_webrtc_roll_branch'
+TRYJOB_STATUS_SLEEP_SECONDS = 30
+
+# Use a shell for subcommands on Windows to get a PATH search.
+USE_SHELL = sys.platform.startswith('win')
+WEBRTC_PATH = 'third_party/webrtc'
+LIBJINGLE_PATH = 'third_party/libjingle/source/talk'
+LIBJINGLE_README = 'third_party/libjingle/README.chromium'
+
+# Result codes from build/third_party/buildbot_8_4p1/buildbot/status/results.py
+# plus the -1 code which is used when there's no result yet.
+TRYJOB_STATUS = {
+  -1: 'RUNNING',
+  0: 'SUCCESS',
+  1: 'WARNINGS',
+  2: 'FAILURE',
+  3: 'SKIPPED',
+  4: 'EXCEPTION',
+  5: 'RETRY',
+}
+SUCCESS_STATUS = (0, 1, 3)
+FAILURE_STATUS = (2, 4, 5)
+
+CommitInfo = collections.namedtuple('CommitInfo', ['commit_position',
+                                                   'git_commit',
+                                                   'git_repo_url'])
+CLInfo = collections.namedtuple('CLInfo', ['issue', 'url', 'rietveld_server'])
+
+
+def _ParseGitCommitPosition(description):
+  for line in reversed(description.splitlines()):
+    m = COMMIT_POSITION_RE.match(line.strip())
+    if m:
+      return m.group(1)
+  logging.error('Failed to parse svn revision id from:\n%s\n', description)
+  sys.exit(-1)
+
+
+def _ParseGitCommitHash(description):
+  for line in description.splitlines():
+    if line.startswith('commit '):
+      return line.split()[1]
+  logging.error('Failed to parse git commit id from:\n%s\n', description)
+  sys.exit(-1)
+  return None
+
+
+def _ParseDepsFile(filename):
+  with open(filename, 'rb') as f:
+    deps_content = f.read()
+  return _ParseDepsDict(deps_content)
+
+
+def _ParseDepsDict(deps_content):
+  local_scope = {}
+  var = GClientKeywords.VarImpl({}, local_scope)
+  global_scope = {
+    'File': GClientKeywords.FileImpl,
+    'From': GClientKeywords.FromImpl,
+    'Var': var.Lookup,
+    'deps_os': {},
+  }
+  exec(deps_content, global_scope, local_scope)
+  return local_scope
+
+
+def _WaitForTrybots(issue, rietveld_server):
+  """Wait until all trybots have passed or at least one have failed.
+
+  Returns:
+    An exit code of 0 if all trybots passed or non-zero otherwise.
+  """
+  assert type(issue) is int
+  print 'Trybot status for https://%s/%d:' % (rietveld_server, issue)
+  remote = rietveld.Rietveld('https://' + rietveld_server, None, None)
+
+  attempt = 0
+  max_tries = 60*60/TRYJOB_STATUS_SLEEP_SECONDS # Max one hour
+  while attempt < max_tries:
+    # Get patches for the issue so we can use the latest one.
+    data = remote.get_issue_properties(issue, messages=False)
+    patchsets = data['patchsets']
+
+    # Get trybot status for the latest patch set.
+    data = remote.get_patchset_properties(issue, patchsets[-1])
+
+    tryjob_results = data['try_job_results']
+    if len(tryjob_results) == 0:
+      logging.debug('No trybots have yet been triggered for https://%s/%d' ,
+                    rietveld_server, issue)
+    else:
+      _PrintTrybotsStatus(tryjob_results)
+      if any(r['result'] in FAILURE_STATUS for r in tryjob_results):
+        logging.error('Found failing tryjobs (see above)')
+        return 1
+      if all(r['result'] in SUCCESS_STATUS for r in tryjob_results):
+        return 0
+
+    logging.debug('Waiting for %d seconds before next check...',
+                  TRYJOB_STATUS_SLEEP_SECONDS)
+    time.sleep(TRYJOB_STATUS_SLEEP_SECONDS)
+    attempt += 1
+
+
+def _PrintTrybotsStatus(tryjob_results):
+  status_to_name = {}
+  for trybot_result in tryjob_results:
+    status = TRYJOB_STATUS.get(trybot_result['result'], 'UNKNOWN')
+    status_to_name.setdefault(status, [])
+    status_to_name[status].append(trybot_result['builder'])
+
+  print '\n========== TRYJOBS STATUS =========='
+  for status,name_list in status_to_name.iteritems():
+    print '%s: %s' % (status, ','.join(sorted(name_list)))
+
+
+def _GenerateCLDescription(webrtc_current, libjingle_current,
+                           webrtc_new, libjingle_new):
+  delim = ''
+  webrtc_str = ''
+  def GetChangeLogURL(git_repo_url, current_hash, new_hash):
+    return '%s/+log/%s..%s' % (git_repo_url, current_hash[0:7], new_hash[0:7])
+
+  if webrtc_current.git_commit != webrtc_new.git_commit:
+    webrtc_str = 'WebRTC %s:%s' % (webrtc_current.commit_position,
+                                   webrtc_new.commit_position)
+    webrtc_changelog_url = GetChangeLogURL(webrtc_current.git_repo_url,
+                                           webrtc_current.git_commit,
+                                           webrtc_new.git_commit)
+
+  libjingle_str = ''
+  if libjingle_current.git_commit != libjingle_new.git_commit:
+    if webrtc_str:
+      delim += ', '
+    libjingle_str = 'Libjingle %s:%s' % (libjingle_current.commit_position,
+                                         libjingle_new.commit_position)
+    libjingle_changelog_url = GetChangeLogURL(libjingle_current.git_repo_url,
+                                              libjingle_current.git_commit,
+                                              libjingle_new.git_commit)
+
+  description = 'Roll ' + webrtc_str + delim + libjingle_str + '\n\n'
+  if webrtc_str:
+    description += webrtc_str + '\n'
+    description += 'Changes: %s\n\n' % webrtc_changelog_url
+  if libjingle_str:
+    description += libjingle_str + '\n'
+    description += 'Changes: %s\n' % libjingle_changelog_url
+  description += '\nTBR='
+  return description
+
+
+class AutoRoller(object):
+  def __init__(self, chromium_src):
+    self._chromium_src = chromium_src
+
+  def _RunCommand(self, command, working_dir=None, ignore_exit_code=False,
+                  extra_env=None):
+    """Runs a command and returns the stdout from that command.
+
+    If the command fails (exit code != 0), the function will exit the process.
+    """
+    working_dir = working_dir or self._chromium_src
+    logging.debug('cmd: %s cwd: %s', ' '.join(command), working_dir)
+    env = os.environ.copy()
+    if extra_env:
+      logging.debug('extra env: %s', extra_env)
+      env.update(extra_env)
+    p = subprocess.Popen(command, stdout=subprocess.PIPE,
+                         stderr=subprocess.PIPE, shell=USE_SHELL, env=env,
+                         cwd=working_dir, universal_newlines=True)
+    output = p.stdout.read()
+    p.wait()
+    p.stdout.close()
+    p.stderr.close()
+
+    if not ignore_exit_code and p.returncode != 0:
+      logging.error('Command failed: %s\n%s', str(command), output)
+      sys.exit(p.returncode)
+    return output
+
+  def _GetCommitInfo(self, path_below_src, git_hash=None, git_repo_url=None):
+    working_dir = os.path.join(self._chromium_src, path_below_src)
+    self._RunCommand(['git', 'fetch', 'origin'], working_dir=working_dir)
+    revision_range = git_hash or 'origin'
+    ret = self._RunCommand(
+        ['git', '--no-pager', 'log', revision_range, '--pretty=full', '-1'],
+        working_dir=working_dir)
+    return CommitInfo(_ParseGitCommitPosition(ret), _ParseGitCommitHash(ret),
+                      git_repo_url)
+
+  def _GetDepsCommitInfo(self, deps_dict, path_below_src):
+    entry = deps_dict['deps']['src/%s' % path_below_src]
+    at_index = entry.find('@')
+    git_repo_url = entry[:at_index]
+    git_hash = entry[at_index + 1:]
+    return self._GetCommitInfo(path_below_src, git_hash, git_repo_url)
+
+  def _GetCLInfo(self):
+    cl_output = self._RunCommand(['git', 'cl', 'issue'])
+    m = CL_ISSUE_RE.match(cl_output.strip())
+    if not m:
+      logging.error('Cannot find any CL info. Output was:\n%s', cl_output)
+      sys.exit(-1)
+    issue_number = int(m.group(1))
+    url = m.group(2)
+
+    # Parse the Rietveld host from the URL.
+    m = RIETVELD_URL_RE.match(url)
+    if not m:
+      logging.error('Cannot parse Rietveld host from URL: %s', url)
+      sys.exit(-1)
+    rietveld_server = m.group(1)
+    return CLInfo(issue_number, url, rietveld_server)
+
+  def _GetCurrentBranchName(self):
+    return self._RunCommand(
+        ['git', 'rev-parse', '--abbrev-ref', 'HEAD']).splitlines()[0]
+
+  def _IsTreeClean(self):
+    lines = self._RunCommand(['git', 'status', '--porcelain']).splitlines()
+    if len(lines) == 0:
+      return True
+
+    logging.debug('Dirty/unversioned files:\n%s', '\n'.join(lines))
+    return False
+
+  def _UpdateReadmeFile(self, readme_path, new_revision):
+    readme = open(os.path.join(self._chromium_src, readme_path), 'r+')
+    txt = readme.read()
+    m = re.sub(re.compile('.*^Revision\: ([0-9]*).*', re.MULTILINE),
+        ('Revision: %s' % new_revision), txt)
+    readme.seek(0)
+    readme.write(m)
+    readme.truncate()
+
+  def PrepareRoll(self, dry_run, ignore_checks, no_commit, close_previous_roll):
+    # TODO(kjellander): use os.path.normcase, os.path.join etc for all paths for
+    # cross platform compatibility.
+
+    if not ignore_checks:
+      if self._GetCurrentBranchName() != 'master':
+        logging.error('Please checkout the master branch.')
+        return -1
+      if not self._IsTreeClean():
+        logging.error('Please make sure you don\'t have any modified files.')
+        return -1
+
+    logging.debug('Checking for a previous roll branch.')
+    if close_previous_roll:
+      self.Abort()
+
+    logging.debug('Pulling latest changes')
+    if not ignore_checks:
+      self._RunCommand(['git', 'pull'])
+
+    self._RunCommand(['git', 'checkout', '-b', ROLL_BRANCH_NAME])
+
+    # Modify Chromium's DEPS file.
+
+    # Parse current hashes.
+    deps_filename = os.path.join(self._chromium_src, 'DEPS')
+    deps = _ParseDepsFile(deps_filename)
+    webrtc_current = self._GetDepsCommitInfo(deps, WEBRTC_PATH)
+    libjingle_current = self._GetDepsCommitInfo(deps, LIBJINGLE_PATH)
+
+    # Find ToT revisions.
+    webrtc_latest = self._GetCommitInfo(WEBRTC_PATH)
+    libjingle_latest = self._GetCommitInfo(LIBJINGLE_PATH)
+
+    self._UpdateDep(deps_filename, WEBRTC_PATH, webrtc_latest)
+    self._UpdateDep(deps_filename, LIBJINGLE_PATH, libjingle_latest)
+
+    if self._IsTreeClean():
+      logging.debug('Tree is clean - no changes detected.')
+      self._DeleteRollBranch()
+    else:
+      self._UpdateReadmeFile(LIBJINGLE_README, libjingle_latest.commit_position)
+      description = _GenerateCLDescription(webrtc_current, libjingle_current,
+                                           webrtc_latest, libjingle_latest)
+      logging.debug('Committing changes locally.')
+      self._RunCommand(['git', 'add', '--update', '.'])
+      self._RunCommand(['git', 'commit', '-m', description])
+      logging.debug('Uploading changes...')
+      self._RunCommand(['git', 'cl', 'upload', '-m', description],
+                       extra_env={'EDITOR': 'true'})
+      cl_info = self._GetCLInfo()
+      logging.debug('Issue: %d URL: %s', cl_info.issue, cl_info.url)
+
+      if not dry_run and not no_commit:
+        logging.debug('Sending the CL to the CQ...')
+        self._RunCommand(['git', 'cl', 'set_commit'])
+        logging.debug('Sent the CL to the CQ. Monitor here: %s', cl_info.url)
+
+    # TODO(kjellander): Checkout masters/previous branches again.
+    return 0
+
+  def _UpdateDep(self, deps_filename, dep_relative_to_src, commit_info):
+    dep_name = os.path.join('src', dep_relative_to_src)
+    comment = 'commit position %s' % commit_info.commit_position
+
+    # roll_dep.py relies on cwd being the Chromium checkout, so let's
+    # temporarily change the working directory and then change back.
+    cwd = os.getcwd()
+    os.chdir(os.path.dirname(deps_filename))
+    roll_dep.update_deps(deps_filename, dep_relative_to_src, dep_name,
+                         commit_info.git_commit, comment)
+    os.chdir(cwd)
+
+  def _DeleteRollBranch(self):
+    self._RunCommand(['git', 'checkout', 'master'])
+    self._RunCommand(['git', 'branch', '-D', ROLL_BRANCH_NAME])
+    logging.debug('Deleted the local roll branch (%s)', ROLL_BRANCH_NAME)
+
+
+  def _GetBranches(self):
+    """Returns a tuple of active,branches.
+
+    The 'active' is the name of the currently active branch and 'branches' is a
+    list of all branches.
+    """
+    lines = self._RunCommand(['git', 'branch']).split('\n')
+    branches = []
+    active = ''
+    for l in lines:
+      if '*' in l:
+        # The assumption is that the first char will always be the '*'.
+        active = l[1:].strip()
+        branches.append(active)
+      else:
+        b = l.strip()
+        if b:
+          branches.append(b)
+    return (active, branches)
+
+  def Abort(self):
+    active_branch, branches = self._GetBranches()
+    if active_branch == ROLL_BRANCH_NAME:
+      active_branch = 'master'
+    if ROLL_BRANCH_NAME in branches:
+      print 'Aborting pending roll.'
+      self._RunCommand(['git', 'checkout', ROLL_BRANCH_NAME])
+      # Ignore an error here in case an issue wasn't created for some reason.
+      self._RunCommand(['git', 'cl', 'set_close'], ignore_exit_code=True)
+      self._RunCommand(['git', 'checkout', active_branch])
+      self._RunCommand(['git', 'branch', '-D', ROLL_BRANCH_NAME])
+    return 0
+
+  def WaitForTrybots(self):
+    active_branch, _ = self._GetBranches()
+    if active_branch != ROLL_BRANCH_NAME:
+      self._RunCommand(['git', 'checkout', ROLL_BRANCH_NAME])
+    cl_info = self._GetCLInfo()
+    return _WaitForTrybots(cl_info.issue, cl_info.rietveld_server)
+
+
+def main():
+  if sys.platform in ('win32', 'cygwin'):
+    logging.error('Only Linux and Mac platforms are supported right now.')
+    return -1
+
+  parser = argparse.ArgumentParser(
+      description='Find webrtc and libjingle revisions for roll.')
+  parser.add_argument('--abort',
+    help=('Aborts a previously prepared roll. '
+          'Closes any associated issues and deletes the roll branches'),
+    action='store_true')
+  parser.add_argument('--no-commit',
+    help=('Don\'t send the CL to the CQ. This is useful if additional changes '
+          'are needed to the CL (like for API changes).'),
+    action='store_true')
+  parser.add_argument('--wait-for-trybots',
+    help=('Waits until all trybots from a previously created roll are either '
+          'successful or at least one has failed. This is useful to be able to '
+          'continuously run this script but not initiating new rolls until a '
+          'previous one is known to have passed or failed.'),
+    action='store_true')
+  parser.add_argument('--close-previous-roll', action='store_true',
+                      help='Abort a previous roll if one exists.')
+  parser.add_argument('--dry-run', action='store_true', default=False,
+      help='Create branches and CLs but doesn\'t send tryjobs or commit.')
+  parser.add_argument('--ignore-checks', action='store_true', default=False,
+      help=('Skips checks for being on the master branch, dirty workspaces and '
+            'the updating of the checkout. Will still delete and create local '
+            'Git branches.'))
+  parser.add_argument('-v', '--verbose', action='store_true', default=False,
+      help='Be extra verbose in printing of log messages.')
+  args = parser.parse_args()
+
+  if args.verbose:
+    logging.basicConfig(level=logging.DEBUG)
+  else:
+    logging.basicConfig(level=logging.ERROR)
+
+  autoroller = AutoRoller(SRC_DIR)
+  if args.abort:
+    return autoroller.Abort()
+  elif args.wait_for_trybots:
+    return autoroller.WaitForTrybots()
+  else:
+    return autoroller.PrepareRoll(args.dry_run, args.ignore_checks,
+                                  args.no_commit, args.close_previous_roll)
+
+if __name__ == '__main__':
+  sys.exit(main())