Revert of Delete lots of svn logic from bot_update

recipe_modules/bot_update/resources/bot_update.py

 #!/usr/bin/env python
 # Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 # TODO(hinoka): Use logging.
 
 import cStringIO
 import codecs
 import collections
@@ skipping 68 matching lines @@
         path.join(ROOT_DIR,      # .recipe_deps
                   path.pardir,   # slave
                   path.pardir,   # scripts
                   path.pardir),  # build_internal
     ])
 
 
 CHROMIUM_GIT_HOST = 'https://chromium.googlesource.com'
 CHROMIUM_SRC_URL = CHROMIUM_GIT_HOST + '/chromium/src.git'
 
+# Official builds use buildspecs, so this is a special case.
+BUILDSPEC_TYPE = collections.namedtuple('buildspec',
+    ('container', 'version'))
+BUILDSPEC_RE = (r'^/chrome-internal/trunk/tools/buildspec/'
+                 '(build|branches|releases)/(.+)$')
+GIT_BUILDSPEC_PATH = ('https://chrome-internal.googlesource.com/chrome/tools/'
+                      'buildspec')
 BRANCH_HEADS_REFSPEC = '+refs/branch-heads/*'
 
 BUILDSPEC_COMMIT_RE = (
     re.compile(r'Buildspec for.*version (\d+\.\d+\.\d+\.\d+)'),
     re.compile(r'Create (\d+\.\d+\.\d+\.\d+) buildspec'),
     re.compile(r'Auto-converted (\d+\.\d+\.\d+\.\d+) buildspec to git'),
 )
 
 # Regular expression that matches a single commit footer line.
 COMMIT_FOOTER_ENTRY_RE = re.compile(r'([^:]+):\s+(.+)')
 
 # Footer metadata keys for regular and gsubtreed mirrored commit positions.
 COMMIT_POSITION_FOOTER_KEY = 'Cr-Commit-Position'
 COMMIT_ORIGINAL_POSITION_FOOTER_KEY = 'Cr-Original-Commit-Position'
 # Regular expression to parse a commit position
 COMMIT_POSITION_RE = re.compile(r'(.+)@\{#(\d+)\}')
 
 # Regular expression to parse gclient's revinfo entries.
 REVINFO_RE = re.compile(r'^([^:]+):\s+([^@]+)@(.+)$')
 
+# Used by 'ResolveSvnRevisionFromGitiles'
+GIT_SVN_PROJECT_MAP = {
+  'webkit': {
+    'svn_url': 'svn://svn.chromium.org/blink',
+    'branch_map': [
+      (r'trunk', r'refs/heads/master'),
+      (r'branches/([^/]+)', r'refs/branch-heads/\1'),
+    ],
+  },
+  'v8': {
+    'svn_url': 'https://v8.googlecode.com/svn',
+    'branch_map': [
+      (r'trunk', r'refs/heads/candidates'),
+      (r'branches/bleeding_edge', r'refs/heads/master'),
+      (r'branches/([^/]+)', r'refs/branch-heads/\1'),
+    ],
+  },
+  'nacl': {
+    'svn_url': 'svn://svn.chromium.org/native_client',
+    'branch_map': [
+      (r'trunk/src/native_client', r'refs/heads/master'),
+    ],
+  },
+}
+
+# Key for the 'git-svn' ID metadata commit footer entry.
+GIT_SVN_ID_FOOTER_KEY = 'git-svn-id'
+# e.g., git-svn-id: https://v8.googlecode.com/svn/trunk@23117
+#     ce2b1a6d-e550-0410-aec6-3dcde31c8c00
+GIT_SVN_ID_RE = re.compile(r'((?:\w+)://[^@]+)@(\d+)\s+(?:[a-zA-Z0-9\-]+)')
+
+
+# This is the git mirror of the buildspecs repository. We could rely on the svn
+# checkout, now that the git buildspecs are checked in alongside the svn
+# buildspecs, but we're going to want to pull all the buildspecs from here
+# eventually anyhow, and there's already some logic to pull from git (for the
+# old git_buildspecs.git repo), so just stick with that.
+GIT_BUILDSPEC_REPO = (
+    'https://chrome-internal.googlesource.com/chrome/tools/buildspec')
 
 # Copied from scripts/recipes/chromium.py.
 GOT_REVISION_MAPPINGS = {
-    CHROMIUM_SRC_URL: {
+    '/chrome/trunk/src': {
         'src/': 'got_revision',
         'src/native_client/': 'got_nacl_revision',
         'src/tools/swarm_client/': 'got_swarm_client_revision',
         'src/tools/swarming_client/': 'got_swarming_client_revision',
         'src/third_party/WebKit/': 'got_webkit_revision',
         'src/third_party/webrtc/': 'got_webrtc_revision',
         'src/v8/': 'got_v8_revision',
     }
 }
 
 
 BOT_UPDATE_MESSAGE = """
 What is the "Bot Update" step?
 ==============================
 
 This step ensures that the source checkout on the bot (e.g. Chromium's src/ and
 its dependencies) is checked out in a consistent state. This means that all of
 the necessary repositories are checked out, no extra repositories are checked
 out, and no locally modified files are present.
 
 These actions used to be taken care of by the "gclient revert" and "update"
 steps. However, those steps are known to be buggy and occasionally flaky. This
 step has two main advantages over them:
   * it only operates in Git, so the logic can be clearer and cleaner; and
   * it is a slave-side script, so its behavior can be modified without
     restarting the master.
 
+Why Git, you ask? Because that is the direction that the Chromium project is
+heading. This step is an integral part of the transition from using the SVN repo
+at chrome/trunk/src to using the Git repo src.git. Please pardon the dust while
+we fully convert everything to Git. This message will get out of your way
+eventually, and the waterfall will be a happier place because of it.
+
+This step can be activated or deactivated independently on every builder on
+every master. When it is active, the "gclient revert" and "update" steps become
+no-ops. When it is inactive, it prints this message, cleans up after itself, and
+lets everything else continue as though nothing has changed. Eventually, when
+everything is stable enough, this step will replace them entirely.
+
 Debugging information:
 (master/builder/slave may be unspecified on recipes)
 master: %(master)s
 builder: %(builder)s
 slave: %(slave)s
 forced by recipes: %(recipe)s
 CURRENT_DIR: %(CURRENT_DIR)s
 BUILDER_DIR: %(BUILDER_DIR)s
 SLAVE_DIR: %(SLAVE_DIR)s
 THIS_DIR: %(THIS_DIR)s
@@ skipping 29 matching lines @@
   try:
     execfile(os.path.join(
         BUILD_INTERNAL_DIR, 'scripts', 'slave', 'bot_update_cfg.py'),
         local_vars)
   except Exception:
     # Same as if BUILD_INTERNAL_DIR didn't exist in the first place.
     print 'Warning: unable to read internal configuration file.'
     print 'If this is an internal bot, this step may be erroneously inactive.'
   internal_data = local_vars
 
+RECOGNIZED_PATHS = {
+    # If SVN path matches key, the entire URL is rewritten to the Git url.
+    '/chrome/trunk/src':
+        CHROMIUM_SRC_URL,
+    '/chrome/trunk/src/tools/cros.DEPS':
+        CHROMIUM_GIT_HOST + '/chromium/src/tools/cros.DEPS.git',
+    '/chrome-internal/trunk/src-internal':
+        'https://chrome-internal.googlesource.com/chrome/src-internal.git',
+}
+RECOGNIZED_PATHS.update(internal_data.get('RECOGNIZED_PATHS', {}))
 
 ENABLED_MASTERS = [
     'bot_update.always_on',
     'chromium.android',
     'chromium.angle',
     'chromium.chrome',
     'chromium.chromedriver',
     'chromium.chromiumos',
     'chromium',
     'chromium.fyi',
@@ skipping 63 matching lines @@
 
 # Disabled filters get run AFTER enabled filters, so for example if a builder
 # config is enabled, but a bot on that builder is disabled, that bot will
 # be disabled.
 DISABLED_BUILDERS = {}
 DISABLED_BUILDERS.update(internal_data.get('DISABLED_BUILDERS', {}))
 
 DISABLED_SLAVES = {}
 DISABLED_SLAVES.update(internal_data.get('DISABLED_SLAVES', {}))
 
+# These masters work only in Git, meaning for got_revision, always output
+# a git hash rather than a SVN rev.
+GIT_MASTERS = [
+    'client.v8',
+    'client.v8.branches',
+    'client.v8.ports',
+    'tryserver.v8',
+]
+GIT_MASTERS += internal_data.get('GIT_MASTERS', [])
+
+
 # How many times to try before giving up.
 ATTEMPTS = 5
 
 GIT_CACHE_PATH = path.join(DEPOT_TOOLS_DIR, 'git_cache.py')
 
 # Find the patch tool.
 if sys.platform.startswith('win'):
   if not BUILD_INTERNAL_DIR:
     print 'Warning: could not find patch tool because there is no '
     print 'build_internal present.'
@@ skipping 16 matching lines @@
 
 
 class PatchFailed(SubprocessFailed):
   pass
 
 
 class GclientSyncFailed(SubprocessFailed):
   pass
 
 
+class SVNRevisionNotFound(Exception):
+  pass
+
+
 class InvalidDiff(Exception):
   pass
 
 
+class Inactive(Exception):
+  """Not really an exception, just used to exit early cleanly."""
+  pass
+
+
 RETRY = object()
 OK = object()
 FAIL = object()
 
 
 class PsPrinter(object):
   def __init__(self, interval=300):
     self.interval = interval
     self.active = sys.platform.startswith('linux2')
     self.thread = None
@@ skipping 136 matching lines @@
   if slave_list and slave in slave_list:
     return True
   return False
 
 
 def check_valid_host(master, builder, slave):
   return (check_enabled(master, builder, slave)
           and not check_disabled(master, builder, slave))
 
 
+def maybe_ignore_revision(revision, buildspec):
+  """Handle builders that don't care what buildbot tells them to build.
+
+  This is especially the case with branch builders that build from buildspecs
+  and/or trigger off multiple repositories, where the --revision passed in has
+  nothing to do with the solution being built. Clearing the revision in this
+  case causes bot_update to use HEAD rather that trying to checkout an
+  inappropriate version of the solution.
+  """
+  if buildspec and buildspec.container == 'branches':
+    return []
+  return revision
+
+
 def solutions_printer(solutions):
   """Prints gclient solution to stdout."""
   print 'Gclient Solutions'
   print '================='
   for solution in solutions:
     name = solution.get('name')
     url = solution.get('url')
     print '%s (%s)' % (name, url)
     if solution.get('deps_file'):
       print '  Dependencies file is %s' % solution['deps_file']
@@ skipping 14 matching lines @@
           print '    %s: Ignore' % deps_name
     for k, v in solution.iteritems():
       # Print out all the keys we don't know about.
       if k in ['name', 'url', 'deps_file', 'custom_vars', 'custom_deps',
                'managed']:
         continue
       print '  %s is %s' % (k, v)
     print
 
 
-def modify_solutions(input_solutions):
+def solutions_to_git(input_solutions):
   """Modifies urls in solutions to point at Git repos.
 
-  returns: new solution dictionary
+  returns: (git solution, svn root of first solution) tuple.
   """
   assert input_solutions
   solutions = copy.deepcopy(input_solutions)
+  first_solution = True
+  buildspec = None
   for solution in solutions:
     original_url = solution['url']
     parsed_url = urlparse.urlparse(original_url)
     parsed_path = parsed_url.path
 
+    # Rewrite SVN urls into Git urls.
+    buildspec_m = re.match(BUILDSPEC_RE, parsed_path)
+    if first_solution and buildspec_m:
+      solution['url'] = GIT_BUILDSPEC_PATH
+      buildspec = BUILDSPEC_TYPE(
+          container=buildspec_m.group(1),
+          version=buildspec_m.group(2),
+      )
+      solution['deps_file'] = path.join(buildspec.container, buildspec.version,
+                                        'DEPS')
+    elif parsed_path in RECOGNIZED_PATHS:
+      solution['url'] = RECOGNIZED_PATHS[parsed_path]
+      solution['deps_file'] = '.DEPS.git'
+    elif parsed_url.scheme == 'https' and 'googlesource' in parsed_url.netloc:
+      pass
+    else:
+      print 'Warning: %s' % ('path %r not recognized' % parsed_path,)
+
+    # Strip out deps containing $$V8_REV$$, etc.
+    if 'custom_deps' in solution:
+      new_custom_deps = {}
+      for deps_name, deps_value in solution['custom_deps'].iteritems():
+        if deps_value and '$$' in deps_value:
+          print 'Dropping %s:%s from custom deps' % (deps_name, deps_value)
+        else:
+          new_custom_deps[deps_name] = deps_value
+      solution['custom_deps'] = new_custom_deps
+
+    if first_solution:
+      root = parsed_path
+      first_solution = False
+
     solution['managed'] = False
     # We don't want gclient to be using a safesync URL. Instead it should
     # using the lkgr/lkcr branch/tags.
     if 'safesync_url' in solution:
       print 'Removing safesync url %s from %s' % (solution['safesync_url'],
                                                   parsed_path)
       del solution['safesync_url']
-
-  return solutions
+  return solutions, root, buildspec
 
 
 def remove(target):
   """Remove a target by moving it into build.dead."""
   dead_folder = path.join(BUILDER_DIR, 'build.dead')
   if not path.exists(dead_folder):
     os.makedirs(dead_folder)
   os.rename(target, path.join(dead_folder, uuid.uuid4().hex))
 
 
-def ensure_no_checkout(dir_names):
-  """Ensure that there is no undesired checkout under build/."""
-  build_dir = os.getcwd()
-  has_checkout = any(path.exists(path.join(build_dir, dir_name, '.git'))
+def ensure_no_checkout(dir_names, scm_dirname):
+  """Ensure that there is no undesired checkout under build/.
+
+  If there is an incorrect checkout under build/, then
+  move build/ to build.dead/
+  This function will check each directory in dir_names.
+
+  scm_dirname is expected to be either ['.svn', '.git']
+  """
+  assert scm_dirname in ['.svn', '.git', '*']
+  has_checkout = any(path.exists(path.join(os.getcwd(), dir_name, scm_dirname))
                      for dir_name in dir_names)
-  if has_checkout:
+
+  if has_checkout or scm_dirname == '*':
+    build_dir = os.getcwd()
+    prefix = ''
+    if scm_dirname != '*':
+      prefix = '%s detected in checkout, ' % scm_dirname
+
     for filename in os.listdir(build_dir):
       deletion_target = path.join(build_dir, filename)
-      print '.git detected in checkout, deleting %s...' % deletion_target,
+      print '%sdeleting %s...' % (prefix, deletion_target),
       remove(deletion_target)
       print 'done'
 
 
 def gclient_configure(solutions, target_os, target_os_only, git_cache_dir):
   """Should do the same thing as gclient --spec='...'."""
   with codecs.open('.gclient', mode='w', encoding='utf-8') as f:
     f.write(get_gclient_spec(
         solutions, target_os, target_os_only, git_cache_dir))
 
@@ skipping 73 matching lines @@
     footers[m.group(1)] = m.group(2).strip()
   return footers
 
 
 def get_commit_message_footer(message, key):
   """Returns: (str/None) The footer value for 'key', or None if none was found.
   """
   return get_commit_message_footer_map(message).get(key)
 
 
+def get_svn_rev(git_hash, dir_name):
+  log = git('log', '-1', git_hash, cwd=dir_name)
+  git_svn_id = get_commit_message_footer(log, GIT_SVN_ID_FOOTER_KEY)
+  if not git_svn_id:
+    return None
+  m = GIT_SVN_ID_RE.match(git_svn_id)
+  if not m:
+    return None
+  return int(m.group(2))
+
+
+def get_git_hash(revision, branch, sln_dir):
+  """We want to search for the SVN revision on the git-svn branch.
+
+  Note that git will search backwards from origin/master.
+  """
+  match = "^%s: [^ ]*@%s " % (GIT_SVN_ID_FOOTER_KEY, revision)
+  ref = branch if branch.startswith('refs/') else 'origin/%s' % branch
+  cmd = ['log', '-E', '--grep', match, '--format=%H', '--max-count=1', ref]
+  result = git(*cmd, cwd=sln_dir).strip()
+  if result:
+    return result
+  raise SVNRevisionNotFound('We can\'t resolve svn r%s into a git hash in %s' %
+                            (revision, sln_dir))
+
+
 def emit_log_lines(name, lines):
   for line in lines.splitlines():
     print '@@@STEP_LOG_LINE@%s@%s@@@' % (name, line)
   print '@@@STEP_LOG_END@%s@@@' % name
 
 
 def emit_properties(properties):
   for property_name, property_value in sorted(properties.items()):
     print '@@@SET_BUILD_PROPERTY@%s@"%s"@@@' % (property_name, property_value)
 
@@ skipping 34 matching lines @@
 
 
 def force_revision(folder_name, revision):
   split_revision = revision.split(':', 1)
   branch = 'master'
   if len(split_revision) == 2:
     # Support for "branch:revision" syntax.
     branch, revision = split_revision
 
   if revision and revision.upper() != 'HEAD':
-    git('checkout', '--force', revision, cwd=folder_name)
+    if revision and revision.isdigit() and len(revision) < 40:
+      # rev_num is really a svn revision number, convert it into a git hash.
+      git_ref = get_git_hash(int(revision), branch, folder_name)
+    else:
+      # rev_num is actually a git hash or ref, we can just use it.
+      git_ref = revision
+    git('checkout', '--force', git_ref, cwd=folder_name)
   else:
     ref = branch if branch.startswith('refs/') else 'origin/%s' % branch
     git('checkout', '--force', ref, cwd=folder_name)
 
-
 def git_checkout(solutions, revisions, shallow, refs, git_cache_dir):
   build_dir = os.getcwd()
   # Before we do anything, break all git_cache locks.
   if path.isdir(git_cache_dir):
     git('cache', 'unlock', '-vv', '--force', '--all',
         '--cache-dir', git_cache_dir)
     for item in os.listdir(git_cache_dir):
       filename = os.path.join(git_cache_dir, item)
       if item.endswith('.lock'):
         raise Exception('%s exists after cache unlock' % filename)
@@ skipping 40 matching lines @@
         # Exited abnormally, theres probably something wrong.
         # Lets wipe the checkout and try again.
         tries_left -= 1
         if tries_left > 0:
           print 'Something failed: %s.' % str(e)
           print 'waiting 5 seconds and trying again...'
           time.sleep(5)
         else:
           raise
         remove(sln_dir)
+      except SVNRevisionNotFound:
+        tries_left -= 1
+        if tries_left > 0:
+          # If we don't have the correct revision, wait and try again.
+          print 'We can\'t find revision %s.' % revision
+          print 'The svn to git replicator is probably falling behind.'
+          print 'waiting 5 seconds and trying again...'
+          time.sleep(5)
+        else:
+          raise
 
     git('clean', '-dff', cwd=sln_dir)
 
     if first_solution:
       git_ref = git('log', '--format=%H', '--max-count=1',
                     cwd=sln_dir).strip()
     first_solution = False
   return git_ref
 
 
 def _download(url):
   """Fetch url and return content, with retries for flake."""
   for attempt in xrange(ATTEMPTS):
     try:
       return urllib2.urlopen(url).read()
     except Exception:
       if attempt == ATTEMPTS - 1:
         raise
 
 
+def parse_diff(diff):
+  """Takes a unified diff and returns a list of diffed files and their diffs.
+
+  The return format is a list of pairs of:
+    (<filename>, <diff contents>)
+  <diff contents> is inclusive of the diff line.
+  """
+  result = []
+  current_diff = ''
+  current_header = None
+  for line in diff.splitlines():
+    # "diff" is for git style patches, and "Index: " is for SVN style patches.
+    if line.startswith('diff') or line.startswith('Index: '):
+      if current_header:
+        # If we are in a diff portion, then save the diff.
+        result.append((current_header, '%s\n' % current_diff))
+      git_header_match = re.match(r'diff (?:--git )?(\S+) (\S+)', line)
+      svn_header_match = re.match(r'Index: (.*)', line)
+
+      if git_header_match:
+        # First, see if its a git style header.
+        from_file = git_header_match.group(1)
+        to_file = git_header_match.group(2)
+        if from_file != to_file and from_file.startswith('a/'):
+          # Sometimes git prepends 'a/' and 'b/' in front of file paths.
+          from_file = from_file[2:]
+        current_header = from_file
+
+      elif svn_header_match:
+        # Otherwise, check if its an SVN style header.
+        current_header = svn_header_match.group(1)
+
+      else:
+        # Otherwise... I'm not really sure what to do with this.
+        raise InvalidDiff('Can\'t process header: %s\nFull diff:\n%s' %
+                          (line, diff))
+
+      current_diff = ''
+    current_diff += '%s\n' % line
+  if current_header:
+    # We hit EOF, gotta save the last diff.
+    result.append((current_header, current_diff))
+  return result
+
+
 def apply_rietveld_issue(issue, patchset, root, server, _rev_map, _revision,
                          email_file, key_file, whitelist=None, blacklist=None):
   apply_issue_bin = ('apply_issue.bat' if sys.platform.startswith('win')
                      else 'apply_issue')
   cmd = [apply_issue_bin,
          # The patch will be applied on top of this directory.
          '--root_dir', root,
          # Tell apply_issue how to fetch the patch.
          '--issue', issue,
          '--server', server,
@@ skipping 77 matching lines @@
     os.remove(flag_file)
 
 
 def emit_flag(flag_file):
   """Deposit a bot update flag on the system to tell gclient not to run."""
   print 'Emitting flag file at %s' % flag_file
   with open(flag_file, 'wb') as f:
     f.write('Success!')
 
 
+def get_commit_position_for_git_svn(url, revision):
+  """Generates a commit position string for a 'git-svn' URL/revision.
+
+  If the 'git-svn' URL maps to a known project, we will construct a commit
+  position branch value by applying substitution on the SVN URL.
+  """
+  # Identify the base URL so we can strip off trunk/branch name
+  project_config = branch = None
+  for _, project_config in GIT_SVN_PROJECT_MAP.iteritems():
+    if url.startswith(project_config['svn_url']):
+      branch = url[len(project_config['svn_url']):]
+      break
+
+  if branch:
+    # Strip any leading slashes
+    branch = branch.lstrip('/')
+
+    # Try and map the branch
+    for pattern, repl in project_config.get('branch_map', ()):
+      nbranch, subn = re.subn(pattern, repl, branch, count=1)
+      if subn:
+        print 'INFO: Mapped SVN branch to Git branch [%s] => [%s]' % (
+            branch, nbranch)
+        branch = nbranch
+        break
+  else:
+    # Use generic 'svn' branch
+    print 'INFO: Could not resolve project for SVN URL %r' % (url,)
+    branch = 'svn'
+  return '%s@{#%s}' % (branch, revision)
+
+
 def get_commit_position(git_path, revision='HEAD'):
   """Dumps the 'git' log for a specific revision and parses out the commit
   position.
 
   If a commit position metadata key is found, its value will be returned.
+
+  Otherwise, we will search for a 'git-svn' metadata entry. If one is found,
+  we will compose a commit position from it, using its SVN revision value as
+  the revision.
+
+  If the 'git-svn' URL maps to a known project, we will construct a commit
+  position branch value by truncating the URL, mapping 'trunk' to
+  "refs/heads/master". Otherwise, we will return the generic branch, 'svn'.
   """
-  # TODO(iannucci): Use git-footers for this.
   git_log = git('log', '--format=%B', '-n1', revision, cwd=git_path)
   footer_map = get_commit_message_footer_map(git_log)
 
   # Search for commit position metadata
   value = (footer_map.get(COMMIT_POSITION_FOOTER_KEY) or
            footer_map.get(COMMIT_ORIGINAL_POSITION_FOOTER_KEY))
   if value:
     return value
+
+  # Compose a commit position from 'git-svn' metadata
+  value = footer_map.get(GIT_SVN_ID_FOOTER_KEY)
+  if value:
+    m = GIT_SVN_ID_RE.match(value)
+    if not m:
+      raise ValueError("Invalid 'git-svn' value: [%s]" % (value,))
+    return get_commit_position_for_git_svn(m.group(1), m.group(2))
   return None
 
 
-def parse_got_revision(gclient_output, got_revision_mapping):
-  """Translate git gclient revision mapping to build properties."""
+def parse_got_revision(gclient_output, got_revision_mapping, use_svn_revs):
+  """Translate git gclient revision mapping to build properties.
+
+  If use_svn_revs is True, then translate git hashes in the revision mapping
+  to svn revision numbers.
+  """
   properties = {}
   solutions_output = {
       # Make sure path always ends with a single slash.
       '%s/' % path.rstrip('/') : solution_output for path, solution_output
       in gclient_output['solutions'].iteritems()
   }
   for dir_name, property_name in got_revision_mapping.iteritems():
     # Make sure dir_name always ends with a single slash.
     dir_name = '%s/' % dir_name.rstrip('/')
     if dir_name not in solutions_output:
       continue
     solution_output = solutions_output[dir_name]
     if solution_output.get('scm') is None:
       # This is an ignored DEPS, so the output got_revision should be 'None'.
       git_revision = revision = commit_position = None
     else:
       # Since we are using .DEPS.git, everything had better be git.
       assert solution_output.get('scm') == 'git'
-      revision = git('rev-parse', 'HEAD', cwd=dir_name).strip()
+      git_revision = git('rev-parse', 'HEAD', cwd=dir_name).strip()
+      if use_svn_revs:
+        revision = get_svn_rev(git_revision, dir_name)
+        if not revision:
+          revision = git_revision
+      else:
+        revision = git_revision
       commit_position = get_commit_position(dir_name)
 
     properties[property_name] = revision
     if revision != git_revision:
       properties['%s_git' % property_name] = git_revision
     if commit_position:
       properties['%s_cp' % property_name] = commit_position
 
   return properties
 
@@ skipping 11 matching lines @@
 def ensure_deps_revisions(deps_url_mapping, solutions, revisions):
   """Ensure correct DEPS revisions, ignores solutions."""
   for deps_name, deps_data in sorted(deps_url_mapping.items()):
     if deps_name.strip('/') in solutions:
       # This has already been forced to the correct solution by git_checkout().
       continue
     revision = get_target_revision(deps_name, deps_data.get('url', None),
                                    revisions)
     if not revision:
       continue
+    # TODO(hinoka): Catch SVNRevisionNotFound error maybe?
     git('fetch', 'origin', cwd=deps_name)
     force_revision(deps_name, revision)
 
 
 def ensure_checkout(solutions, revisions, first_sln, target_os, target_os_only,
                     patch_root, issue, patchset, rietveld_server,
                     gerrit_repo, gerrit_ref, gerrit_rebase_patch_ref,
                     revision_mapping, apply_issue_email_file,
-                    apply_issue_key_file, gyp_env, shallow, runhooks,
+                    apply_issue_key_file, buildspec, gyp_env, shallow, runhooks,
                     refs, git_cache_dir, gerrit_reset):
   # Get a checkout of each solution, without DEPS or hooks.
   # Calling git directly because there is no way to run Gclient without
   # invoking DEPS.
   print 'Fetching Git checkout'
 
   git_ref = git_checkout(solutions, revisions, shallow, refs, git_cache_dir)
 
   print '===Processing patch solutions==='
   already_patched = []
@@ skipping 16 matching lines @@
         apply_gerrit_ref(gerrit_repo, gerrit_ref, patch_root, gerrit_reset,
                          gerrit_rebase_patch_ref)
         applied_gerrit_patch = True
 
   # Ensure our build/ directory is set up with the correct .gclient file.
   gclient_configure(solutions, target_os, target_os_only, git_cache_dir)
 
   # Let gclient do the DEPS syncing.
   # The branch-head refspec is a special case because its possible Chrome
   # src, which contains the branch-head refspecs, is DEPSed in.
-  gclient_output = gclient_sync(BRANCH_HEADS_REFSPEC in refs, shallow)
+  gclient_output = gclient_sync(buildspec or BRANCH_HEADS_REFSPEC in refs,
+                                shallow)
 
   # Now that gclient_sync has finished, we should revert any .DEPS.git so that
   # presubmit doesn't complain about it being modified.
-  if git('ls-files', '.DEPS.git', cwd=first_sln).strip():
+  if (not buildspec and
+      git('ls-files', '.DEPS.git', cwd=first_sln).strip()):
     git('checkout', 'HEAD', '--', '.DEPS.git', cwd=first_sln)
 
+  if buildspec and runhooks:
+    # Run gclient runhooks if we're on an official builder.
+    # TODO(hinoka): Remove this when the official builders run their own
+    #               runhooks step.
+    gclient_runhooks(gyp_env)
+
   # Finally, ensure that all DEPS are pinned to the correct revision.
   dir_names = [sln['name'] for sln in solutions]
   ensure_deps_revisions(gclient_output.get('solutions', {}),
                         dir_names, revisions)
   # Apply the rest of the patch here (sans DEPS)
   if issue:
     apply_rietveld_issue(issue, patchset, patch_root, rietveld_server,
                          revision_mapping, git_ref, apply_issue_email_file,
                          apply_issue_key_file, blacklist=already_patched)
   elif gerrit_ref and not applied_gerrit_patch:
@@ skipping 25 matching lines @@
     expanded_revisions.extend(revision.split(','))
   for revision in expanded_revisions:
     split_revision = revision.split('@')
     if len(split_revision) == 1:
       # This is just a plain revision, set it as the revision for root.
       results[root] = split_revision[0]
     elif len(split_revision) == 2:
       # This is an alt_root@revision argument.
       current_root, current_rev = split_revision
 
+      # We want to normalize svn/git urls into .git urls.
       parsed_root = urlparse.urlparse(current_root)
-      if parsed_root.scheme in ['http', 'https']:
-        # We want to normalize git urls into .git urls.
+      if parsed_root.scheme == 'svn':
+        if parsed_root.path in RECOGNIZED_PATHS:
+          normalized_root = RECOGNIZED_PATHS[parsed_root.path]
+        else:
+          print 'WARNING: SVN path %s not recognized, ignoring' % current_root
+          continue
+      elif parsed_root.scheme in ['http', 'https']:
         normalized_root = 'https://%s/%s' % (parsed_root.netloc,
                                              parsed_root.path)
         if not normalized_root.endswith('.git'):
           normalized_root = '%s.git' % normalized_root
       elif parsed_root.scheme:
         print 'WARNING: Unrecognized scheme %s, ignoring' % parsed_root.scheme
         continue
       else:
         # This is probably a local path.
         normalized_root = current_root.strip('/')
@@ skipping 35 matching lines @@
                         'bot_update\'s whitelist, bot_update will be noop.')
   parse.add_option('-f', '--force', action='store_true',
                    help='Bypass check to see if we want to be run. '
                         'Should ONLY be used locally or by smart recipes.')
   parse.add_option('--revision_mapping',
                    help='{"path/to/repo/": "property_name"}')
   parse.add_option('--revision_mapping_file',
                    help=('Same as revision_mapping, except its a path to a json'
                          ' file containing that format.'))
   parse.add_option('--revision', action='append', default=[],
-                   help='Revision to check out. Can be any form of git ref. '
-                        'Can prepend root@<rev> to specify which repository, '
-                        'where root is either a filesystem path or git https '
-                        'url. To specify Tip of Tree, set rev to HEAD. ')
+                   help='Revision to check out. Can be an SVN revision number, '
+                        'git hash, or any form of git ref.  Can prepend '
+                        'root@<rev> to specify which repository, where root '
+                        'is either a filesystem path, git https url, or '
+                        'svn url. To specify Tip of Tree, set rev to HEAD.'
+                        'To specify a git branch and an SVN rev, <rev> can be '
+                        'set to <branch>:<revision>.')
   parse.add_option('--output_manifest', action='store_true',
                    help=('Add manifest json to the json output.'))
   parse.add_option('--slave_name', default=socket.getfqdn().split('.')[0],
                    help='Hostname of the current machine, '
                    'used for determining whether or not to activate.')
   parse.add_option('--builder_name', help='Name of the builder, '
                    'used for determining whether or not to activate.')
   parse.add_option('--build_dir', default=os.getcwd())
   parse.add_option('--flag_file', default=path.join(os.getcwd(),
                                                     'update.flag'))
@@ skipping 53 matching lines @@
     options.git_cache_dir = options.git_cache_dir.replace('\\', '\\\\')
 
   return options, args
 
 
 def prepare(options, git_slns, active):
   """Prepares the target folder before we checkout."""
   dir_names = [sln.get('name') for sln in git_slns if 'name' in sln]
   # If we're active now, but the flag file doesn't exist (we weren't active
   # last run) or vice versa, blow away all checkouts.
-  if options.clobber or (bool(active) != bool(check_flag(options.flag_file))):
-    ensure_no_checkout(dir_names)
+  if bool(active) != bool(check_flag(options.flag_file)):
+    ensure_no_checkout(dir_names, '*')
   if options.output_json:
     # Make sure we tell recipes that we didn't run if the script exits here.
     emit_json(options.output_json, did_run=active)
-  emit_flag(options.flag_file)
+  if active:
+    if options.clobber:
+      ensure_no_checkout(dir_names, '*')
+    else:
+      ensure_no_checkout(dir_names, '.svn')
+    emit_flag(options.flag_file)
+  else:
+    delete_flag(options.flag_file)
+    raise Inactive  # This is caught in main() and we exit cleanly.
 
   # Do a shallow checkout if the disk is less than 100GB.
   total_disk_space, free_disk_space = get_total_disk_space()
   total_disk_space_gb = int(total_disk_space / (1024 * 1024 * 1024))
   used_disk_space_gb = int((total_disk_space - free_disk_space)
                            / (1024 * 1024 * 1024))
   percent_used = int(used_disk_space_gb * 100 / total_disk_space_gb)
   step_text = '[%dGB/%dGB used (%d%%)]' % (used_disk_space_gb,
                                            total_disk_space_gb,
                                            percent_used)
   if not options.output_json:
     print '@@@STEP_TEXT@%s@@@' % step_text
   if not options.shallow:
     options.shallow = (total_disk_space < SHALLOW_CLONE_THRESHOLD
                        and not options.no_shallow)
 
   # The first solution is where the primary DEPS file resides.
   first_sln = dir_names[0]
 
   # Split all the revision specifications into a nice dict.
   print 'Revisions: %s' % options.revision
   revisions = parse_revisions(options.revision, first_sln)
   print 'Fetching Git checkout at %s@%s' % (first_sln, revisions[first_sln])
   return revisions, step_text
 
 
-def checkout(options, git_slns, specs, master, revisions, step_text):
+def checkout(options, git_slns, specs, buildspec, master,
+             svn_root, revisions, step_text):
   first_sln = git_slns[0]['name']
   dir_names = [sln.get('name') for sln in git_slns if 'name' in sln]
   try:
     # Outer try is for catching patch failures and exiting gracefully.
     # Inner try is for catching gclient failures and retrying gracefully.
     try:
       checkout_parameters = dict(
           # First, pass in the base of what we want to check out.
           solutions=git_slns,
           revisions=revisions,
           first_sln=first_sln,
 
           # Also, target os variables for gclient.
           target_os=specs.get('target_os', []),
           target_os_only=specs.get('target_os_only', False),
 
           # Then, pass in information about how to patch.
           patch_root=options.patch_root,
           issue=options.issue,
           patchset=options.patchset,
           rietveld_server=options.rietveld_server,
           gerrit_repo=options.gerrit_repo,
           gerrit_ref=options.gerrit_ref,
           gerrit_rebase_patch_ref=not options.gerrit_no_rebase_patch_ref,
           revision_mapping=options.revision_mapping,
           apply_issue_email_file=options.apply_issue_email_file,
           apply_issue_key_file=options.apply_issue_key_file,
 
           # For official builders.
+          buildspec=buildspec,
           gyp_env=options.gyp_env,
           runhooks=not options.no_runhooks,
 
           # Finally, extra configurations such as shallowness of the clone.
           shallow=options.shallow,
           refs=options.refs,
           git_cache_dir=options.git_cache_dir,
           gerrit_reset=not options.gerrit_no_reset)
       gclient_output = ensure_checkout(**checkout_parameters)
     except GclientSyncFailed:
       print 'We failed gclient sync, lets delete the checkout and retry.'
-      ensure_no_checkout(dir_names)
+      ensure_no_checkout(dir_names, '*')
       gclient_output = ensure_checkout(**checkout_parameters)
   except PatchFailed as e:
     if options.output_json:
       # Tell recipes information such as root, got_revision, etc.
       emit_json(options.output_json,
                 did_run=True,
                 root=first_sln,
                 log_lines=[('patch error', e.output),],
                 patch_apply_return_code=e.code,
                 patch_root=options.patch_root,
                 patch_failure=True,
                 step_text='%s PATCH FAILED' % step_text,
                 fixed_revisions=revisions)
     else:
       # If we're not on recipes, tell annotator about our got_revisions.
       emit_log_lines('patch error', e.output)
       print '@@@STEP_TEXT@%s PATCH FAILED@@@' % step_text
     raise
 
+  # Revision is an svn revision, unless it's a git master.
+  use_svn_rev = master not in GIT_MASTERS
+
   # Take care of got_revisions outputs.
-  revision_mapping = GOT_REVISION_MAPPINGS.get(git_slns[0]['url'], {})
+  revision_mapping = dict(GOT_REVISION_MAPPINGS.get(svn_root, {}))
   if options.revision_mapping:
     revision_mapping.update(options.revision_mapping)
 
   # If the repo is not in the default GOT_REVISION_MAPPINGS and no
   # revision_mapping were specified on the command line then
   # default to setting 'got_revision' based on the first solution.
   if not revision_mapping:
     revision_mapping[first_sln] = 'got_revision'
 
-  got_revisions = parse_got_revision(gclient_output, revision_mapping)
+  got_revisions = parse_got_revision(gclient_output, revision_mapping,
+                                     use_svn_rev)
 
   if not got_revisions:
     # TODO(hinoka): We should probably bail out here, but in the interest
     # of giving mis-configured bots some time to get fixed use a dummy
     # revision here.
     got_revisions = { 'got_revision': 'BOT_UPDATE_NO_REV_FOUND' }
     #raise Exception('No got_revision(s) found in gclient output')
 
   if options.output_json:
     manifest = create_manifest() if options.output_manifest else None
@@ skipping 53 matching lines @@
   # Check if this script should activate or not.
   active = options.force or check_valid_host(master, builder, slave)
 
   # Print a helpful message to tell developers whats going on with this step.
   print_help_text(
       options.force, options.output_json, active, master, builder, slave)
 
   # Parse, munipulate, and print the gclient solutions.
   specs = {}
   exec(options.specs, specs)
-  orig_solutions = specs.get('solutions', [])
-  git_slns = modify_solutions(orig_solutions)
+  svn_solutions = specs.get('solutions', [])
+  git_slns, svn_root, buildspec = solutions_to_git(svn_solutions)
+  options.revision = maybe_ignore_revision(options.revision, buildspec)
 
   solutions_printer(git_slns)
 
   try:
     # Dun dun dun, the main part of bot_update.
     revisions, step_text = prepare(options, git_slns, active)
-    checkout(options, git_slns, specs, master, revisions, step_text)
+    checkout(options, git_slns, specs, buildspec, master, svn_root, revisions,
+             step_text)
 
+  except Inactive:
+    # Not active, should count as passing.
+    pass
   except PatchFailed as e:
     emit_flag(options.flag_file)
     # Return a specific non-zero exit code for patch failure (because it is
     # a failure), but make it different than other failures to distinguish
     # between infra failures (independent from patch author), and patch
     # failures (that patch author can fix). However, PatchFailure due to
     # download patch failure is still an infra problem.
     if e.code == 3:
       # Patch download problem.
       return 87
     # Genuine patch problem.
     return 88
   except Exception:
     # Unexpected failure.
     emit_flag(options.flag_file)
     raise
   else:
     emit_flag(options.flag_file)
 
 
 if __name__ == '__main__':
   sys.exit(main())