Allow passing in arbitrary root@revision to bot_update

scripts/slave/bot_update.py

 #!/usr/bin/env python
 # Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import codecs
 import copy
 import cStringIO
 import ctypes
 import json
@@ skipping 446 matching lines @@
   finally:
     os.remove(gclient_output_file)
 
 
 def gclient_runhooks(gyp_envs):
   gclient_bin = 'gclient.bat' if sys.platform.startswith('win') else 'gclient'
   env = dict([env_var.split('=', 1) for env_var in gyp_envs])
   call(gclient_bin, 'runhooks', env=env)
 
 
-def create_less_than_or_equal_regex(number):
-  """ Return a regular expression to test whether an integer less than or equal
-      to 'number' is present in a given string.
-  """
-
-  # In three parts, build a regular expression that match any numbers smaller
-  # than 'number'.
-  # For example, 78656 would give a regular expression that looks like:
-  # Part 1
-  # (78356|            # 78356
-  # Part 2
-  #  7835[0-5]|        # 78350-78355
-  #  783[0-4][0-9]|    # 78300-78349
-  #  78[0-2][0-9]{2}|  # 78000-78299
-  #  7[0-7][0-9]{3}|   # 70000-77999
-  #  [0-6][0-9]{4}|    # 10000-69999
-  # Part 3
-  #  [0-9]{1,4}        # 0-9999
-
-  # Part 1: Create an array with all the regexes, as described above.
-  # Prepopulate it with the number itself.
-  number = str(number)
-  expressions = [number]
-
-  # Convert the number to a list, so we can translate digits in it to
-  # expressions.
-  num_list = list(number)
-  num_len = len(num_list)
-
-  # Part 2: Go through all the digits in the number, starting from the end.
-  # Each iteration appends a line to 'expressions'.
-  for index in range (num_len - 1, -1, -1):
-    # Convert this digit back to an integer.
-    digit = int(num_list[index])
-
-    # Part 2.1: No processing if this digit is a zero.
-    if digit == 0:
-      continue
-
-    # Part 2.2: We switch the current digit X by a range "[0-(X-1)]".
-    if digit == 1:
-      num_list[index] = '0'
-    else:
-      num_list[index] = '[0-%d]' % (digit - 1)
-
-    # Part 2.3: We set all following digits to be "[0-9]".
-    # Since we just decrementented a digit in a most important position, all
-    # following digits don't matter. The possible numbers will always be smaller
-    # than before we decremented.
-    if (index + 1) < num_len:
-      if (num_len - (index + 1)) == 1:
-        num_list[index + 1] = '[0-9]'
-      else:
-        num_list[index + 1] = '[0-9]{%s}' % (num_len - (index + 1))
-
-    # Part 2.4: Add this new sub-expression to the list.
-    expressions.append(''.join(num_list[:min(index+2, num_len)]))
-
-  # Part 3: We add all the full ranges to match all numbers that are at least
-  # one order of magnitude smaller than the original numbers.
-  if num_len == 2:
-    expressions.append('[0-9]')
-  elif num_len > 2:
-    expressions.append('[0-9]{1,%s}' % (num_len - 1))
-
-  # All done. We now have our final regular expression.
-  regex = '(%s)' % ('|'.join(expressions))
-  return regex
-
-
 def get_svn_rev(git_hash, dir_name):
   pattern = r'^\s*git-svn-id: [^ ]*@(\d+) '
-  lkcr_pattern = r'^\s*LK[CG]R w/ DEPS up to revision (\d+)'
   log = git('log', '-1', git_hash, cwd=dir_name)
   match = re.search(pattern, log, re.M)
   if not match:
-    # This might be a patched checkout, try the parent commit.
-    log = git('log', '-1', '%s^' % git_hash, cwd=dir_name)
-    match = re.search(pattern, log, re.M)
-    if not match:
-      # Might be patched on top of LKCR, which has a special message.
-      match = re.search(lkcr_pattern, log, re.M)
-      if not match:
-        return None
+    return None
   return int(match.group(1))
 
 
-def get_git_hash(revision, dir_name):
-  match = "^git-svn-id: [^ ]*@%s " % create_less_than_or_equal_regex(revision)
+def get_git_hash(revision, sln_dir):
+  """We want to search for the SVN revision on the git-svn branch.
+
+  Note that for this to work, the git checkout in sln_dir must be newer and in
+  the same lineage as the commit we are searching for.  This is because Git
+  searches backwards from the current commit.

[iannucci, 2014/05/08 23:50:06]

s/current commit/HEAD

though you could specify that to be something else if you want

[hinoka, 2014/05/10 00:54:08]

Redid this to always start from origin/master

+  """
+  match = "^git-svn-id: [^ ]*@%s " % revision
   cmd = ['log', '-E', '--grep', match, '--format=%H', '--max-count=1']
-  results = git(*cmd, cwd=dir_name).strip().splitlines()
-  if results:
-    return results[0]
-  raise Exception('We can\'t resolve svn revision %s into a git hash' %
-                  revision)
+  result = git(*cmd, cwd=sln_dir).strip()
+  if result:
+    return result
+  raise SVNRevisionNotFound('We can\'t resolve svn r%s into a git hash in %s' %
+                            (revision, sln_dir))
 
 
 def get_revision_mapping(root, addl_rev_map):
   result = {}
   if root in GOT_REVISION_MAPPINGS:
     result.update(GOT_REVISION_MAPPINGS[root])
   if addl_rev_map:
     result.update(json.loads(addl_rev_map))
   return result
 
@@ skipping 129 matching lines @@
       raise ctypes.WinError()
     return (total.value, free.value)
 
   else:
     st = os.statvfs(cwd)
     free = st.f_bavail * st.f_frsize
     total = st.f_blocks * st.f_frsize
     return (total, free)
 
 
-def git_checkout(solutions, revision, shallow):
+def get_revision(folder_name, git_url, revisions):
+  normalized_name = folder_name.strip('/')
+  if normalized_name in revisions:
+    return revisions[normalized_name]
+  if git_url in revisions:
+    return revisions[git_url]
+  return None
+
+
+def force_revision(folder_name, revision):
+  # We do this first because git log --grep only traverses backwards.
+  # TODO(hinoka): Prefer origin/git-svn if it exists.

[iannucci, 2014/05/08 23:50:06]

You should just pass origin/master to `git log` instead of doing an extra two
checkouts, right?

[hinoka, 2014/05/10 00:54:08]

Done.

+  git('checkout', 'origin/master', cwd=folder_name)
+  if revision and revision.lower() != 'head':
+    if revision and revision.isdigit() and len(revision) < 40:
+      # rev_num is really a svn revision number, convert it into a git hash.
+      git_ref = get_git_hash(int(revision), folder_name)
+    else:
+      # rev_num is actually a git hash or ref, we can just use it.
+      git_ref = revision
+    git('checkout', git_ref, cwd=folder_name)
+
+
+def git_checkout(solutions, revisions, shallow):
   build_dir = os.getcwd()
   # Before we do anything, break all git_cache locks.
   if path.isdir(CACHE_DIR):
     git('cache', 'unlock', '-vv', '--force', '--all', '--cache-dir', CACHE_DIR)
     for item in os.listdir(CACHE_DIR):
       filename = os.path.join(CACHE_DIR, item)
       if item.endswith('.lock'):
         raise Exception('%s exists after cache unlock' % filename)
-  # Revision only applies to the first solution.
   first_solution = True
   for sln in solutions:
     # This is so we can loop back and try again if we need to wait for the
     # git mirrors to update from SVN.
     done = False
     tries_left = 60
     while not done:
       done = True
       name = sln['name']
       url = sln['url']
@@ skipping 21 matching lines @@
           # Lets wipe the checkout and try again.
           chromium_utils.RemoveDirectory(sln_dir)
           git(*clone_cmd)
           git('checkout', '--force', 'origin/master', cwd=sln_dir)
           git('reset', '--hard', cwd=sln_dir)
         else:
           raise
 
       git('clean', '-df', cwd=sln_dir)
       git('fetch', 'origin', cwd=sln_dir)
-      # TODO(hinoka): We probably have to make use of revision mapping.
-      if first_solution and revision and revision.lower() != 'head':
-        if revision and revision.isdigit() and len(revision) < 40:
-          # rev_num is really a svn revision number, convert it into a git hash.
-          git_ref = get_git_hash(int(revision), name)
-          # Lets make sure our checkout has the correct revision number.
-          got_revision = get_svn_rev(git_ref, sln_dir)
-          if not got_revision or int(got_revision) != int(revision):
-            tries_left -= 1
-            if tries_left > 0:
-              # If we don't have the correct revision, wait and try again.
-              print 'We want revision %s, but got revision %s.' % (
-                  revision, got_revision)
-              print 'The svn to git replicator is probably falling behind.'
-              print 'waiting 5 seconds and trying again...'
-              time.sleep(5)
-              done = False
-              continue
-            raise SVNRevisionNotFound
-        else:
-          # rev_num is actually a git hash or ref, we can just use it.
-          git_ref = revision
-        git('checkout', git_ref, cwd=sln_dir)
-      else:
-        git('checkout', 'origin/master', cwd=sln_dir)
-        if first_solution:
-          git_ref = git('log', '--format=%H', '--max-count=1',
-                        cwd=sln_dir).strip()
 
+      revision = get_revision(name, url, revisions) or 'HEAD'
+      try:
+        force_revision(sln_dir, revision)
+      except SVNRevisionNotFound:
+        tries_left -= 1
+        if tries_left > 0:
+          # If we don't have the correct revision, wait and try again.
+          print 'We can\'t find revision %s.' % revision
+          print 'The svn to git replicator is probably falling behind.'
+          print 'waiting 5 seconds and trying again...'
+          time.sleep(5)

[iannucci, 2014/05/08 23:50:06]

ick :/

[hinoka, 2014/05/10 00:54:08]

As long as we are mirroring, we need a sleep somewhere :/

+          done = False
+          continue
+        raise
+
+      if first_solution:
+        git_ref = git('log', '--format=%H', '--max-count=1',
+                      cwd=sln_dir).strip()
       first_solution = False
   return git_ref
 
 
 def _download(url):
   """Fetch url and return content, with retries for flake."""
   for attempt in xrange(RETRIES):
     try:
       return urllib2.urlopen(url).read()
     except Exception:
@@ skipping 58 matching lines @@
   for dir_name, property_name in got_revision_mapping.iteritems():
     if dir_name not in solutions_output:
       continue
     solution_output = solutions_output[dir_name]
     if solution_output.get('scm') is None:
       # This is an ignored DEPS, so the output got_revision should be 'None'.
       git_revision = revision = None
     else:
       # Since we are using .DEPS.git, everything had better be git.
       assert solution_output.get('scm') == 'git'
-      git_revision = get_real_git_hash(solution_output['revision'], dir_name)
+      git_revision = git('log', '-1', '--format=%H', cwd=dir_name).strip()

[iannucci, 2014/05/08 23:50:06]

er... `git rev-parse HEAD` ?

[hinoka, 2014/05/10 00:54:08]

Done.

       if use_svn_revs:
         revision = get_svn_rev(git_revision, dir_name)
         if not revision:
           revision = git_revision
       else:
         revision = git_revision
 
     properties[property_name] = revision
     if revision != git_revision:
       properties['%s_git' % property_name] = git_revision
 
   return properties
 
 
 def emit_json(out_file, did_run, gclient_output=None, **kwargs):
   """Write run information into a JSON file."""
   output = {}
   output.update(gclient_output if gclient_output else {})
   output.update({'did_run': did_run})
   output.update(kwargs)
   with open(out_file, 'wb') as f:
     f.write(json.dumps(output))
 
 
-def ensure_checkout(solutions, revision, first_sln, target_os, target_os_only,
+def ensure_deps_revisions(deps_url_mapping, solutions, revisions):
+  """Ensure correct DEPS revisions, ignores solutions."""
+  for deps_name, deps_data in sorted(deps_url_mapping.items()):
+    if deps_name.strip('/') in solutions:
+      # This has already been forced to the correct solution by git_checkout().
+      continue
+    revision = get_revision(deps_name, deps_data.get('url', None), revisions)
+    if not revision:
+      continue
+    # TODO(hinoka): Catch SVNRevisionNotFound error maybe?
+    force_revision(deps_name, revision)
+
+
+def ensure_checkout(solutions, revisions, first_sln, target_os, target_os_only,
                     root, issue, patchset, patch_url, rietveld_server,
                     revision_mapping, buildspec_name, gyp_env, shallow):
   # Get a checkout of each solution, without DEPS or hooks.
   # Calling git directly because there is no way to run Gclient without
   # invoking DEPS.
   print 'Fetching Git checkout'
-  git_ref = git_checkout(solutions, revision, shallow)
+  git_ref = git_checkout(solutions, revisions, shallow)
 
   # If either patch_url or issue is passed in, then we need to apply a patch.
   if patch_url:
     # patch_url takes precidence since its only passed in on gcl try/git try.
     apply_issue_svn(root, patch_url)
   elif issue:
     apply_issue_rietveld(issue, patchset, root, rietveld_server,
                          revision_mapping, git_ref)
 
   if buildspec_name:
     buildspecs2git(root, buildspec_name)
   elif first_sln == root:
     # Run deps2git if there is a DEPS commit after the last .DEPS.git commit.
     # We only need to ensure deps2git if the root is not overridden, since
     # if the root is overridden, it means we are working with a sub repository
     # patch, which means its impossible for it to touch DEPS.
     ensure_deps2git(root, shallow)
 
   # Ensure our build/ directory is set up with the correct .gclient file.
   gclient_configure(solutions, target_os, target_os_only)
 
   # Let gclient do the DEPS syncing.
   gclient_output = gclient_sync(buildspec_name)
   if buildspec_name:
     # Run gclient runhooks if we're on an official builder.
     # TODO(hinoka): Remove this when the official builders run their own
     #               runhooks step.
     gclient_runhooks(gyp_env)
+
+  # Finally, ensure that all DEPS are pinned to the correct revision.
+  dir_names = [sln['name'] for sln in solutions]
+  ensure_deps_revisions(gclient_output.get('solutions', {}),
+                        dir_names, revisions)
   return gclient_output
 
 
+def parse_revisions(revisions, root):
+  """Turn a list of revision specs into a nice dictionary.
+
+  We will always return a dict with {root: something}.  By default if root

[iannucci, 2014/05/08 23:50:06]

is 'root' like 'src' or 'src/tools/gyp' ?

[hinoka, 2014/05/10 00:54:08]

both!
I'm expecting things like src and src/third_party/WebKit getting passed in.

+  is unspecified, or if revisions is [], then revision will be assigned 'HEAD'
+  """
+  results = {root.strip('/'): 'HEAD'}
+  for revision in revisions:
+    split_revision = revision.split('@')
+    if len(split_revision) == 1:
+      # This is just a plain revision, set it as the revision for root.
+      results[root] = split_revision[0]
+    elif len(split_revision) == 2:
+      # This is an alt_root@revision argument.
+      current_root, current_rev = split_revision
+
+      # We want to normalize svn/git urls into .git urls.
+      parsed_root = urlparse.urlparse(current_root)
+      if parsed_root.scheme == 'svn':
+        if parsed_root.path in RECOGNIZED_PATHS:
+          normalized_root = RECOGNIZED_PATHS[parsed_root.path]
+        else:
+          print 'WARNING: SVN path %s not recognized, ignoring' % current_root

[iannucci, 2014/05/08 23:50:06]

in general I would recommend using the logging module for this sort of thing
instead of print.

[hinoka, 2014/05/10 00:54:08]

I'll agree, but that sounds like a rabbit hole.  I've added a TODO.

+          continue
+      elif parsed_root.scheme in ['http', 'https']:
+        normalized_root = 'https://%s/%s' % (parsed_root.netloc,
+                                             parsed_root.path)
+        if not normalized_root.endswith('.git'):
+          normalized_root = '%s.git' % normalized_root
+      elif parsed_root.scheme:
+        print 'WARNING: Unrecognized scheme %s, ignoring' % parsed_root.scheme
+        continue
+      else:
+        # This is probably a local path.
+        normalized_root = current_root.strip('/')
+
+      results[normalized_root] = current_rev
+    else:
+      print ('WARNING: %s is not recognized as a valid revision specification,'

[iannucci, 2014/05/08 23:50:06]

I would use %r (everywhere) to get automatic quoting (in case it has spaces or
something).

[hinoka, 2014/05/10 00:54:08]

Done.

+             'skipping' % revision)
+  return results
+
+
 def parse_args():
   parse = optparse.OptionParser()
 
   parse.add_option('--issue', help='Issue number to patch from.')
   parse.add_option('--patchset',
                    help='Patchset from issue to patch from, if applicable.')
   parse.add_option('--patch_url', help='Optional URL to SVN patch.')
   parse.add_option('--root', help='Repository root.')
   parse.add_option('--rietveld_server',
                    default='codereview.chromium.org',
                    help='Rietveld server.')
   parse.add_option('--specs', help='Gcilent spec.')
   parse.add_option('--master', help='Master name.')
   parse.add_option('-f', '--force', action='store_true',
                    help='Bypass check to see if we want to be run. '
                         'Should ONLY be used locally.')
   parse.add_option('--revision_mapping')
   parse.add_option('--revision-mapping')  # Backwards compatability.
-  # TODO(hinoka): Support root@revision format.
-  parse.add_option('--revision',
+  parse.add_option('--revision', action='append', default=[],
                    help='Revision to check out. Can be an SVN revision number, '
-                        'git hash, or any form of git ref.')
+                        'git hash, or any form of git ref.  Can prepend '
+                        'root@<rev> to specify which repository, where root '
+                        'is either a filesystem path, git https url, or '
+                        'svn url. To specify Tip of Tree, set rev to HEAD.')

[iannucci, 2014/05/08 23:50:06]

Note that gclient would accept `--revision src@<rev>` as well, so we should
filter the 'root' portion based on the .gclient spec entries.

Where is the code that does that (I see in your examples you tried that already
:))

[hinoka, 2014/05/10 00:54:08]

Discussed offline - All revision handling is done by bot_update.
If --revision matches a spec, it enforces this at the git_checkout() level
(before gclient sync)
Otherwise, it enforces this after gclient sync.  The main reason we don't pass
it directly into "gclient sync" is because gclient wouldn't support
git_url@revision and we have no way to 100% get a correct url->folder mapping
before running gclient sync... besides rewriting a DEPS parser.

   parse.add_option('--slave_name', default=socket.getfqdn().split('.')[0],
                    help='Hostname of the current machine, '
                    'used for determining whether or not to activate.')
   parse.add_option('--builder_name', help='Name of the builder, '
                    'used for determining whether or not to activate.')
   parse.add_option('--build_dir', default=os.getcwd())
   parse.add_option('--flag_file', default=path.join(os.getcwd(),
                                                     'update.flag'))
   parse.add_option('--shallow', action='store_true',
                    help='Use shallow clones for cache repositories.')
@@ skipping 70 matching lines @@
     print '@@@STEP_TEXT@%s@@@' % step_text
   if not options.shallow:
     options.shallow = total_disk_space < SHALLOW_CLONE_THRESHOLD
 
   # By default, the root should be the name of the first solution, but
   # also make it overridable. The root is where patches are applied on top of.
   options.root =  options.root or dir_names[0]
   # The first solution is where the primary DEPS file resides.
   first_sln = dir_names[0]
 
+  # Split all the revision specifications into a nice dict.
+  print 'Revisions: %s' % options.revision
+  revisions = parse_revisions(options.revision, options.root)
+  # This is meant to be just an alias to the revision of the main solution.
+  root_revision = revisions[options.root]
+  print 'Fetching Git checkout at %s@%s' % (options.root, root_revision)
+
   try:
     checkout_parameters = dict(
         # First, pass in the base of what we want to check out.
         solutions=git_solutions,
-        revision=options.revision,
+        revisions=revisions,
         first_sln=first_sln,
 
         # Also, target os variables for gclient.
         target_os=specs.get('target_os', []),
         target_os_only=specs.get('target_os_only', False),
 
         # Then, pass in information about how to patch on top of the checkout.
         root=options.root,
         issue=options.issue,
         patchset=options.patchset,
@@ skipping 28 matching lines @@
               patch_root=options.root,
               step_text=step_text,
               properties=got_revisions)
   else:
     # If we're not on recipes, tell annotator about our got_revisions.
     emit_properties(got_revisions)
 
 
 if __name__ == '__main__':
   sys.exit(main())