Apply patch for bot_update

scripts/slave/bot_update.py

 #!/usr/bin/env python
 # Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import codecs
 import copy
+import cStringIO
+import json
 import optparse
 import os
 import pprint
+import re
 import shutil
 import socket
 import subprocess
 import sys
 import time
+import urllib2
 import urlparse
 
 import os.path as path
 
 from common import chromium_utils
 
 
 RECOGNIZED_PATHS = {
     # If SVN path matches key, the entire URL is rewritten to the Git url.
     '/chrome/trunk/src':
@@ skipping 83 matching lines @@
 THIS_DIR = path.abspath(os.getcwd())
 BUILDER_DIR = path.dirname(THIS_DIR)
 SLAVE_DIR = path.dirname(BUILDER_DIR)
 CACHE_DIR = path.join(SLAVE_DIR, 'cache_dir')
 # Because we print CACHE_DIR out into a .gclient file, and then later run
 # eval() on it, backslashes need to be escaped, otherwise "E:\b\build" gets
 # parsed as "E:[\x08][\x08]uild".
 if sys.platform.startswith('win'):
   CACHE_DIR = CACHE_DIR.replace('\\', '\\\\')
 
+# Find the patch tool.
+ROOT_BUILD_DIR = path.dirname(SLAVE_DIR)
+ROOT_B_DIR = path.dirname(ROOT_BUILD_DIR)
+BUILD_INTERNAL_DIR = path.join(ROOT_B_DIR, 'build_internal')
+if sys.platform.startswith('win'):
+  PATCH_TOOL = path.join(BUILD_INTERNAL_DIR, 'tools', 'patch.EXE')
+else:
+  PATCH_TOOL = '/usr/bin/patch'
+
 
 class SubprocessFailed(Exception):
   def __init__(self, message, code):
     Exception.__init__(self, message)
     self.code = code
 
 
 def call(*args, **kwargs):
   """Interactive subprocess call."""
   kwargs['stdout'] = subprocess.PIPE
   kwargs['stderr'] = subprocess.STDOUT
+  stdin_data = kwargs.pop('stdin_data', None)
+  if stdin_data:
+    kwargs['stdin'] = subprocess.PIPE
+  out = cStringIO.StringIO()
   for attempt in xrange(RETRIES):
     attempt_msg = ' (retry #%d)' % attempt if attempt else ''
     print '===Running %s%s===' % (' '.join(args), attempt_msg)
     start_time = time.time()
     proc = subprocess.Popen(args, **kwargs)
+    if stdin_data:
+      proc.stdin.write(stdin_data)
+      proc.stdin.close()
     # This is here because passing 'sys.stdout' into stdout for proc will
     # produce out of order output.
     while True:
       buf = proc.stdout.read(1)
       if not buf:
         break
       sys.stdout.write(buf)
+      out.write(buf)
     code = proc.wait()
     elapsed_time = ((time.time() - start_time) / 60.0)
     if not code:
       print '===Succeeded in %.1f mins===' % elapsed_time
       print
-      return 0
+      return out.getvalue()
     print '===Failed in %.1f mins===' % elapsed_time
     print
 
   raise SubprocessFailed('%s failed with code %d in %s after %d attempts.' %
                          (' '.join(args), code, os.getcwd(), RETRIES), code)
 
 
 def git(*args, **kwargs):
   """Wrapper around call specifically for Git commands."""
   git_executable = 'git'
   # On windows, subprocess doesn't fuzzy-match 'git' to 'git.bat', so we
   # have to do it explicitly. This is better than passing shell=True.
   if sys.platform.startswith('win'):
     git_executable += '.bat'
   cmd = (git_executable,) + args
-  call(*cmd, **kwargs)
+  return call(*cmd, **kwargs)
 
 
 def get_gclient_spec(solutions):
   return GCLIENT_TEMPLATE % {
       'solutions': pprint.pformat(solutions, indent=4),
       'cache_dir': '"%s"' % CACHE_DIR
   }
 
 
 def check_enabled(master, builder, slave):
@@ skipping 103 matching lines @@
   with codecs.open('.gclient', mode='w', encoding='utf-8') as f:
     f.write(get_gclient_spec(solutions))
 
 
 def gclient_sync():
   gclient_bin = 'gclient.bat' if sys.platform.startswith('win') else 'gclient'
   call(gclient_bin, 'sync', '--verbose', '--reset', '--force',
        '--nohooks', '--noprehooks')
 
 
+def create_less_than_or_equal_regex(number):

[agable, 2014/02/20 00:13:06]

Nooooooooooooo.

At the very least can we update this to use
7[0-7][0-9]{3}
style synatx?

But also I'm confused as to why this is actually necessary. Why don't the
revisions line up exactly so we can grep for an exact number?

[Ryan Tseng, 2014/02/20 01:04:37]

Imagine if you set chrome's src repo to some pinned revision x, and revision x
has files in /chrome/src/base, but you reuse that revision for
/chrome/src/tools, but the last revision checked into the /chrome/src/tools repo
is y (where y = x - 6), without this ref resolution in /chrome/src/tools will
fail.

+  """ Return a regular expression to test whether an integer less than or equal
+      to 'number' is present in a given string.
+  """
+
+  # In three parts, build a regular expression that match any numbers smaller
+  # than 'number'.
+  # For example, 78656 would give a regular expression that looks like:
+  # Part 1
+  # (78356|                        # 78356
+  # Part 2
+  #  7835[0-5]|                    # 78350-78355
+  #  783[0-4][0-9]|                # 78300-78349
+  #  78[0-2][0-9][0-9]|            # 78000-78299
+  #  7[0-7][0-9][0-9][0-9]|        # 70000-77999
+  #  [0-6][0-9][0-9][0-9][0-9]|    # 10000-69999
+  # Part 3
+  #  [0-9][0-9][0-9][0-9]|         # 1000-9999
+  #  [0-9][0-9][0-9]|              # 100-999
+  #  [0-9][0-9]|                   # 10-99
+  #  [0-9])                        # 0-9
+
+  # Part 1: Create an array with all the regexes, as described above.
+  # Prepopulate it with the number itself.
+  number = str(number)
+  expressions = [number]
+
+  # Convert the number to a list, so we can translate digits in it to
+  # expressions.
+  num_list = list(number)
+  num_len = len(num_list)
+
+  # Part 2: Go through all the digits in the number, starting from the end.
+  # Each iteration appends a line to 'expressions'.
+  for index in range (num_len - 1, -1, -1):
+    # Convert this digit back to an integer.
+    digit = int(num_list[index])
+
+    # Part 2.1: No processing if this digit is a zero.
+    if digit == 0:
+      continue
+
+    # Part 2.2: We switch the current digit X by a range "[0-(X-1)]".
+    num_list[index] = '[0-%d]' % (digit - 1)
+
+    # Part 2.3: We set all following digits to be "[0-9]".
+    # Since we just decrementented a digit in a most important position, all
+    # following digits don't matter. The possible numbers will always be smaller
+    # than before we decremented.
+    for next_digit in range(index + 1, num_len):
+      num_list[next_digit] = '[0-9]'
+
+    # Part 2.4: Add this new sub-expression to the list.
+    expressions.append(''.join(num_list))
+
+  # Part 3: We add all the full ranges to match all numbers that are at least
+  # one order of magnitude smaller than the original numbers.
+  for index in range(1, num_len):
+    expressions.append('[0-9]'*index)
+
+  # All done. We now have our final regular expression.
+  regex = '(%s)' % ('|'.join(expressions))
+  return regex
+
+
 def get_git_hash(revision, dir_name):
-  match = "^git-svn-id: [^ ]*@%d" % revision
-  cmd = ['git', 'log', '--grep', match, '--format=%H', dir_name]
-  return subprocess.check_output(cmd).strip() or None
+  match = "^git-svn-id: [^ ]*@%s " % create_less_than_or_equal_regex(revision)
+  cmd = ['git', 'log', '-E', '--grep', match, '--format=%H', '--max-count=1']
+  results = call(*cmd, cwd=dir_name).strip().splitlines()
+  if results:
+    return results[0]
+  raise Exception('We can\'t resolve svn revision %s into a git hash' %
+                  revision)
 
 
 def deps2git(sln_dirs):
   for sln_dir in sln_dirs:
     deps_file = path.join(os.getcwd(), sln_dir, 'DEPS')
     deps_git_file = path.join(os.getcwd(), sln_dir, '.DEPS.git')
     if not path.isfile(deps_file):
       return
     # Do we have a better way of doing this....?
     repo_type = 'internal' if 'internal' in sln_dir else 'public'
     call(sys.executable, DEPS2GIT_PATH, '-t', repo_type,
          '--cache_dir=%s' % CACHE_DIR,
          '--deps=%s' % deps_file, '--out=%s' % deps_git_file)
 
 
+def emit_got_revision(revision):
+  print '@@@SET_BUILD_PROPERTY@got_revision@%s@@@' % revision
+
 def git_checkout(solutions, revision):
   build_dir = os.getcwd()
   # Revision only applies to the first solution.
   first_solution = True
   for sln in solutions:
     name = sln['name']
     url = sln['url']
     sln_dir = path.join(build_dir, name)
     if not path.isdir(sln_dir):
       git('clone', url, sln_dir)
 
     # Clean out .DEPS.git changes first.
     try:
       git('reset', '--hard', cwd=sln_dir)
     except SubprocessFailed as e:
       if e.code == 128:
         # Exited abnormally, theres probably something wrong with the checkout.
         # Lets wipe the checkout and try again.
         chromium_utils.RemoveDirectory(sln_dir)
         git('clone', url, sln_dir)
         git('reset', '--hard', cwd=sln_dir)
       else:
         raise
 
     git('clean', '-df', cwd=sln_dir)
     git('pull', 'origin', 'master', cwd=sln_dir)
     # TODO(hinoka): We probably have to make use of revision mapping.
     if first_solution and revision and revision.lower() != 'head':
+      emit_got_revision(revision)
       if revision and revision.isdigit() and len(revision) < 40:
         # rev_num is really a svn revision number, convert it into a git hash.
-        git_ref = get_git_hash(revision, name)
+        git_ref = get_git_hash(int(revision), name)
       else:
         # rev_num is actually a git hash or ref, we can just use it.
         git_ref = revision
       git('checkout', git_ref, cwd=sln_dir)
     else:
       git('checkout', 'origin/master', cwd=sln_dir)
+      if first_solution:
+        git_ref = git('log', '--format=%H', '--max-count=1',
+                      cwd=sln_dir).strip()
 
     first_solution = False
+  return git_ref
 
 
-def apply_issue(issue, patchset, root, server):
-  pass
+def _download(url):
+  """Fetch url and return content, with retries for flake."""
+  for attempt in xrange(RETRIES):
+    try:
+      return urllib2.urlopen(url).read()
+    except Exception:
+      if attempt == RETRIES - 1:
+        raise
+      pass
+
+
+def apply_issue_svn(root, patch_url):
+  patch_data = call('svn', 'cat', patch_url)
+  call(PATCH_TOOL, '-p0', '--remove-empty-files', '--force', '--forward',
+       stdin_data=patch_data, cwd=root)
+
+
+def apply_issue_rietveld(issue, patchset, root, server, rev_map, revision):
+  apply_issue_bin = ('apply_issue.bat' if sys.platform.startswith('win')

[agable, 2014/02/20 00:13:06]

It's really truly necessary to be using apply_issue? It does scary things like
commit the patch and stuff. Why was the switch away from 'git apply' necessary?

[Ryan Tseng, 2014/02/20 01:04:37]

apply_issue has a crapton of logic to deal with every corner case that rietveld
throws at us, including svn patches, git patches, binary files that don't fit in
patches so need to be downloaded individually, etc.  The intent is to not
reimplement all of that here, and just match the current behavior.

+                     else 'apply_issue')
+  rev_map = json.loads(rev_map)
+  if root in rev_map and rev_map[root] == 'got_revision':
+    rev_map[root] = revision
+  call(apply_issue_bin,
+       '--root_dir', root,
+       '--issue', issue,
+       '--patchset', patchset,
+       '--no-auth',
+       '--server', server,
+       '--revision-mapping', json.dumps(rev_map),
+       '--base_ref', revision,
+       '--force')
 
 
 def check_flag(flag_file):
   """Returns True if the flag file is present."""
   return os.path.isfile(flag_file)
 
 
 def delete_flag(flag_file):
   """Remove bot update flag."""
   if os.path.isfile(flag_file):
     os.remove(flag_file)
 
 
 def emit_flag(flag_file):
   """Deposit a bot update flag on the system to tell gclient not to run."""
   print 'Emitting flag file at %s' % flag_file
   with open(flag_file, 'wb') as f:
     f.write('Success!')
 
 
 def parse_args():
   parse = optparse.OptionParser()
 
   parse.add_option('--issue', help='Issue number to patch from.')
   parse.add_option('--patchset',
                    help='Patchset from issue to patch from, if applicable.')
   parse.add_option('--patch_url', help='Optional URL to SVN patch.')
   parse.add_option('--root', help='Repository root.')
-  parse.add_option('--rietveld_server', help='Rietveld server.')
+  parse.add_option('--rietveld_server',
+                   default='codereview.chromium.org',
+                   help='Rietveld server.')
   parse.add_option('--specs', help='Gcilent spec.')
   parse.add_option('--master', help='Master name.')
   parse.add_option('-f', '--force', action='store_true',
                    help='Bypass check to see if we want to be run. '
                         'Should ONLY be used locally.')
-  # TODO(hinoka): We don't actually use this yet, we should factor this in.
-  parse.add_option('--revision-mapping')
+  parse.add_option('--revision_mapping')
   parse.add_option('--revision')
   parse.add_option('--slave_name', default=socket.getfqdn().split('.')[0],
                    help='Hostname of the current machine, '
                    'used for determining whether or not to activate.')
   parse.add_option('--builder_name', help='Name of the builder, '
                    'used for determining whether or not to activate.')
   parse.add_option('--build_dir', default=os.getcwd())
   parse.add_option('--flag_file', default=path.join(os.getcwd(),
                                                           'update.flag'))
 
@@ skipping 35 matching lines @@
     ensure_no_checkout(dir_names, '.svn')
     emit_flag(options.flag_file)
   else:
     delete_flag(options.flag_file)
     return
 
   # Get a checkout of each solution, without DEPS or hooks.
   # Calling git directory because there is no way to run Gclient without
   # invoking DEPS.
   print 'Fetching Git checkout'
-  git_checkout(git_solutions, options.revision)
+  got_revision = git_checkout(git_solutions, options.revision)
 
-  # TODO(hinoka): This must be implemented before we can turn this on for TS.
-  # if options.issue:
-  #   apply_issue(options.issue, options.patchset, options.root, options.server)
+  options.root =  options.root or dir_names[0]
+  if options.patch_url:
+    apply_issue_svn(options.root, options.patch_url)
+  elif options.issue:
+    apply_issue_rietveld(options.issue, options.patchset, options.root,
+                         options.rietveld_server, options.revision_mapping,
+                         got_revision)
 
   # Magic to get deps2git to work with internal DEPS.
   shutil.copyfile(S2G_INTERNAL_FROM_PATH, S2G_INTERNAL_DEST_PATH)
   deps2git(dir_names)
 
   gclient_configure(git_solutions)
   gclient_sync()
 
 
 if __name__ == '__main__':
   sys.exit(main())