Experimental parser: merge to r19637

tools/push-to-trunk/common_includes.py

 #!/usr/bin/env python
 # Copyright 2013 the V8 project authors. All rights reserved.
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 #       notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
 #       copyright notice, this list of conditions and the following
 #       disclaimer in the documentation and/or other materials provided
 #       with the distribution.
 #     * Neither the name of Google Inc. nor the names of its
 #       contributors may be used to endorse or promote products derived
 #       from this software without specific prior written permission.
 #
 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 import datetime
+import json
 import os
 import re
 import subprocess
 import sys
 import textwrap
 import time
 import urllib2
 
+from git_recipes import GitRecipesMixin
+
 PERSISTFILE_BASENAME = "PERSISTFILE_BASENAME"
 TEMP_BRANCH = "TEMP_BRANCH"
 BRANCHNAME = "BRANCHNAME"
 DOT_GIT_LOCATION = "DOT_GIT_LOCATION"
 VERSION_FILE = "VERSION_FILE"
 CHANGELOG_FILE = "CHANGELOG_FILE"
 CHANGELOG_ENTRY_FILE = "CHANGELOG_ENTRY_FILE"
 COMMITMSG_FILE = "COMMITMSG_FILE"
 PATCH_FILE = "PATCH_FILE"
 
@@ skipping 164 matching lines @@
   def GetDate(self):
     return datetime.date.today().strftime("%Y-%m-%d")
 
 DEFAULT_SIDE_EFFECT_HANDLER = SideEffectHandler()
 
 
 class NoRetryException(Exception):
   pass
 
 
+class GitFailedException(Exception):
+  pass
+
+
 class CommonOptions(object):
   def __init__(self, options, manual=True):
     self.requires_editor = True
     self.wait_for_lgtm = True
-    self.s = options.s
+    self.step = options.step
     self.force_readline_defaults = not manual
     self.force_upload = not manual
     self.manual = manual
+    self.reviewer = getattr(options, 'reviewer', "")
+    self.author = getattr(options, 'author', "")
 
 
-class Step(object):
+class Step(GitRecipesMixin):
   def __init__(self, text, requires, number, config, state, options, handler):
     self._text = text
     self._requires = requires
     self._number = number
     self._config = config
     self._state = state
     self._options = options
     self._side_effect_handler = handler
     assert self._number >= 0
     assert self._config is not None
     assert self._state is not None
     assert self._side_effect_handler is not None
     assert isinstance(options, CommonOptions)
 
+  def __getitem__(self, key):
+    # Convenience method to allow direct [] access on step classes for
+    # manipulating the backed state dict.
+    return self._state[key]
+
+  def __setitem__(self, key, value):
+    # Convenience method to allow direct [] access on step classes for
+    # manipulating the backed state dict.
+    self._state[key] = value
+
   def Config(self, key):
     return self._config[key]
 
   def Run(self):
-    if self._requires:
-      self.RestoreIfUnset(self._requires)
-      if not self._state[self._requires]:
-        return
+    # Restore state.
+    state_file = "%s-state.json" % self._config[PERSISTFILE_BASENAME]
+    if not self._state and os.path.exists(state_file):
+      self._state.update(json.loads(FileToText(state_file)))
+
+    # Skip step if requirement is not met.
+    if self._requires and not self._state.get(self._requires):
+      return
+
     print ">>> Step %d: %s" % (self._number, self._text)
     self.RunStep()
 
+    # Persist state.
+    TextToFile(json.dumps(self._state), state_file)
+
   def RunStep(self):
     raise NotImplementedError
 
   def Retry(self, cb, retry_on=None, wait_plan=None):
     """ Retry a function.
     Params:
       cb: The function to retry.
       retry_on: A callback that takes the result of the function and returns
                 True if the function should be retried. A function throwing an
                 exception is always retried.
@@ skipping 24 matching lines @@
   def ReadLine(self, default=None):
     # Don't prompt in forced mode.
     if self._options.force_readline_defaults and default is not None:
       print "%s (forced)" % default
       return default
     else:
       return self._side_effect_handler.ReadLine()
 
   def Git(self, args="", prefix="", pipe=True, retry_on=None):
     cmd = lambda: self._side_effect_handler.Command("git", args, prefix, pipe)
+    result = self.Retry(cmd, retry_on, [5, 30])
+    if result is None:
+      raise GitFailedException("'git %s' failed." % args)
+    return result
+
+  def SVN(self, args="", prefix="", pipe=True, retry_on=None):
+    cmd = lambda: self._side_effect_handler.Command("svn", args, prefix, pipe)
     return self.Retry(cmd, retry_on, [5, 30])
 
   def Editor(self, args):
     if self._options.requires_editor:
       return self._side_effect_handler.Command(os.environ["EDITOR"], args,
                                                pipe=False)
 
   def ReadURL(self, url, params=None, retry_on=None, wait_plan=None):
     wait_plan = wait_plan or [3, 60, 600]
     cmd = lambda: self._side_effect_handler.ReadURL(url, params)
@@ skipping 12 matching lines @@
     if not self._options.manual:
       msg = msg or "Only available in manual mode."
       self.Die(msg)
 
   def Confirm(self, msg):
     print "%s [Y/n] " % msg,
     answer = self.ReadLine(default="Y")
     return answer == "" or answer == "Y" or answer == "y"
 
   def DeleteBranch(self, name):
-    git_result = self.Git("branch").strip()
-    for line in git_result.splitlines():
+    for line in self.GitBranch().splitlines():
       if re.match(r".*\s+%s$" % name, line):
         msg = "Branch %s exists, do you want to delete it?" % name
         if self.Confirm(msg):
-          if self.Git("branch -D %s" % name) is None:
-            self.Die("Deleting branch '%s' failed." % name)
+          self.GitDeleteBranch(name)
           print "Branch %s deleted." % name
         else:
           msg = "Can't continue. Please delete branch %s and try again." % name
           self.Die(msg)
 
-  def Persist(self, var, value):
-    value = value or "__EMPTY__"
-    TextToFile(value, "%s-%s" % (self._config[PERSISTFILE_BASENAME], var))
-
-  def Restore(self, var):
-    value = FileToText("%s-%s" % (self._config[PERSISTFILE_BASENAME], var))
-    value = value or self.Die("Variable '%s' could not be restored." % var)
-    return "" if value == "__EMPTY__" else value
-
-  def RestoreIfUnset(self, var_name):
-    if self._state.get(var_name) is None:
-      self._state[var_name] = self.Restore(var_name)
-
   def InitialEnvironmentChecks(self):
     # Cancel if this is not a git checkout.
     if not os.path.exists(self._config[DOT_GIT_LOCATION]):
       self.Die("This is not a git checkout, this script won't work for you.")
 
     # Cancel if EDITOR is unset or not executable.
     if (self._options.requires_editor and (not os.environ.get("EDITOR") or
         Command("which", os.environ["EDITOR"]) is None)):
       self.Die("Please set your EDITOR environment variable, you'll need it.")
 
   def CommonPrepare(self):
     # Check for a clean workdir.
-    if self.Git("status -s -uno").strip() != "":
+    if not self.GitIsWorkdirClean():
       self.Die("Workspace is not clean. Please commit or undo your changes.")
 
     # Persist current branch.
-    current_branch = ""
-    git_result = self.Git("status -s -b -uno").strip()
-    for line in git_result.splitlines():
-      match = re.match(r"^## (.+)", line)
-      if match:
-        current_branch = match.group(1)
-        break
-    self.Persist("current_branch", current_branch)
+    self["current_branch"] = self.GitCurrentBranch()
 
     # Fetch unfetched revisions.
-    if self.Git("svn fetch") is None:
-      self.Die("'git svn fetch' failed.")
+    self.GitSVNFetch()
 
   def PrepareBranch(self):
     # Get ahold of a safe temporary branch and check it out.
-    self.RestoreIfUnset("current_branch")
-    if self._state["current_branch"] != self._config[TEMP_BRANCH]:
+    if self["current_branch"] != self._config[TEMP_BRANCH]:
       self.DeleteBranch(self._config[TEMP_BRANCH])
-      self.Git("checkout -b %s" % self._config[TEMP_BRANCH])
+      self.GitCreateBranch(self._config[TEMP_BRANCH])
 
     # Delete the branch that will be created later if it exists already.
     self.DeleteBranch(self._config[BRANCHNAME])
 
   def CommonCleanup(self):
-    self.RestoreIfUnset("current_branch")
-    self.Git("checkout -f %s" % self._state["current_branch"])
-    if self._config[TEMP_BRANCH] != self._state["current_branch"]:
-      self.Git("branch -D %s" % self._config[TEMP_BRANCH])
-    if self._config[BRANCHNAME] != self._state["current_branch"]:
-      self.Git("branch -D %s" % self._config[BRANCHNAME])
+    self.GitCheckout(self["current_branch"])
+    if self._config[TEMP_BRANCH] != self["current_branch"]:
+      self.GitDeleteBranch(self._config[TEMP_BRANCH])
+    if self._config[BRANCHNAME] != self["current_branch"]:
+      self.GitDeleteBranch(self._config[BRANCHNAME])
 
     # Clean up all temporary files.
     Command("rm", "-f %s*" % self._config[PERSISTFILE_BASENAME])
 
   def ReadAndPersistVersion(self, prefix=""):
     def ReadAndPersist(var_name, def_name):
       match = re.match(r"^#define %s\s+(\d*)" % def_name, line)
       if match:
         value = match.group(1)
-        self.Persist("%s%s" % (prefix, var_name), value)
-        self._state["%s%s" % (prefix, var_name)] = value
+        self["%s%s" % (prefix, var_name)] = value
     for line in LinesInFile(self._config[VERSION_FILE]):
       for (var_name, def_name) in [("major", "MAJOR_VERSION"),
                                    ("minor", "MINOR_VERSION"),
                                    ("build", "BUILD_NUMBER"),
                                    ("patch", "PATCH_LEVEL")]:
         ReadAndPersist(var_name, def_name)
 
-  def RestoreVersionIfUnset(self, prefix=""):
-    for v in ["major", "minor", "build", "patch"]:
-      self.RestoreIfUnset("%s%s" % (prefix, v))
-
   def WaitForLGTM(self):
     print ("Please wait for an LGTM, then type \"LGTM<Return>\" to commit "
            "your change. (If you need to iterate on the patch or double check "
            "that it's sane, do so in another shell, but remember to not "
            "change the headline of the uploaded CL.")
     answer = ""
     while answer != "LGTM":
       print "> ",
       answer = self.ReadLine(None if self._options.wait_for_lgtm else "LGTM")
       if answer != "LGTM":
         print "That was not 'LGTM'."
 
   def WaitForResolvingConflicts(self, patch_file):
     print("Applying the patch \"%s\" failed. Either type \"ABORT<Return>\", "
           "or resolve the conflicts, stage *all* touched files with "
           "'git add', and type \"RESOLVED<Return>\"")
     self.DieNoManualMode()
     answer = ""
     while answer != "RESOLVED":
       if answer == "ABORT":
         self.Die("Applying the patch failed.")
       if answer != "":
         print "That was not 'RESOLVED' or 'ABORT'."
       print "> ",
       answer = self.ReadLine()
 
   # Takes a file containing the patch to apply as first argument.
-  def ApplyPatch(self, patch_file, reverse_patch=""):
-    args = "apply --index --reject %s \"%s\"" % (reverse_patch, patch_file)
-    if self.Git(args) is None:
+  def ApplyPatch(self, patch_file, revert=False):
+    try:
+      self.GitApplyPatch(patch_file, revert)
+    except GitFailedException:
       self.WaitForResolvingConflicts(patch_file)
 
+  def FindLastTrunkPush(self, parent_hash=""):
+    push_pattern = "^Version [[:digit:]]*\.[[:digit:]]*\.[[:digit:]]* (based"
+    branch = "" if parent_hash else "svn/trunk"
+    return self.GitLog(n=1, format="%H", grep=push_pattern,
+                       parent_hash=parent_hash, branch=branch)
+
 
 class UploadStep(Step):
   MESSAGE = "Upload for code review."
 
   def RunStep(self):
-    if self._options.r:
-      print "Using account %s for review." % self._options.r
-      reviewer = self._options.r
+    if self._options.reviewer:
+      print "Using account %s for review." % self._options.reviewer
+      reviewer = self._options.reviewer
     else:
       print "Please enter the email address of a V8 reviewer for your patch: ",
       self.DieNoManualMode("A reviewer must be specified in forced mode.")
       reviewer = self.ReadLine()
-    force_flag = " -f" if self._options.force_upload else ""
-    args = "cl upload -r \"%s\" --send-mail%s" % (reviewer, force_flag)
-    # TODO(machenbach): Check output in forced mode. Verify that all required
-    # base files were uploaded, if not retry.
-    if self.Git(args, pipe=False) is None:
-      self.Die("'git cl upload' failed, please try again.")
+    self.GitUpload(reviewer, self._options.author, self._options.force_upload)
 
 
 def MakeStep(step_class=Step, number=0, state=None, config=None,
              options=None, side_effect_handler=DEFAULT_SIDE_EFFECT_HANDLER):
     # Allow to pass in empty dictionaries.
     state = state if state is not None else {}
     config = config if config is not None else {}
 
     try:
       message = step_class.MESSAGE
     except AttributeError:
       message = step_class.__name__
     try:
       requires = step_class.REQUIRES
     except AttributeError:
       requires = None
 
     return step_class(message, requires, number=number, config=config,
                       state=state, options=options,
                       handler=side_effect_handler)
 
 
 def RunScript(step_classes,
               config,
               options,
               side_effect_handler=DEFAULT_SIDE_EFFECT_HANDLER):
+  state_file = "%s-state.json" % config[PERSISTFILE_BASENAME]
+  if options.step == 0 and os.path.exists(state_file):
+    os.remove(state_file)
   state = {}
   steps = []
   for (number, step_class) in enumerate(step_classes):
     steps.append(MakeStep(step_class, number, state, config,
                           options, side_effect_handler))
 
-  for step in steps[options.s:]:
+  for step in steps[options.step:]:
     step.Run()