Experimental parser: merge to r19637

tools/push-to-trunk/push_to_trunk.py

Index: tools/push-to-trunk/push_to_trunk.py
diff --git a/tools/push-to-trunk/push_to_trunk.py b/tools/push-to-trunk/push_to_trunk.py
index 06e277ee468c23b02ba2988722ac94657b244999..d227a50bede6b05af191ddb65cbd61ecb720613a 100755
--- a/tools/push-to-trunk/push_to_trunk.py
+++ b/tools/push-to-trunk/push_to_trunk.py
@@ -26,7 +26,7 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-import optparse
+import argparse
 import sys
 import tempfile
 import urllib2
@@ -51,30 +51,38 @@ CONFIG = {
   DEPS_FILE: "DEPS",
 }
 
+PUSH_MESSAGE_SUFFIX = " (based on bleeding_edge revision r%d)"
+PUSH_MESSAGE_RE = re.compile(r".* \(based on bleeding_edge revision r(\d+)\)$")
+
 
 class PushToTrunkOptions(CommonOptions):
   @staticmethod
-  def MakeForcedOptions(reviewer, chrome_path):
+  def MakeForcedOptions(author, reviewer, chrome_path):
     """Convenience wrapper."""
     class Options(object):
       pass
     options = Options()
-    options.s = 0
-    options.l = None
-    options.f = True
-    options.m = False
-    options.r = reviewer
-    options.c = chrome_path
+    options.step = 0
+    options.last_push = None
+    options.last_bleeding_edge = None
+    options.force = True
+    options.manual = False
+    options.chromium = chrome_path
+    options.reviewer = reviewer
+    options.author = author
     return PushToTrunkOptions(options)
 
   def __init__(self, options):
-    super(PushToTrunkOptions, self).__init__(options, options.m)
-    self.requires_editor = not options.f
-    self.wait_for_lgtm = not options.f
-    self.tbr_commit = not options.m
-    self.l = options.l
-    self.r = options.r
-    self.c = options.c
+    super(PushToTrunkOptions, self).__init__(options, options.manual)
+    self.requires_editor = not options.force
+    self.wait_for_lgtm = not options.force
+    self.tbr_commit = not options.manual
+    self.last_push = options.last_push
+    self.reviewer = options.reviewer
+    self.chromium = options.chromium
+    self.last_bleeding_edge = getattr(options, 'last_bleeding_edge', None)
+    self.author = getattr(options, 'author', None)
+
 
 class Preparation(Step):
   MESSAGE = "Preparation."
@@ -90,26 +98,45 @@ class FreshBranch(Step):
   MESSAGE = "Create a fresh branch."
 
   def RunStep(self):
-    args = "checkout -b %s svn/bleeding_edge" % self.Config(BRANCHNAME)
-    if self.Git(args) is None:
-      self.Die("Creating branch %s failed." % self.Config(BRANCHNAME))
+    self.GitCreateBranch(self.Config(BRANCHNAME), "svn/bleeding_edge")
 
 
 class DetectLastPush(Step):
   MESSAGE = "Detect commit ID of last push to trunk."
 
   def RunStep(self):
-    last_push = (self._options.l or
-                 self.Git("log -1 --format=%H ChangeLog").strip())
+    last_push = self._options.last_push or self.FindLastTrunkPush()
     while True:
       # Print assumed commit, circumventing git's pager.
-      print self.Git("log -1 %s" % last_push)
+      print self.GitLog(n=1, git_hash=last_push)
       if self.Confirm("Is the commit printed above the last push to trunk?"):
         break
-      args = "log -1 --format=%H %s^ ChangeLog" % last_push
-      last_push = self.Git(args).strip()
-    self.Persist("last_push", last_push)
-    self._state["last_push"] = last_push
+      last_push = self.FindLastTrunkPush(parent_hash=last_push)
+
+    if self._options.last_bleeding_edge:
+      # Read the bleeding edge revision of the last push from a command-line
+      # option.
+      last_push_bleeding_edge = self._options.last_bleeding_edge
+    else:
+      # Retrieve the bleeding edge revision of the last push from the text in
+      # the push commit message.
+      last_push_title = self.GitLog(n=1, format="%s", git_hash=last_push)
+      last_push_be_svn = PUSH_MESSAGE_RE.match(last_push_title).group(1)
+      if not last_push_be_svn:
+        self.Die("Could not retrieve bleeding edge revision for trunk push %s"
+                 % last_push)
+      last_push_bleeding_edge = self.GitSVNFindGitHash(last_push_be_svn)
+      if not last_push_bleeding_edge:
+        self.Die("Could not retrieve bleeding edge git hash for trunk push %s"
+                 % last_push)
+
+    # TODO(machenbach): last_push_trunk points to the svn revision on trunk.
+    # It is not used yet but we'll need it for retrieving the current version.
+    self["last_push_trunk"] = last_push
+    # TODO(machenbach): This currently points to the prepare push revision that
+    # will be deprecated soon. After the deprecation it will point to the last
+    # bleeding_edge revision that went into the last push.
+    self["last_push_bleeding_edge"] = last_push_bleeding_edge
 
 
 class PrepareChangeLog(Step):
@@ -123,7 +150,8 @@ class PrepareChangeLog(Step):
     match = re.search(r"^Review URL: https://codereview\.chromium\.org/(\d+)$",
                       body, flags=re.M)
     if match:
-      cl_url = "https://codereview.chromium.org/%s/description" % match.group(1)
+      cl_url = ("https://codereview.chromium.org/%s/description"
+                % match.group(1))
       try:
         # Fetch from Rietveld but only retry once with one second delay since
         # there might be many revisions.
@@ -133,28 +161,24 @@ class PrepareChangeLog(Step):
     return body
 
   def RunStep(self):
-    self.RestoreIfUnset("last_push")
-
     # These version numbers are used again later for the trunk commit.
     self.ReadAndPersistVersion()
-
-    date = self.GetDate()
-    self.Persist("date", date)
-    output = "%s: Version %s.%s.%s\n\n" % (date,
-                                           self._state["major"],
-                                           self._state["minor"],
-                                           self._state["build"])
+    self["date"] = self.GetDate()
+    self["version"] = "%s.%s.%s" % (self["major"],
+                                    self["minor"],
+                                    self["build"])
+    output = "%s: Version %s\n\n" % (self["date"],
+                                     self["version"])
     TextToFile(output, self.Config(CHANGELOG_ENTRY_FILE))
-
-    args = "log %s..HEAD --format=%%H" % self._state["last_push"]
-    commits = self.Git(args).strip()
+    commits = self.GitLog(format="%H",
+        git_hash="%s..HEAD" % self["last_push_bleeding_edge"])
 
     # Cache raw commit messages.
     commit_messages = [
       [
-        self.Git("log -1 %s --format=\"%%s\"" % commit),
-        self.Reload(self.Git("log -1 %s --format=\"%%B\"" % commit)),
-        self.Git("log -1 %s --format=\"%%an\"" % commit),
+        self.GitLog(n=1, format="%s", git_hash=commit),
+        self.Reload(self.GitLog(n=1, format="%B", git_hash=commit)),
+        self.GitLog(n=1, format="%an", git_hash=commit),
       ] for commit in commits.splitlines()
     ]
 
@@ -207,8 +231,7 @@ class IncrementVersion(Step):
   MESSAGE = "Increment version number."
 
   def RunStep(self):
-    self.RestoreIfUnset("build")
-    new_build = str(int(self._state["build"]) + 1)
+    new_build = str(int(self["build"]) + 1)
 
     if self.Confirm(("Automatically increment BUILD_NUMBER? (Saying 'n' will "
                      "fire up your EDITOR on %s so you can make arbitrary "
@@ -229,18 +252,19 @@ class CommitLocal(Step):
   MESSAGE = "Commit to local branch."
 
   def RunStep(self):
-    self.RestoreVersionIfUnset("new_")
-    prep_commit_msg = ("Prepare push to trunk.  "
-        "Now working on version %s.%s.%s." % (self._state["new_major"],
-                                              self._state["new_minor"],
-                                              self._state["new_build"]))
-    self.Persist("prep_commit_msg", prep_commit_msg)
+    self["prep_commit_msg"] = ("Prepare push to trunk.  "
+        "Now working on version %s.%s.%s." % (self["new_major"],
+                                              self["new_minor"],
+                                              self["new_build"]))
 
     # Include optional TBR only in the git command. The persisted commit
     # message is used for finding the commit again later.
-    review = "\n\nTBR=%s" % self._options.r if self._options.tbr_commit else ""
-    if self.Git("commit -a -m \"%s%s\"" % (prep_commit_msg, review)) is None:
-      self.Die("'git commit -a' failed.")
+    if self._options.tbr_commit:
+      message = "%s\n\nTBR=%s" % (self["prep_commit_msg"],
+                                  self._options.reviewer)
+    else:
+      message = "%s" % self["prep_commit_msg"]
+    self.GitCommit(message)
 
 
 class CommitRepository(Step):
@@ -253,11 +277,8 @@ class CommitRepository(Step):
     TextToFile(GetLastChangeLogEntries(self.Config(CHANGELOG_FILE)),
                self.Config(CHANGELOG_ENTRY_FILE))
 
-    if self.Git("cl presubmit", "PRESUBMIT_TREE_CHECK=\"skip\"") is None:
-      self.Die("'git cl presubmit' failed, please try again.")
-
-    if self.Git("cl dcommit -f --bypass-hooks") is None:
-      self.Die("'git cl dcommit' failed, please try again.")
+    self.GitPresubmit()
+    self.GitDCommit()
 
 
 class StragglerCommits(Step):
@@ -265,13 +286,10 @@ class StragglerCommits(Step):
              "started.")
 
   def RunStep(self):
-    if self.Git("svn fetch") is None:
-      self.Die("'git svn fetch' failed.")
-    self.Git("checkout svn/bleeding_edge")
-    self.RestoreIfUnset("prep_commit_msg")
-    args = "log -1 --format=%%H --grep=\"%s\"" % self._state["prep_commit_msg"]
-    prepare_commit_hash = self.Git(args).strip()
-    self.Persist("prepare_commit_hash", prepare_commit_hash)
+    self.GitSVNFetch()
+    self.GitCheckout("svn/bleeding_edge")
+    self["prepare_commit_hash"] = self.GitLog(n=1, format="%H",
+                                              grep=self["prep_commit_msg"])
 
 
 class SquashCommits(Step):
@@ -280,16 +298,20 @@ class SquashCommits(Step):
   def RunStep(self):
     # Instead of relying on "git rebase -i", we'll just create a diff, because
     # that's easier to automate.
-    self.RestoreIfUnset("prepare_commit_hash")
-    args = "diff svn/trunk %s" % self._state["prepare_commit_hash"]
-    TextToFile(self.Git(args), self.Config(PATCH_FILE))
+    TextToFile(self.GitDiff("svn/trunk", self["prepare_commit_hash"]),
+               self.Config(PATCH_FILE))
 
     # Convert the ChangeLog entry to commit message format.
-    self.RestoreIfUnset("date")
     text = FileToText(self.Config(CHANGELOG_ENTRY_FILE))
 
     # Remove date and trailing white space.
-    text = re.sub(r"^%s: " % self._state["date"], "", text.rstrip())
+    text = re.sub(r"^%s: " % self["date"], "", text.rstrip())
+
+    # Retrieve svn revision for showing the used bleeding edge revision in the
+    # commit message.
+    self["svn_revision"] = self.GitSVNFindSVNRev(self["prepare_commit_hash"])
+    suffix = PUSH_MESSAGE_SUFFIX % int(self["svn_revision"])
+    text = MSub(r"^(Version \d+\.\d+\.\d+)$", "\\1%s" % suffix, text)
 
     # Remove indentation and merge paragraphs into single long lines, keeping
     # empty lines between them.
@@ -308,9 +330,7 @@ class NewBranch(Step):
   MESSAGE = "Create a new branch from trunk."
 
   def RunStep(self):
-    if self.Git("checkout -b %s svn/trunk" % self.Config(TRUNKBRANCH)) is None:
-      self.Die("Checking out a new branch '%s' failed." %
-               self.Config(TRUNKBRANCH))
+    self.GitCreateBranch(self.Config(TRUNKBRANCH), "svn/trunk")
 
 
 class ApplyChanges(Step):
@@ -325,15 +345,14 @@ class SetVersion(Step):
   MESSAGE = "Set correct version for trunk."
 
   def RunStep(self):
-    self.RestoreVersionIfUnset()
     output = ""
     for line in FileToText(self.Config(VERSION_FILE)).splitlines():
       if line.startswith("#define MAJOR_VERSION"):
-        line = re.sub("\d+$", self._state["major"], line)
+        line = re.sub("\d+$", self["major"], line)
       elif line.startswith("#define MINOR_VERSION"):
-        line = re.sub("\d+$", self._state["minor"], line)
+        line = re.sub("\d+$", self["minor"], line)
       elif line.startswith("#define BUILD_NUMBER"):
-        line = re.sub("\d+$", self._state["build"], line)
+        line = re.sub("\d+$", self["build"], line)
       elif line.startswith("#define PATCH_LEVEL"):
         line = re.sub("\d+$", "0", line)
       elif line.startswith("#define IS_CANDIDATE_VERSION"):
@@ -346,9 +365,8 @@ class CommitTrunk(Step):
   MESSAGE = "Commit to local trunk branch."
 
   def RunStep(self):
-    self.Git("add \"%s\"" % self.Config(VERSION_FILE))
-    if self.Git("commit -F \"%s\"" % self.Config(COMMITMSG_FILE)) is None:
-      self.Die("'git commit' failed.")
+    self.GitAdd(self.Config(VERSION_FILE))
+    self.GitCommit(file_name = self.Config(COMMITMSG_FILE))
     Command("rm", "-f %s*" % self.Config(COMMITMSG_FILE))
 
 
@@ -366,53 +384,46 @@ class CommitSVN(Step):
   MESSAGE = "Commit to SVN."
 
   def RunStep(self):
-    result = self.Git("svn dcommit 2>&1")
+    result = self.GitSVNDCommit()
     if not result:
       self.Die("'git svn dcommit' failed.")
     result = filter(lambda x: re.search(r"^Committed r[0-9]+", x),
                     result.splitlines())
     if len(result) > 0:
-      trunk_revision = re.sub(r"^Committed r([0-9]+)", r"\1", result[0])
+      self["trunk_revision"] = re.sub(r"^Committed r([0-9]+)", r"\1",result[0])
 
     # Sometimes grepping for the revision fails. No idea why. If you figure
     # out why it is flaky, please do fix it properly.
-    if not trunk_revision:
+    if not self["trunk_revision"]:
       print("Sorry, grepping for the SVN revision failed. Please look for it "
             "in the last command's output above and provide it manually (just "
             "the number, without the leading \"r\").")
       self.DieNoManualMode("Can't prompt in forced mode.")
-      while not trunk_revision:
+      while not self["trunk_revision"]:
         print "> ",
-        trunk_revision = self.ReadLine()
-    self.Persist("trunk_revision", trunk_revision)
+        self["trunk_revision"] = self.ReadLine()
 
 
 class TagRevision(Step):
   MESSAGE = "Tag the new revision."
 
   def RunStep(self):
-    self.RestoreVersionIfUnset()
-    ver = "%s.%s.%s" % (self._state["major"],
-                        self._state["minor"],
-                        self._state["build"])
-    if self.Git("svn tag %s -m \"Tagging version %s\"" % (ver, ver)) is None:
-      self.Die("'git svn tag' failed.")
+    self.GitSVNTag(self["version"])
 
 
 class CheckChromium(Step):
   MESSAGE = "Ask for chromium checkout."
 
   def Run(self):
-    chrome_path = self._options.c
-    if not chrome_path:
+    self["chrome_path"] = self._options.chromium
+    if not self["chrome_path"]:
       self.DieNoManualMode("Please specify the path to a Chromium checkout in "
                           "forced mode.")
       print ("Do you have a \"NewGit\" Chromium checkout and want "
           "this script to automate creation of the roll CL? If yes, enter the "
           "path to (and including) the \"src\" directory here, otherwise just "
           "press <Return>: "),
-      chrome_path = self.ReadLine()
-    self.Persist("chrome_path", chrome_path)
+      self["chrome_path"] = self.ReadLine()
 
 
 class SwitchChromium(Step):
@@ -420,12 +431,11 @@ class SwitchChromium(Step):
   REQUIRES = "chrome_path"
 
   def RunStep(self):
-    v8_path = os.getcwd()
-    self.Persist("v8_path", v8_path)
-    os.chdir(self._state["chrome_path"])
+    self["v8_path"] = os.getcwd()
+    os.chdir(self["chrome_path"])
     self.InitialEnvironmentChecks()
     # Check for a clean workdir.
-    if self.Git("status -s -uno").strip() != "":
+    if not self.GitIsWorkdirClean():
       self.Die("Workspace is not clean. Please commit or undo your changes.")
     # Assert that the DEPS file is there.
     if not os.path.exists(self.Config(DEPS_FILE)):
@@ -437,16 +447,10 @@ class UpdateChromiumCheckout(Step):
   REQUIRES = "chrome_path"
 
   def RunStep(self):
-    os.chdir(self._state["chrome_path"])
-    if self.Git("checkout master") is None:
-      self.Die("'git checkout master' failed.")
-    if self.Git("pull") is None:
-      self.Die("'git pull' failed, please try again.")
-
-    self.RestoreIfUnset("trunk_revision")
-    args = "checkout -b v8-roll-%s" % self._state["trunk_revision"]
-    if self.Git(args) is None:
-      self.Die("Failed to checkout a new branch.")
+    os.chdir(self["chrome_path"])
+    self.GitCheckout("master")
+    self.GitPull()
+    self.GitCreateBranch("v8-roll-%s" % self["trunk_revision"])
 
 
 class UploadCL(Step):
@@ -454,33 +458,27 @@ class UploadCL(Step):
   REQUIRES = "chrome_path"
 
   def RunStep(self):
-    os.chdir(self._state["chrome_path"])
+    os.chdir(self["chrome_path"])
 
     # Patch DEPS file.
-    self.RestoreIfUnset("trunk_revision")
     deps = FileToText(self.Config(DEPS_FILE))
     deps = re.sub("(?<=\"v8_revision\": \")([0-9]+)(?=\")",
-                  self._state["trunk_revision"],
+                  self["trunk_revision"],
                   deps)
     TextToFile(deps, self.Config(DEPS_FILE))
 
-    self.RestoreVersionIfUnset()
-    ver = "%s.%s.%s" % (self._state["major"],
-                        self._state["minor"],
-                        self._state["build"])
-    if self._options.r:
-      print "Using account %s for review." % self._options.r
-      rev = self._options.r
+    if self._options.reviewer:
+      print "Using account %s for review." % self._options.reviewer
+      rev = self._options.reviewer
     else:
       print "Please enter the email address of a reviewer for the roll CL: ",
       self.DieNoManualMode("A reviewer must be specified in forced mode.")
       rev = self.ReadLine()
-    args = "commit -am \"Update V8 to version %s.\n\nTBR=%s\"" % (ver, rev)
-    if self.Git(args) is None:
-      self.Die("'git commit' failed.")
-    force_flag = " -f" if self._options.force_upload else ""
-    if self.Git("cl upload --send-mail%s" % force_flag, pipe=False) is None:
-      self.Die("'git cl upload' failed, please try again.")
+    suffix = PUSH_MESSAGE_SUFFIX % int(self["svn_revision"])
+    self.GitCommit("Update V8 to version %s%s.\n\nTBR=%s"
+                   % (self["version"], suffix, rev))
+    self.GitUpload(author=self._options.author,
+                   force=self._options.force_upload)
     print "CL uploaded."
 
 
@@ -489,34 +487,28 @@ class SwitchV8(Step):
   REQUIRES = "chrome_path"
 
   def RunStep(self):
-    self.RestoreIfUnset("v8_path")
-    os.chdir(self._state["v8_path"])
+    os.chdir(self["v8_path"])
 
 
 class CleanUp(Step):
   MESSAGE = "Done!"
 
   def RunStep(self):
-    self.RestoreVersionIfUnset()
-    ver = "%s.%s.%s" % (self._state["major"],
-                        self._state["minor"],
-                        self._state["build"])
-    self.RestoreIfUnset("trunk_revision")
-    self.RestoreIfUnset("chrome_path")
-
-    if self._state["chrome_path"]:
+    if self["chrome_path"]:
       print("Congratulations, you have successfully created the trunk "
             "revision %s and rolled it into Chromium. Please don't forget to "
-            "update the v8rel spreadsheet:" % ver)
+            "update the v8rel spreadsheet:" % self["version"])
     else:
       print("Congratulations, you have successfully created the trunk "
             "revision %s. Please don't forget to roll this new version into "
-            "Chromium, and to update the v8rel spreadsheet:" % ver)
-    print "%s\ttrunk\t%s" % (ver, self._state["trunk_revision"])
+            "Chromium, and to update the v8rel spreadsheet:"
+            % self["version"])
+    print "%s\ttrunk\t%s" % (self["version"],
+                             self["trunk_revision"])
 
     self.CommonCleanup()
-    if self.Config(TRUNKBRANCH) != self._state["current_branch"]:
-      self.Git("branch -D %s" % self.Config(TRUNKBRANCH))
+    if self.Config(TRUNKBRANCH) != self["current_branch"]:
+      self.GitDeleteBranch(self.Config(TRUNKBRANCH))
 
 
 def RunPushToTrunk(config,
@@ -553,46 +545,52 @@ def RunPushToTrunk(config,
 
 
 def BuildOptions():
-  result = optparse.OptionParser()
-  result.add_option("-c", "--chromium", dest="c",
-                    help=("Specify the path to your Chromium src/ "
-                          "directory to automate the V8 roll."))
-  result.add_option("-f", "--force", dest="f",
-                    help="Don't prompt the user.",
-                    default=False, action="store_true")
-  result.add_option("-l", "--last-push", dest="l",
-                    help=("Manually specify the git commit ID "
-                          "of the last push to trunk."))
-  result.add_option("-m", "--manual", dest="m",
-                    help="Prompt the user at every important step.",
-                    default=False, action="store_true")
-  result.add_option("-r", "--reviewer", dest="r",
-                    help=("Specify the account name to be used for reviews."))
-  result.add_option("-s", "--step", dest="s",
-                    help="Specify the step where to start work. Default: 0.",
-                    default=0, type="int")
-  return result
+  parser = argparse.ArgumentParser()
+  group = parser.add_mutually_exclusive_group()
+  group.add_argument("-f", "--force",
+                     help="Don't prompt the user.",
+                     default=False, action="store_true")
+  group.add_argument("-m", "--manual",
+                     help="Prompt the user at every important step.",
+                     default=False, action="store_true")
+  parser.add_argument("-a", "--author",
+                      help="The author email used for rietveld.")
+  parser.add_argument("-b", "--last-bleeding-edge",
+                      help=("The git commit ID of the last bleeding edge "
+                            "revision that was pushed to trunk. This is used "
+                            "for the auto-generated ChangeLog entry."))
+  parser.add_argument("-c", "--chromium",
+                      help=("The path to your Chromium src/ directory to "
+                            "automate the V8 roll."))
+  parser.add_argument("-l", "--last-push",
+                      help="The git commit ID of the last push to trunk.")
+  parser.add_argument("-r", "--reviewer",
+                      help="The account name to be used for reviews.")
+  parser.add_argument("-s", "--step",
+                      help="The step where to start work. Default: 0.",
+                      default=0, type=int)
+  return parser
 
 
 def ProcessOptions(options):
-  if options.s < 0:
-    print "Bad step number %d" % options.s
+  if options.step < 0:
+    print "Bad step number %d" % options.step
     return False
-  if not options.m and not options.r:
+  if not options.manual and not options.reviewer:
     print "A reviewer (-r) is required in (semi-)automatic mode."
     return False
-  if options.f and options.m:
-    print "Manual and forced mode cannot be combined."
-    return False
-  if not options.m and not options.c:
+  if not options.manual and not options.chromium:
     print "A chromium checkout (-c) is required in (semi-)automatic mode."
     return False
+  if not options.manual and not options.author:
+    print "Specify your chromium.org email with -a in (semi-)automatic mode."
+    return False
   return True
 
 
 def Main():
   parser = BuildOptions()
-  (options, args) = parser.parse_args()
+  options = parser.parse_args()
   if not ProcessOptions(options):
     parser.print_help()
     return 1