A64: Synchronize with r18084.

tools/push-to-trunk/push_to_trunk.py

 #!/usr/bin/env python
 # Copyright 2013 the V8 project authors. All rights reserved.
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 #       notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
 #       copyright notice, this list of conditions and the following
@@ skipping 12 matching lines @@
 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 import datetime
 import optparse
 import sys
 import tempfile
+import urllib2
 
 from common_includes import *
 
 TRUNKBRANCH = "TRUNKBRANCH"
 CHROMIUM = "CHROMIUM"
 DEPS_FILE = "DEPS_FILE"
 
 CONFIG = {
   BRANCHNAME: "prepare-push",
   TRUNKBRANCH: "trunk-push",
   PERSISTFILE_BASENAME: "/tmp/v8-push-to-trunk-tempfile",
   TEMP_BRANCH: "prepare-push-temporary-branch-created-by-script",
   DOT_GIT_LOCATION: ".git",
   VERSION_FILE: "src/version.cc",
   CHANGELOG_FILE: "ChangeLog",
   CHANGELOG_ENTRY_FILE: "/tmp/v8-push-to-trunk-tempfile-changelog-entry",
   PATCH_FILE: "/tmp/v8-push-to-trunk-tempfile-patch-file",
   COMMITMSG_FILE: "/tmp/v8-push-to-trunk-tempfile-commitmsg",
   DEPS_FILE: "DEPS",
 }
 
 
 class Preparation(Step):
-  def __init__(self):
-    Step.__init__(self, "Preparation.")
+  MESSAGE = "Preparation."
 
   def RunStep(self):
     self.InitialEnvironmentChecks()
     self.CommonPrepare()
     self.PrepareBranch()
     self.DeleteBranch(self.Config(TRUNKBRANCH))
 
 
 class FreshBranch(Step):
-  def __init__(self):
-    Step.__init__(self, "Create a fresh branch.")
+  MESSAGE = "Create a fresh branch."
 
   def RunStep(self):
     args = "checkout -b %s svn/bleeding_edge" % self.Config(BRANCHNAME)
     if self.Git(args) is None:
       self.Die("Creating branch %s failed." % self.Config(BRANCHNAME))
 
 
 class DetectLastPush(Step):
-  def __init__(self):
-    Step.__init__(self, "Detect commit ID of last push to trunk.")
+  MESSAGE = "Detect commit ID of last push to trunk."
 
   def RunStep(self):
     last_push = (self._options.l or
                  self.Git("log -1 --format=%H ChangeLog").strip())
     while True:
       # Print assumed commit, circumventing git's pager.
       print self.Git("log -1 %s" % last_push)
       if self.Confirm("Is the commit printed above the last push to trunk?"):
         break
       args = "log -1 --format=%H %s^ ChangeLog" % last_push
       last_push = self.Git(args).strip()
     self.Persist("last_push", last_push)
     self._state["last_push"] = last_push
 
 
 class PrepareChangeLog(Step):
-  def __init__(self):
-    Step.__init__(self, "Prepare raw ChangeLog entry.")
+  MESSAGE = "Prepare raw ChangeLog entry."
+
+  def Reload(self, body):
+    """Attempts to reload the commit message from rietveld in order to allow
+    late changes to the LOG flag. Note: This is brittle to future changes of
+    the web page name or structure.
+    """
+    match = re.search(r"^Review URL: https://codereview\.chromium\.org/(\d+)$",
+                      body, flags=re.M)
+    if match:
+      cl_url = "https://codereview.chromium.org/%s/description" % match.group(1)
+      try:
+        body = self.ReadURL(cl_url)
+      except urllib2.URLError:
+        pass
+    return body
 
   def RunStep(self):
     self.RestoreIfUnset("last_push")
 
     # These version numbers are used again later for the trunk commit.
     self.ReadAndPersistVersion()
 
     date = datetime.date.today().strftime("%Y-%m-%d")
     self.Persist("date", date)
     output = "%s: Version %s.%s.%s\n\n" % (date,
                                            self._state["major"],
                                            self._state["minor"],
                                            self._state["build"])
     TextToFile(output, self.Config(CHANGELOG_ENTRY_FILE))
 
     args = "log %s..HEAD --format=%%H" % self._state["last_push"]
     commits = self.Git(args).strip()
 
-    def GetCommitMessages():
-      for commit in commits.splitlines():
-        yield [
-          self.Git("log -1 %s --format=\"%%w(80,8,8)%%s\"" % commit),
-          self.Git("log -1 %s --format=\"%%B\"" % commit),
-          self.Git("log -1 %s --format=\"%%w(80,8,8)(%%an)\"" % commit),
-        ]
+    # Cache raw commit messages.
+    commit_messages = [
+      [
+        self.Git("log -1 %s --format=\"%%s\"" % commit),
+        self.Reload(self.Git("log -1 %s --format=\"%%B\"" % commit)),
+        self.Git("log -1 %s --format=\"%%an\"" % commit),
+      ] for commit in commits.splitlines()
+    ]
 
-    body = MakeChangeLogBody(GetCommitMessages)
+    # Auto-format commit messages.
+    body = MakeChangeLogBody(commit_messages, auto_format=True)
     AppendToFile(body, self.Config(CHANGELOG_ENTRY_FILE))
 
-    msg = "        Performance and stability improvements on all platforms.\n"
+    msg = ("        Performance and stability improvements on all platforms."
+           "\n#\n# The change log above is auto-generated. Please review if "
+           "all relevant\n# commit messages from the list below are included."
+           "\n# All lines starting with # will be stripped.\n#\n")
     AppendToFile(msg, self.Config(CHANGELOG_ENTRY_FILE))
 
+    # Include unformatted commit messages as a reference in a comment.
+    comment_body = MakeComment(MakeChangeLogBody(commit_messages))
+    AppendToFile(comment_body, self.Config(CHANGELOG_ENTRY_FILE))
+
 
 class EditChangeLog(Step):
-  def __init__(self):
-    Step.__init__(self, "Edit ChangeLog entry.")
+  MESSAGE = "Edit ChangeLog entry."
 
   def RunStep(self):
     print ("Please press <Return> to have your EDITOR open the ChangeLog "
            "entry, then edit its contents to your liking. When you're done, "
            "save the file and exit your EDITOR. ")
     self.ReadLine(default="")
 
     # TODO(machenbach): Don't use EDITOR in forced mode as soon as script is
     # well tested.
     self.Editor(self.Config(CHANGELOG_ENTRY_FILE))
     handle, new_changelog = tempfile.mkstemp()
     os.close(handle)
 
-    # (1) Eliminate tabs, (2) fix too little and (3) too much indentation, and
-    # (4) eliminate trailing whitespace.
+    # Strip comments and reformat with correct indentation.
     changelog_entry = FileToText(self.Config(CHANGELOG_ENTRY_FILE)).rstrip()
-    changelog_entry = MSub(r"\t", r"        ", changelog_entry)
-    changelog_entry = MSub(r"^ {1,7}([^ ])", r"        \1", changelog_entry)
-    changelog_entry = MSub(r"^ {9,80}([^ ])", r"        \1", changelog_entry)
-    changelog_entry = MSub(r" +$", r"", changelog_entry)
+    changelog_entry = StripComments(changelog_entry)
+    changelog_entry = "\n".join(map(Fill80, changelog_entry.splitlines()))
+    changelog_entry = changelog_entry.lstrip()
 
     if changelog_entry == "":
       self.Die("Empty ChangeLog entry.")
 
     with open(new_changelog, "w") as f:
       f.write(changelog_entry)
       f.write("\n\n\n")  # Explicitly insert two empty lines.
 
     AppendToFile(FileToText(self.Config(CHANGELOG_FILE)), new_changelog)
     TextToFile(FileToText(new_changelog), self.Config(CHANGELOG_FILE))
     os.remove(new_changelog)
 
 
 class IncrementVersion(Step):
-  def __init__(self):
-    Step.__init__(self, "Increment version number.")
+  MESSAGE = "Increment version number."
 
   def RunStep(self):
     self.RestoreIfUnset("build")
     new_build = str(int(self._state["build"]) + 1)
 
     if self.Confirm(("Automatically increment BUILD_NUMBER? (Saying 'n' will "
                      "fire up your EDITOR on %s so you can make arbitrary "
                      "changes. When you're done, save the file and exit your "
                      "EDITOR.)" % self.Config(VERSION_FILE))):
       text = FileToText(self.Config(VERSION_FILE))
       text = MSub(r"(?<=#define BUILD_NUMBER)(?P<space>\s+)\d*$",
                   r"\g<space>%s" % new_build,
                   text)
       TextToFile(text, self.Config(VERSION_FILE))
     else:
       self.Editor(self.Config(VERSION_FILE))
 
     self.ReadAndPersistVersion("new_")
 
 
 class CommitLocal(Step):
-  def __init__(self):
-    Step.__init__(self, "Commit to local branch.")
+  MESSAGE = "Commit to local branch."
 
   def RunStep(self):
     self.RestoreVersionIfUnset("new_")
     prep_commit_msg = ("Prepare push to trunk.  "
         "Now working on version %s.%s.%s." % (self._state["new_major"],
                                               self._state["new_minor"],
                                               self._state["new_build"]))
     self.Persist("prep_commit_msg", prep_commit_msg)
     if self.Git("commit -a -m \"%s\"" % prep_commit_msg) is None:
       self.Die("'git commit -a' failed.")
 
 
 class CommitRepository(Step):
-  def __init__(self):
-    Step.__init__(self, "Commit to the repository.")
+  MESSAGE = "Commit to the repository."
 
   def RunStep(self):
     self.WaitForLGTM()
     # Re-read the ChangeLog entry (to pick up possible changes).
     # FIXME(machenbach): This was hanging once with a broken pipe.
     TextToFile(GetLastChangeLogEntries(self.Config(CHANGELOG_FILE)),
                self.Config(CHANGELOG_ENTRY_FILE))
 
     if self.Git("cl dcommit -f", "PRESUBMIT_TREE_CHECK=\"skip\"") is None:
       self.Die("'git cl dcommit' failed, please try again.")
 
 
 class StragglerCommits(Step):
-  def __init__(self):
-    Step.__init__(self, "Fetch straggler commits that sneaked in since this "
-                        "script was started.")
+  MESSAGE = ("Fetch straggler commits that sneaked in since this script was "
+             "started.")
 
   def RunStep(self):
     if self.Git("svn fetch") is None:
       self.Die("'git svn fetch' failed.")
     self.Git("checkout svn/bleeding_edge")
     self.RestoreIfUnset("prep_commit_msg")
     args = "log -1 --format=%%H --grep=\"%s\"" % self._state["prep_commit_msg"]
     prepare_commit_hash = self.Git(args).strip()
     self.Persist("prepare_commit_hash", prepare_commit_hash)
 
 
 class SquashCommits(Step):
-  def __init__(self):
-    Step.__init__(self, "Squash commits into one.")
+  MESSAGE = "Squash commits into one."
 
   def RunStep(self):
     # Instead of relying on "git rebase -i", we'll just create a diff, because
     # that's easier to automate.
     self.RestoreIfUnset("prepare_commit_hash")
     args = "diff svn/trunk %s" % self._state["prepare_commit_hash"]
     TextToFile(self.Git(args), self.Config(PATCH_FILE))
 
     # Convert the ChangeLog entry to commit message format:
     # - remove date
@@ skipping 21 matching lines @@
             }\
           }'" % (changelog_entry, self._state["date"]))
 
     if not text:
       self.Die("Commit message editing failed.")
     TextToFile(text, self.Config(COMMITMSG_FILE))
     os.remove(self.Config(CHANGELOG_ENTRY_FILE))
 
 
 class NewBranch(Step):
-  def __init__(self):
-    Step.__init__(self, "Create a new branch from trunk.")
+  MESSAGE = "Create a new branch from trunk."
 
   def RunStep(self):
     if self.Git("checkout -b %s svn/trunk" % self.Config(TRUNKBRANCH)) is None:
       self.Die("Checking out a new branch '%s' failed." %
                self.Config(TRUNKBRANCH))
 
 
 class ApplyChanges(Step):
-  def __init__(self):
-    Step.__init__(self, "Apply squashed changes.")
+  MESSAGE = "Apply squashed changes."
 
   def RunStep(self):
     self.ApplyPatch(self.Config(PATCH_FILE))
     Command("rm", "-f %s*" % self.Config(PATCH_FILE))
 
 
 class SetVersion(Step):
-  def __init__(self):
-    Step.__init__(self, "Set correct version for trunk.")
+  MESSAGE = "Set correct version for trunk."
 
   def RunStep(self):
     self.RestoreVersionIfUnset()
     output = ""
     for line in FileToText(self.Config(VERSION_FILE)).splitlines():
       if line.startswith("#define MAJOR_VERSION"):
         line = re.sub("\d+$", self._state["major"], line)
       elif line.startswith("#define MINOR_VERSION"):
         line = re.sub("\d+$", self._state["minor"], line)
       elif line.startswith("#define BUILD_NUMBER"):
         line = re.sub("\d+$", self._state["build"], line)
       elif line.startswith("#define PATCH_LEVEL"):
         line = re.sub("\d+$", "0", line)
       elif line.startswith("#define IS_CANDIDATE_VERSION"):
         line = re.sub("\d+$", "0", line)
       output += "%s\n" % line
     TextToFile(output, self.Config(VERSION_FILE))
 
 
 class CommitTrunk(Step):
-  def __init__(self):
-    Step.__init__(self, "Commit to local trunk branch.")
+  MESSAGE = "Commit to local trunk branch."
 
   def RunStep(self):
     self.Git("add \"%s\"" % self.Config(VERSION_FILE))
     if self.Git("commit -F \"%s\"" % self.Config(COMMITMSG_FILE)) is None:
       self.Die("'git commit' failed.")
     Command("rm", "-f %s*" % self.Config(COMMITMSG_FILE))
 
 
 class SanityCheck(Step):
-  def __init__(self):
-    Step.__init__(self, "Sanity check.")
+  MESSAGE = "Sanity check."
 
   def RunStep(self):
     if not self.Confirm("Please check if your local checkout is sane: Inspect "
         "%s, compile, run tests. Do you want to commit this new trunk "
         "revision to the repository?" % self.Config(VERSION_FILE)):
       self.Die("Execution canceled.")
 
 
 class CommitSVN(Step):
-  def __init__(self):
-    Step.__init__(self, "Commit to SVN.")
+  MESSAGE = "Commit to SVN."
 
   def RunStep(self):
     result = self.Git("svn dcommit 2>&1")
     if not result:
       self.Die("'git svn dcommit' failed.")
     result = filter(lambda x: re.search(r"^Committed r[0-9]+", x),
                     result.splitlines())
     if len(result) > 0:
       trunk_revision = re.sub(r"^Committed r([0-9]+)", r"\1", result[0])
 
     # Sometimes grepping for the revision fails. No idea why. If you figure
     # out why it is flaky, please do fix it properly.
     if not trunk_revision:
       print("Sorry, grepping for the SVN revision failed. Please look for it "
             "in the last command's output above and provide it manually (just "
             "the number, without the leading \"r\").")
       self.DieInForcedMode("Can't prompt in forced mode.")
       while not trunk_revision:
         print "> ",
         trunk_revision = self.ReadLine()
     self.Persist("trunk_revision", trunk_revision)
 
 
 class TagRevision(Step):
-  def __init__(self):
-    Step.__init__(self, "Tag the new revision.")
+  MESSAGE = "Tag the new revision."
 
   def RunStep(self):
     self.RestoreVersionIfUnset()
     ver = "%s.%s.%s" % (self._state["major"],
                         self._state["minor"],
                         self._state["build"])
     if self.Git("svn tag %s -m \"Tagging version %s\"" % (ver, ver)) is None:
       self.Die("'git svn tag' failed.")
 
 
 class CheckChromium(Step):
-  def __init__(self):
-    Step.__init__(self, "Ask for chromium checkout.")
+  MESSAGE = "Ask for chromium checkout."
 
   def Run(self):
     chrome_path = self._options.c
     if not chrome_path:
       self.DieInForcedMode("Please specify the path to a Chromium checkout in "
                           "forced mode.")
       print ("Do you have a \"NewGit\" Chromium checkout and want "
           "this script to automate creation of the roll CL? If yes, enter the "
           "path to (and including) the \"src\" directory here, otherwise just "
           "press <Return>: "),
       chrome_path = self.ReadLine()
     self.Persist("chrome_path", chrome_path)
 
 
 class SwitchChromium(Step):
-  def __init__(self):
-    Step.__init__(self, "Switch to Chromium checkout.", requires="chrome_path")
+  MESSAGE = "Switch to Chromium checkout."
+  REQUIRES = "chrome_path"
 
   def RunStep(self):
     v8_path = os.getcwd()
     self.Persist("v8_path", v8_path)
     os.chdir(self._state["chrome_path"])
     self.InitialEnvironmentChecks()
     # Check for a clean workdir.
     if self.Git("status -s -uno").strip() != "":
       self.Die("Workspace is not clean. Please commit or undo your changes.")
     # Assert that the DEPS file is there.
     if not os.path.exists(self.Config(DEPS_FILE)):
       self.Die("DEPS file not present.")
 
 
 class UpdateChromiumCheckout(Step):
-  def __init__(self):
-    Step.__init__(self, "Update the checkout and create a new branch.",
-                  requires="chrome_path")
+  MESSAGE = "Update the checkout and create a new branch."
+  REQUIRES = "chrome_path"
 
   def RunStep(self):
     os.chdir(self._state["chrome_path"])
     if self.Git("checkout master") is None:
       self.Die("'git checkout master' failed.")
     if self.Git("pull") is None:
       self.Die("'git pull' failed, please try again.")
 
     self.RestoreIfUnset("trunk_revision")
     args = "checkout -b v8-roll-%s" % self._state["trunk_revision"]
     if self.Git(args) is None:
       self.Die("Failed to checkout a new branch.")
 
 
 class UploadCL(Step):
-  def __init__(self):
-    Step.__init__(self, "Create and upload CL.", requires="chrome_path")
+  MESSAGE = "Create and upload CL."
+  REQUIRES = "chrome_path"
 
   def RunStep(self):
     os.chdir(self._state["chrome_path"])
 
     # Patch DEPS file.
     self.RestoreIfUnset("trunk_revision")
     deps = FileToText(self.Config(DEPS_FILE))
     deps = re.sub("(?<=\"v8_revision\": \")([0-9]+)(?=\")",
                   self._state["trunk_revision"],
                   deps)
     TextToFile(deps, self.Config(DEPS_FILE))
 
     self.RestoreVersionIfUnset()
     ver = "%s.%s.%s" % (self._state["major"],
                         self._state["minor"],
                         self._state["build"])
     if self._options and self._options.r:
       print "Using account %s for review." % self._options.r
       rev = self._options.r
     else:
       print "Please enter the email address of a reviewer for the roll CL: ",
       self.DieInForcedMode("A reviewer must be specified in forced mode.")
       rev = self.ReadLine()
     args = "commit -am \"Update V8 to version %s.\n\nTBR=%s\"" % (ver, rev)
     if self.Git(args) is None:
       self.Die("'git commit' failed.")
-    if self.Git("cl upload --send-mail", pipe=False) is None:
+    force_flag = " -f" if self._options.f else ""
+    if self.Git("cl upload --send-mail%s" % force_flag, pipe=False) is None:
       self.Die("'git cl upload' failed, please try again.")
     print "CL uploaded."
 
 
 class SwitchV8(Step):
-  def __init__(self):
-    Step.__init__(self, "Returning to V8 checkout.", requires="chrome_path")
+  MESSAGE = "Returning to V8 checkout."
+  REQUIRES = "chrome_path"
 
   def RunStep(self):
     self.RestoreIfUnset("v8_path")
     os.chdir(self._state["v8_path"])
 
 
 class CleanUp(Step):
-  def __init__(self):
-    Step.__init__(self, "Done!")
+  MESSAGE = "Done!"
 
   def RunStep(self):
     self.RestoreVersionIfUnset()
     ver = "%s.%s.%s" % (self._state["major"],
                         self._state["minor"],
                         self._state["build"])
     self.RestoreIfUnset("trunk_revision")
     self.RestoreIfUnset("chrome_path")
 
     if self._state["chrome_path"]:
@@ skipping 60 matching lines @@
   result.add_option("-s", "--step", dest="s",
                     help="Specify the step where to start work. Default: 0.",
                     default=0, type="int")
   return result
 
 
 def ProcessOptions(options):
   if options.s < 0:
     print "Bad step number %d" % options.s
     return False
+  if options.f and not options.r:
+    print "A reviewer (-r) is required in forced mode."
+    return False
+  if options.f and not options.c:
+    print "A chromium checkout (-c) is required in forced mode."
+    return False
   return True
 
 
 def Main():
   parser = BuildOptions()
   (options, args) = parser.parse_args()
   if not ProcessOptions(options):
     parser.print_help()
     return 1
   RunPushToTrunk(CONFIG, options)
 
 if __name__ == "__main__":
   sys.exit(Main())