Update upload.py @ fae51921ad9d

third_party/upload.py

 #!/usr/bin/env python
+# coding: utf-8
 #
 # Copyright 2007 Google Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #     http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
@@ skipping 196 matching lines @@
     """Returns an OpenerDirector for making HTTP requests.
 
     Returns:
       A urllib2.OpenerDirector object.
     """
     raise NotImplementedError()
 
   def _CreateRequest(self, url, data=None):
     """Creates a new urllib request."""
     logging.debug("Creating request for: '%s' with payload:\n%s", url, data)
-    req = urllib2.Request(url, data=data)
+    req = urllib2.Request(url, data=data, headers={"Accept": "text/plain"})
     if self.host_override:
       req.add_header("Host", self.host_override)
     for key, value in self.extra_headers.iteritems():
       req.add_header(key, value)
     return req
 
   def _GetAuthToken(self, email, password):
     """Uses ClientLogin to authenticate the user, returning an auth token.
 
     Args:
@@ skipping 163 matching lines @@
         try:
           f = self.opener.open(req)
           response = f.read()
           f.close()
           return response
         except urllib2.HTTPError, e:
           if tries > 3:
             raise
           elif e.code == 401 or e.code == 302:
             self._Authenticate()
-##           elif e.code >= 500 and e.code < 600:
-##             # Server Error - try again.
-##             continue
           elif e.code == 301:
             # Handle permanent redirect manually.
             url = e.info()["location"]
             url_loc = urlparse.urlparse(url)
             self.host = '%s://%s' % (url_loc[0], url_loc[1])
+          elif e.code >= 500:
+            ErrorExit(e.read())
           else:
             raise
     finally:
       socket.setdefaulttimeout(old_timeout)
 
 
 class HttpRpcServer(AbstractRpcServer):
   """Provides a simplified RPC-style interface for HTTP requests."""
 
   def _Authenticate(self):
@@ skipping 105 matching lines @@
                  dest="save_cookies", default=True,
                  help="Do not save authentication cookies to local disk.")
 group.add_option("--account_type", action="store", dest="account_type",
                  metavar="TYPE", default=AUTH_ACCOUNT_TYPE,
                  choices=["GOOGLE", "HOSTED"],
                  help=("Override the default account type "
                        "(defaults to '%default', "
                        "valid choices are 'GOOGLE' and 'HOSTED')."))
 # Issue
 group = parser.add_option_group("Issue options")
-group.add_option("-d", "--description", action="store", dest="description",
-                 metavar="DESCRIPTION", default=None,
-                 help="Optional description when creating an issue.")
-group.add_option("-f", "--description_file", action="store",
-                 dest="description_file", metavar="DESCRIPTION_FILE",
+group.add_option("-t", "--title", action="store", dest="title",
+                 help="New issue subject or new patch set title")
+group.add_option("-m", "--message", action="store", dest="message",
                  default=None,
-                 help="Optional path of a file that contains "
-                      "the description when creating an issue.")
+                 help="New issue description or new patch set message")
+group.add_option("-F", "--file", action="store", dest="file",
+                 default=None, help="Read the message above from file.")
 group.add_option("-r", "--reviewers", action="store", dest="reviewers",
                  metavar="REVIEWERS", default=None,
                  help="Add reviewers (comma separated email addresses).")
 group.add_option("--cc", action="store", dest="cc",
                  metavar="CC", default=None,
                  help="Add CC (comma separated email addresses).")
 group.add_option("--private", action="store_true", dest="private",
                  default=False,
                  help="Make the issue restricted to reviewers and those CCed")
 # Upload options
 group = parser.add_option_group("Patch options")
-group.add_option("-m", "--message", action="store", dest="message",
-                 metavar="MESSAGE", default=None,
-                 help="A message to identify the patch. "
-                      "Will prompt if omitted.")
 group.add_option("-i", "--issue", type="int", action="store",
                  metavar="ISSUE", default=None,
                  help="Issue number to which to add. Defaults to new issue.")
 group.add_option("--base_url", action="store", dest="base_url", default=None,
                  help="Base URL path for files (listed as \"Base URL\" when "
                  "viewing issue).  If omitted, will be guessed automatically "
                  "for SVN repos and left blank for others.")
 group.add_option("--download_base", action="store_true",
                  dest="download_base", default=False,
                  help="Base files will be downloaded by the server "
@@ skipping 210 matching lines @@
 class VersionControlSystem(object):
   """Abstract base class providing an interface to the VCS."""
 
   def __init__(self, options):
     """Constructor.
 
     Args:
       options: Command line options.
     """
     self.options = options
-    
+
   def GetGUID(self):
     """Return string to distinguish the repository from others, for example to
     query all opened review issues for it"""
     raise NotImplementedError(
         "abstract method -- subclass %s must override" % self.__class__)
 
   def PostProcessDiff(self, diff):
     """Return the diff with any special post processing this VCS needs, e.g.
     to include an svn-style "Index:"."""
     return diff
@@ skipping 139 matching lines @@
       self.rev_end = match.group(3)
     else:
       self.rev_start = self.rev_end = None
     # Cache output from "svn list -r REVNO dirname".
     # Keys: dirname, Values: 2-tuple (ouput for start rev and end rev).
     self.svnls_cache = {}
     # Base URL is required to fetch files deleted in an older revision.
     # Result is cached to not guess it over and over again in GetBaseFile().
     required = self.options.download_base or self.options.revision is not None
     self.svn_base = self._GuessBase(required)
-    
+
   def GetGUID(self):
     return self._GetInfo("Repository UUID")
 
   def GuessBase(self, required):
     """Wrapper for _GuessBase."""
     return self.svn_base
 
   def _GuessBase(self, required):
     """Returns base URL for current diff.
 
@@ skipping 14 matching lines @@
           scheme = "http"
           guess = "Google Code "
         path = path + "/"
         base = urlparse.urlunparse((scheme, netloc, path, params,
                                     query, fragment))
         logging.info("Guessed %sbase = %s", guess, base)
         return base
     if required:
       ErrorExit("Can't find URL in output from svn info")
     return None
-    
+
   def _GetInfo(self, key):
     """Parses 'svn info' for current dir. Returns value for key or None"""
     for line in RunShell(["svn", "info"]).splitlines():
       if line.startswith(key + ": "):
         return line.split(":", 1)[1].strip()
 
   def _EscapeFilename(self, filename):
     """Escapes filename for SVN commands."""
     if "@" in filename and not filename.endswith("@"):
       filename = "%s@" % filename
@@ skipping 222 matching lines @@
 class GitVCS(VersionControlSystem):
   """Implementation of the VersionControlSystem interface for Git."""
 
   def __init__(self, options):
     super(GitVCS, self).__init__(options)
     # Map of filename -> (hash before, hash after) of base file.
     # Hashes for "no such file" are represented as None.
     self.hashes = {}
     # Map of new filename -> old filename for renames.
     self.renames = {}
-    
+
   def GetGUID(self):
     revlist = RunShell("git rev-list --parents HEAD".split()).splitlines()
     # M-A: Return the 1st root hash, there could be multiple when a
     # subtree is merged. In that case, more analysis would need to
     # be done to figure out which HEAD is the 'most representative'.
     for r in revlist:
       if ' ' not in r:
         return r
 
   def PostProcessDiff(self, gitdiff):
@@ skipping 66 matching lines @@
     # this by overriding the environment (but there is still a problem if the
     # git config key "diff.external" is used).
     env = os.environ.copy()
     if 'GIT_EXTERNAL_DIFF' in env: del env['GIT_EXTERNAL_DIFF']
     # -M/-C will not print the diff for the deleted file when a file is renamed.
     # This is confusing because the original file will not be shown on the
     # review when a file is renamed. So first get the diff of all deleted files,
     # then the diff of everything except deleted files with rename and copy
     # support enabled.
     cmd = [
-        "git", "diff", "--no-ext-diff", "--full-index", "--ignore-submodules"
+        "git", "diff", "--no-color", "--no-ext-diff", "--full-index", "--ignore-submodules"
     ]
     diff = RunShell(cmd + ["--diff-filter=D"] + extra_args, env=env,
         silent_ok=True)
     diff += RunShell(cmd + ["-C", "--diff-filter=ACMRT"] + extra_args, env=env,
         silent_ok=True)
     if not diff:
       ErrorExit("No output from %s" % (cmd + extra_args))
     return diff
 
   def GetUnknownFiles(self):
@@ skipping 133 matching lines @@
 
   def GetGUID(self):
     # See chapter "Uniquely identifying a repository"
     # http://hgbook.red-bean.com/read/customizing-the-output-of-mercurial.html
     info = RunShell("hg log -r0 --template {node}".split())
     return info.strip()
 
   def _GetRelPath(self, filename):
     """Get relative path of a file according to the current directory,
     given its logical path in the repo."""
-    assert filename.startswith(self.subdir), (filename, self.subdir)
-    return filename[len(self.subdir):].lstrip(r"\/")
+    absname = os.path.join(self.repo_dir, filename)
+    return os.path.relpath(absname)
 
   def GenerateDiff(self, extra_args):
-    cmd = ["hg", "diff", "--git", "-r", self.base_rev] + extra_args
+    cmd = [
+        "hg", "diff", "--color", "never", "--git", "-r", self.base_rev
+        ] + extra_args
     data = RunShell(cmd, silent_ok=True)
     svndiff = []
     filecount = 0
     for line in data.splitlines():
       m = re.match("diff --git a/(\S+) b/(\S+)", line)
       if m:
         # Modify line to make it look like as it comes from svn diff.
         # With this modification no changes on the server side are required
         # to make upload.py work with Mercurial repos.
         # NOTE: for proper handling of moved/copied files, we have to use
         # the second filename.
         filename = m.group(2)
         svndiff.append("Index: %s" % filename)
         svndiff.append("=" * 67)
         filecount += 1
         logging.info(line)
       else:
         svndiff.append(line)
     if not filecount:
       ErrorExit("No valid patches found in output from hg diff")
     return "\n".join(svndiff) + "\n"
 
   def GetUnknownFiles(self):
     """Return a list of files unknown to the VCS."""
     args = []
-    status = RunShell(["hg", "status", "--rev", self.base_rev, "-u", "."],
+    status = RunShell(
+        ["hg", "status", "--color", "never", "--rev", self.base_rev, "-u", "."],
         silent_ok=True)
     unknown_files = []
     for line in status.splitlines():
       st, fn = line.split(" ", 1)
       if st == "?":
         unknown_files.append(fn)
     return unknown_files
 
   def GetBaseFile(self, filename):
-    # "hg status" and "hg cat" both take a path relative to the current subdir
-    # rather than to the repo root, but "hg diff" has given us the full path
-    # to the repo root.
+    # "hg status" and "hg cat" both take a path relative to the current subdir,
+    # but "hg diff" has given us the path relative to the repo root.
     base_content = ""
     new_content = None
     is_binary = False
     oldrelpath = relpath = self._GetRelPath(filename)
     # "hg status -C" returns two lines for moved/copied files, one otherwise
-    out = RunShell(["hg", "status", "-C", "--rev", self.base_rev, relpath])
+    out = RunShell(
+        [ "hg", "status", "--color", "never", "-C", "--rev", self.base_rev,
+          relpath])
     out = out.splitlines()
     # HACK: strip error message about missing file/directory if it isn't in
     # the working copy
     if out[0].startswith('%s: ' % relpath):
       out = out[1:]
     status, _ = out[0].split(' ', 1)
     if len(out) > 1 and status == "A":
       # Moved/copied => considered as modified, use old filename to
       # retrieve base contents
       oldrelpath = out[1].strip()
@@ skipping 42 matching lines @@
       ErrorExit("A changelist id is required")
     if (options.revision):
       ErrorExit("--rev is not supported for perforce")
 
     self.p4_port = options.p4_port
     self.p4_client = options.p4_client
     self.p4_user = options.p4_user
 
     ConfirmLogin()
 
-    if not options.message:
+    if not options.title:
       description = self.RunPerforceCommand(["describe", self.p4_changelist],
                                             marshal_output=True)
       if description and "desc" in description:
         # Rietveld doesn't support multi-line descriptions
-        raw_message = description["desc"].strip()
-        lines = raw_message.splitlines()
+        raw_title = description["desc"].strip()
+        lines = raw_title.splitlines()
         if len(lines):
-          options.message = lines[0]
+          options.title = lines[0]
 
   def GetGUID(self):
     """For now we don't know how to get repository ID for Perforce"""
     return
 
   def RunPerforceCommandWithReturnCode(self, extra_args, marshal_output=False,
                                        universal_newlines=True):
     args = ["p4"]
     if marshal_output:
       # -G makes perforce format its output as marshalled python objects
@@ skipping 370 matching lines @@
       if errcode != errno.ENOENT:  # command not found code
         raise
 
   # Mercurial has a command to get the base directory of a repository
   # Try running it, but don't die if we don't have hg installed.
   # NOTE: we try Mercurial first as it can sit on top of an SVN working copy.
   res = RunDetectCommand(VCS_MERCURIAL, ["hg", "root"])
   if res != None:
     return res
 
-  # Subversion has a .svn in all working directories.
-  if os.path.isdir('.svn'):
-    logging.info("Guessed VCS = Subversion")
-    return (VCS_SUBVERSION, None)
+  # Subversion from 1.7 has a single centralized .svn folder
+  # ( see http://subversion.apache.org/docs/release-notes/1.7.html#wc-ng )
+  # That's why we use 'svn info' instead of checking for .svn dir
+  res = RunDetectCommand(VCS_SUBVERSION, ["svn", "info"])
+  if res != None:
+    return res
 
   # Git has a command to test if you're in a git tree.
   # Try running it, but don't die if we don't have git installed.
   res = RunDetectCommand(VCS_GIT, ["git", "rev-parse",
                                    "--is-inside-work-tree"])
   if res != None:
     return res
 
   # detect CVS repos use `cvs status && $? == 0` rules
   res = RunDetectCommand(VCS_CVS, ["cvs", "status"])
@@ skipping 221 matching lines @@
   if data is None:
     data = vcs.GenerateDiff(args)
   data = vcs.PostProcessDiff(data)
   if options.print_diffs:
     print "Rietveld diff start:*****"
     print data
     print "Rietveld diff end:*****"
   files = vcs.GetBaseFiles(data)
   if verbosity >= 1:
     print "Upload server:", options.server, "(change with -s/--server)"
-  if options.issue:
-    prompt = "Message describing this patch set: "
-  else:
-    prompt = "New issue subject: "
-  message = options.message or raw_input(prompt).strip()
-  if not message:
-    ErrorExit("A non-empty message is required")
   rpc_server = GetRpcServer(options.server,
                             options.email,
                             options.host,
                             options.save_cookies,
                             options.account_type)
-  form_fields = [("subject", message)]
-  
+  form_fields = []
+
   repo_guid = vcs.GetGUID()
   if repo_guid:
     form_fields.append(("repo_guid", repo_guid))
   if base:
     b = urlparse.urlparse(base)
     username, netloc = urllib.splituser(b.netloc)
     if username:
       logging.info("Removed username from base URL")
       base = urlparse.urlunparse((b.scheme, netloc, b.path, b.params,
                                   b.query, b.fragment))
     form_fields.append(("base", base))
   if options.issue:
     form_fields.append(("issue", str(options.issue)))
   if options.email:
     form_fields.append(("user", options.email))
   if options.reviewers:
     for reviewer in options.reviewers.split(','):
       CheckReviewer(reviewer)
     form_fields.append(("reviewers", options.reviewers))
   if options.cc:
     for cc in options.cc.split(','):
       CheckReviewer(cc)
     form_fields.append(("cc", options.cc))
-  description = options.description
-  if options.description_file:
-    if options.description:
-      ErrorExit("Can't specify description and description_file")
-    file = open(options.description_file, 'r')
-    description = file.read()
+
+  # Process --message, --title and --file.
+  message = options.message or ""
+  title = options.title or ""
+  if options.file:
+    if options.message:
+      ErrorExit("Can't specify both message and message file options")
+    file = open(options.file, 'r')
+    message = file.read()
     file.close()
-  if description:
-    form_fields.append(("description", description))
+  if options.issue:
+    prompt = "Title describing this patch set: "
+  else:
+    prompt = "New issue subject: "
+  title = (
+      title or message.split('\n', 1)[0].strip() or raw_input(prompt).strip())
+  if not title and not options.issue:
+    ErrorExit("A non-empty title is required for a new issue")
+  # For existing issues, it's fine to give a patchset an empty name. Rietveld
+  # doesn't accept that so use a whitespace.
+  title = title or " "
+  if len(title) > 100:
+    title = title[:99] + '…'
+  if title and not options.issue:
+    message = message or title
+
+  form_fields.append(("subject", title))
+  if message:
+    if not options.issue:
+      form_fields.append(("description", message))
+    else:
+      # TODO: [ ] Use /<issue>/publish to add a comment.
+      pass
+
   # Send a hash of all the base file so the server can determine if a copy
   # already exists in an earlier patchset.
   base_hashes = ""
   for file, info in files.iteritems():
     if not info[0] is None:
       checksum = md5(info[0]).hexdigest()
       if base_hashes:
         base_hashes += "|"
       base_hashes += checksum + ":" + file
   form_fields.append(("base_hashes", base_hashes))
   if options.private:
     if options.issue:
       print "Warning: Private flag ignored when updating an existing issue."
     else:
       form_fields.append(("private", "1"))
   if options.send_patch:
     options.send_mail = True
-  # If we're uploading base files, don't send the email before the uploads, so
-  # that it contains the file status.
-  if options.send_mail and options.download_base:
-    form_fields.append(("send_mail", "1"))
   if not options.download_base:
     form_fields.append(("content_upload", "1"))
   if len(data) > MAX_UPLOAD_SIZE:
     print "Patch is large, so uploading file patches separately."
     uploaded_diff_file = []
     form_fields.append(("separate_patches", "1"))
   else:
     uploaded_diff_file = [("data", "data.diff", data)]
   ctype, body = EncodeMultipartFormData(form_fields, uploaded_diff_file)
   response_body = rpc_server.Send("/upload", body, content_type=ctype)
@@ skipping 14 matching lines @@
     sys.exit(0)
   issue = msg[msg.rfind("/")+1:]
 
   if not uploaded_diff_file:
     result = UploadSeparatePatches(issue, rpc_server, patchset, data, options)
     if not options.download_base:
       patches = result
 
   if not options.download_base:
     vcs.UploadBaseFiles(issue, rpc_server, patches, patchset, options, files)
-    if options.send_mail:
-      payload = ""
-      if options.send_patch:
-        payload=urllib.urlencode({"attach_patch": "yes"})
-      rpc_server.Send("/" + issue + "/mail", payload=payload)
+
+  payload = {}  # payload for final request
+  if options.send_mail:
+    payload["send_mail"] = "yes"
+    if options.send_patch:
+      payload["attach_patch"] = "yes"
+  payload = urllib.urlencode(payload)
+  rpc_server.Send("/" + issue + "/upload_complete/" + (patchset or ""),
+                  payload=payload)
   return issue, patchset
 
 
 def main():
   try:
     logging.basicConfig(format=("%(asctime).19s %(levelname)s %(filename)s:"
                                 "%(lineno)s %(message)s "))
     os.environ['LC_ALL'] = 'C'
     RealMain(sys.argv)
   except KeyboardInterrupt:
     print
     StatusUpdate("Interrupted.")
     sys.exit(1)
 
 
 if __name__ == "__main__":
   main()