Update upload.py from http://rietveld.googlecode.com/svn/trunk/upload.py at r546

third_party/upload.py

 #!/usr/bin/env python
 #
 # Copyright 2007 Google Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #     http://www.apache.org/licenses/LICENSE-2.0
 #
@@ skipping 40 matching lines @@
 try:
   from hashlib import md5
 except ImportError:
   from md5 import md5
 
 try:
   import readline
 except ImportError:
   pass
 
+try:
+  import keyring
+except ImportError:
+  keyring = None
+
 # The logging verbosity:
 #  0: Errors only.
 #  1: Status messages.
 #  2: Info logs.
 #  3: Debug logs.
 verbosity = 1
 
+# The account type used for authentication.
+# This line could be changed by the review server (see handler for
+# upload.py).
+AUTH_ACCOUNT_TYPE = "GOOGLE"
+
+# URL of the default review server. As for AUTH_ACCOUNT_TYPE, this line could be
+# changed by the review server (see handler for upload.py).
+DEFAULT_REVIEW_SERVER = "codereview.appspot.com"
+
 # Max size of patch or base file.
 MAX_UPLOAD_SIZE = 900 * 1024
 
 # Constants for version control names.  Used by GuessVCSName.
 VCS_GIT = "Git"
 VCS_MERCURIAL = "Mercurial"
 VCS_SUBVERSION = "Subversion"
 VCS_UNKNOWN = "Unknown"
 
 # whitelist for non-binary filetypes which do not start with "text/"
 # .mm (Objective-C) shows up as application/x-freemind on my Linux box.
 TEXT_MIMETYPES = ['application/javascript', 'application/x-javascript',
-                  'application/xml', 'application/x-freemind']
+                  'application/xml', 'application/x-freemind', 
+                  'application/x-sh']
 
 VCS_ABBREVIATIONS = {
   VCS_MERCURIAL.lower(): VCS_MERCURIAL,
   "hg": VCS_MERCURIAL,
   VCS_SUBVERSION.lower(): VCS_SUBVERSION,
   "svn": VCS_SUBVERSION,
   VCS_GIT.lower(): VCS_GIT,
 }
 
 # The result of parsing Subversion's [auto-props] setting.
@@ skipping 55 matching lines @@
   def __init__(self, url, code, msg, headers, args):
     urllib2.HTTPError.__init__(self, url, code, msg, headers, None)
     self.args = args
     self.reason = args["Error"]
 
 
 class AbstractRpcServer(object):
   """Provides a common interface for a simple RPC server."""
 
   def __init__(self, host, auth_function, host_override=None, extra_headers={},
-               save_cookies=False):
+               save_cookies=False, account_type=AUTH_ACCOUNT_TYPE):
     """Creates a new HttpRpcServer.
 
     Args:
       host: The host to send requests to.
       auth_function: A function that takes no arguments and returns an
         (email, password) tuple when called. Will be called if authentication
         is required.
       host_override: The host header to send to the server (defaults to host).
       extra_headers: A dict of extra headers to append to every request.
       save_cookies: If True, save the authentication cookies to local disk.
         If False, use an in-memory cookiejar instead.  Subclasses must
         implement this functionality.  Defaults to False.
+      account_type: Account type used for authentication. Defaults to
+        AUTH_ACCOUNT_TYPE.
     """
     self.host = host
     if (not self.host.startswith("http://") and
         not self.host.startswith("https://")):
       self.host = "http://" + self.host
     self.host_override = host_override
     self.auth_function = auth_function
     self.authenticated = False
     self.extra_headers = extra_headers
     self.save_cookies = save_cookies
+    self.account_type = account_type
     self.opener = self._GetOpener()
     if self.host_override:
       logging.info("Server: %s; Host: %s", self.host, self.host_override)
     else:
       logging.info("Server: %s", self.host)
 
   def _GetOpener(self):
     """Returns an OpenerDirector for making HTTP requests.
 
     Returns:
@@ skipping 18 matching lines @@
       email:    The user's email address
       password: The user's password
 
     Raises:
       ClientLoginError: If there was an error authenticating with ClientLogin.
       HTTPError: If there was some other form of HTTP error.
 
     Returns:
       The authentication token returned by ClientLogin.
     """
-    account_type = "GOOGLE"
+    account_type = self.account_type
     if self.host.endswith(".google.com"):
       # Needed for use inside Google.
       account_type = "HOSTED"
     req = self._CreateRequest(
         url="https://www.google.com/accounts/ClientLogin",
         data=urllib.urlencode({
             "Email": email,
             "Passwd": password,
             "service": "ah",
             "source": "rietveld-codereview-upload",
@@ skipping 59 matching lines @@
       try:
         auth_token = self._GetAuthToken(credentials[0], credentials[1])
       except ClientLoginError, e:
         if e.reason == "BadAuthentication":
           print >>sys.stderr, "Invalid username or password."
           continue
         if e.reason == "CaptchaRequired":
           print >>sys.stderr, (
               "Please go to\n"
               "https://www.google.com/accounts/DisplayUnlockCaptcha\n"
-              "and verify you are a human.  Then try again.")
+              "and verify you are a human.  Then try again.\n"
+              "If you are using a Google Apps account the URL is:\n"
+              "https://www.google.com/a/yourdomain.com/UnlockCaptcha")
           break
         if e.reason == "NotVerified":
           print >>sys.stderr, "Account not verified."
           break
         if e.reason == "TermsNotAgreed":
           print >>sys.stderr, "User has not agreed to TOS."
           break
         if e.reason == "AccountDeleted":
           print >>sys.stderr, "The user account has been deleted."
           break
@@ skipping 128 matching lines @@
 group.add_option("-q", "--quiet", action="store_const", const=0,
                  dest="verbose", help="Print errors only.")
 group.add_option("-v", "--verbose", action="store_const", const=2,
                  dest="verbose", default=1,
                  help="Print info level logs (default).")
 group.add_option("--noisy", action="store_const", const=3,
                  dest="verbose", help="Print all logs.")
 # Review server
 group = parser.add_option_group("Review server options")
 group.add_option("-s", "--server", action="store", dest="server",
-                 default="codereview.appspot.com",
+                 default=DEFAULT_REVIEW_SERVER,
                  metavar="SERVER",
                  help=("The server to upload to. The format is host[:port]. "
                        "Defaults to '%default'."))
 group.add_option("-e", "--email", action="store", dest="email",
                  metavar="EMAIL", default=None,
                  help="The username to use. Will prompt if omitted.")
 group.add_option("-H", "--host", action="store", dest="host",
                  metavar="HOST", default=None,
                  help="Overrides the Host header sent with all RPCs.")
 group.add_option("--no_cookies", action="store_false",
                  dest="save_cookies", default=True,
                  help="Do not save authentication cookies to local disk.")
+group.add_option("--account_type", action="store", dest="account_type",
+                 metavar="TYPE", default=AUTH_ACCOUNT_TYPE,
+                 choices=["GOOGLE", "HOSTED"],
+                 help=("Override the default account type "
+                       "(defaults to '%default', "
+                       "valid choices are 'GOOGLE' and 'HOSTED')."))
 # Issue
 group = parser.add_option_group("Issue options")
 group.add_option("-d", "--description", action="store", dest="description",
                  metavar="DESCRIPTION", default=None,
                  help="Optional description when creating an issue.")
 group.add_option("-f", "--description_file", action="store",
                  dest="description_file", metavar="DESCRIPTION_FILE",
                  default=None,
                  help="Optional path of a file that contains "
                       "the description when creating an issue.")
@@ skipping 32 matching lines @@
                  help="Send notification email to reviewers.")
 group.add_option("--vcs", action="store", dest="vcs",
                  metavar="VCS", default=None,
                  help=("Version control system (optional, usually upload.py "
                        "already guesses the right VCS)."))
 group.add_option("--emulate_svn_auto_props", action="store_true",
                  dest="emulate_svn_auto_props", default=False,
                  help=("Emulate Subversion's auto properties feature."))
 
 
-def GetRpcServer(server, email=None, host_override=None, save_cookies=True):
+def GetRpcServer(server, email=None, host_override=None, save_cookies=True,
+                 account_type=AUTH_ACCOUNT_TYPE):
   """Returns an instance of an AbstractRpcServer.
 
   Args:
     server: String containing the review server URL.
     email: String containing user's email address.
     host_override: If not None, string containing an alternate hostname to use
       in the host header.
     save_cookies: Whether authentication cookies should be saved to disk.
+    account_type: Account type for authentication, either 'GOOGLE'
+      or 'HOSTED'. Defaults to AUTH_ACCOUNT_TYPE.
 
   Returns:
     A new AbstractRpcServer, on which RPC calls can be made.
   """
 
   rpc_server_class = HttpRpcServer
 
   # If this is the dev_appserver, use fake authentication.
   host = (host_override or server).lower()
   if host == "localhost" or host.startswith("localhost:"):
     if email is None:
       email = "test@example.com"
       logging.info("Using debug user %s.  Override with --email" % email)
     server = rpc_server_class(
         server,
         lambda: (email, "password"),
         host_override=host_override,
         extra_headers={"Cookie":
                        'dev_appserver_login="%s:False"' % email},
-        save_cookies=save_cookies)
+        save_cookies=save_cookies,
+        account_type=account_type)
     # Don't try to talk to ClientLogin.
     server.authenticated = True
     return server
 
   def GetUserCredentials():
     """Prompts the user for a username and password."""
     # Create a local alias to the email variable to avoid Python's crazy
     # scoping rules.
     local_email = email
     if local_email is None:
       local_email = GetEmail("Email (login for uploading to %s)" % server)
-    password = getpass.getpass("Password for %s: " % local_email)
+    password = None
+    if keyring:
+      password = keyring.get_password(host, local_email)
+    if password is not None:
+      print "Using password from system keyring."
+    else:
+      password = getpass.getpass("Password for %s: " % local_email)
+      if keyring:
+        answer = raw_input("Store password in system keyring?(y/N) ").strip()
+        if answer == "y":
+          keyring.set_password(host, local_email, password)
     return (local_email, password)
 
   return rpc_server_class(server,
                           GetUserCredentials,
                           host_override=host_override,
                           save_cookies=save_cookies)
 
 
 def EncodeMultipartFormData(fields, files):
   """Encode form fields for multipart/form-data.
 
   Args:
     fields: A sequence of (name, value) elements for regular form fields.
     files: A sequence of (name, filename, value) elements for data to be
            uploaded as files.
   Returns:
     (content_type, body) ready for httplib.HTTP instance.
 
   Source:
     http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/146306
   """
   BOUNDARY = '-M-A-G-I-C---B-O-U-N-D-A-R-Y-'
   CRLF = '\r\n'
   lines = []
   for (key, value) in fields:
     lines.append('--' + BOUNDARY)
     lines.append('Content-Disposition: form-data; name="%s"' % key)
     lines.append('')
+    if isinstance(value, unicode):
+      value = value.encode('utf-8')
     lines.append(value)
   for (key, filename, value) in files:
     lines.append('--' + BOUNDARY)
     lines.append('Content-Disposition: form-data; name="%s"; filename="%s"' %
              (key, filename))
     lines.append('Content-Type: %s' % GetContentType(filename))
     lines.append('')
+    if isinstance(value, unicode):
+      value = value.encode('utf-8')
     lines.append(value)
   lines.append('--' + BOUNDARY + '--')
   lines.append('')
   body = CRLF.join(lines)
   content_type = 'multipart/form-data; boundary=%s' % BOUNDARY
   return content_type, body
 
 
 def GetContentType(filename):
   """Helper to guess the content-type from the filename."""
@@ skipping 437 matching lines @@
         else:
           universal_newlines = True
         if self.rev_start:
           # "svn cat -r REV delete_file.txt" doesn't work. cat requires
           # the full URL with "@REV" appended instead of using "-r" option.
           url = "%s/%s@%s" % (self.svn_base, filename, self.rev_start)
           base_content = RunShell(["svn", "cat", url],
                                   universal_newlines=universal_newlines,
                                   silent_ok=True)
         else:
-          base_content = RunShell(["svn", "cat", filename],
-                                  universal_newlines=universal_newlines,
-                                  silent_ok=True)
+          base_content, ret_code = RunShellWithReturnCode(
+            ["svn", "cat", filename], universal_newlines=universal_newlines)
+          if ret_code and status[0] == "R":
+            # It's a replaced file without local history (see issue208).
+            # The base file needs to be fetched from the server.
+            url = "%s/%s" % (self.svn_base, filename)
+            base_content = RunShell(["svn", "cat", url],
+                                    universal_newlines=universal_newlines,
+                                    silent_ok=True)
+          elif ret_code:
+            ErrorExit("Got error status from 'svn cat %s'", filename)
         if not is_binary:
           args = []
           if self.rev_start:
             url = "%s/%s@%s" % (self.svn_base, filename, self.rev_start)
           else:
             url = filename
             args += ["-r", "BASE"]
           cmd = ["svn"] + args + ["propget", "svn:keywords", url]
           keywords, returncode = RunShellWithReturnCode(cmd)
           if keywords and not returncode:
@@ skipping 561 matching lines @@
   if options.issue:
     prompt = "Message describing this patch set: "
   else:
     prompt = "New issue subject: "
   message = options.message or raw_input(prompt).strip()
   if not message:
     ErrorExit("A non-empty message is required")
   rpc_server = GetRpcServer(options.server,
                             options.email,
                             options.host,
-                            options.save_cookies)
+                            options.save_cookies,
+                            options.account_type)
   form_fields = [("subject", message)]
   if base:
     form_fields.append(("base", base))
   if options.issue:
     form_fields.append(("issue", str(options.issue)))
   if options.email:
     form_fields.append(("user", options.email))
   if options.reviewers:
     for reviewer in options.reviewers.split(','):
       CheckReviewer(reviewer)
@@ skipping 73 matching lines @@
   try:
     RealMain(sys.argv)
   except KeyboardInterrupt:
     print
     StatusUpdate("Interrupted.")
     sys.exit(1)
 
 
 if __name__ == "__main__":
   main()