Update upload.py in depot_tools to infra.git@3d174a0959ba68a4cd83a5abce9f497e30bfce68

third_party/upload.py

 #!/usr/bin/env python
 # coding: utf-8
 #
 # Copyright 2007 Google Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #     http://www.apache.org/licenses/LICENSE-2.0
@@ skipping 85 matching lines @@
 
 # Constants for version control names.  Used by GuessVCSName.
 VCS_GIT = "Git"
 VCS_MERCURIAL = "Mercurial"
 VCS_SUBVERSION = "Subversion"
 VCS_PERFORCE = "Perforce"
 VCS_CVS = "CVS"
 VCS_UNKNOWN = "Unknown"
 
 VCS = [
-{
-    'name': VCS_MERCURIAL,
-    'aliases': ['hg', 'mercurial'],
-}, {
-    'name': VCS_SUBVERSION,
-    'aliases': ['svn', 'subversion'],
-}, {
-    'name': VCS_PERFORCE,
-    'aliases': ['p4', 'perforce'],
-}, {
-    'name': VCS_GIT,
-    'aliases': ['git'],
-}, {
-    'name': VCS_CVS,
-    'aliases': ['cvs'],
-}]
+  {'name': VCS_MERCURIAL,
+   'aliases': ['hg', 'mercurial']},
+  {'name': VCS_SUBVERSION,
+   'aliases': ['svn', 'subversion'],},
+  {'name': VCS_PERFORCE,
+   'aliases': ['p4', 'perforce']},
+  {'name': VCS_GIT,
+   'aliases': ['git']},
+  {'name': VCS_CVS,
+   'aliases': ['cvs']},
+  ]
+
 
 VCS_SHORT_NAMES = []    # hg, svn, ...
 VCS_ABBREVIATIONS = {}  # alias: name, ...
 for vcs in VCS:
   VCS_SHORT_NAMES.append(min(vcs['aliases'], key=len))
   VCS_ABBREVIATIONS.update((alias, vcs['name']) for alias in vcs['aliases'])
 
 
 # OAuth 2.0-Related Constants
 LOCALHOST_IP = '127.0.0.1'
@@ skipping 79 matching lines @@
 
   Args:
     msg: The string to print.
   """
   if verbosity > 0:
     print msg
 
 
 def ErrorExit(msg):
   """Print an error message to stderr and exit."""
-  print >>sys.stderr, msg
+  print >> sys.stderr, msg
   sys.exit(1)
 
 
 class ClientLoginError(urllib2.HTTPError):
   """Raised to indicate there was an error authenticating with ClientLogin."""
 
   def __init__(self, url, code, msg, headers, args):
     urllib2.HTTPError.__init__(self, url, code, msg, headers, None)
     self.args = args
     self._reason = args["Error"]
@@ skipping 142 matching lines @@
 
     If we attempt to access the upload API without first obtaining an
     authentication cookie, it returns a 401 response (or a 302) and
     directs us to authenticate ourselves with ClientLogin.
     """
     for i in range(3):
       credentials = self.auth_function()
       try:
         auth_token = self._GetAuthToken(credentials[0], credentials[1])
       except ClientLoginError, e:
-        print >>sys.stderr, ''
+        print >> sys.stderr, ''
         if e.reason == "BadAuthentication":
           if e.info == "InvalidSecondFactor":
-            print >>sys.stderr, (
+            print >> sys.stderr, (
                 "Use an application-specific password instead "
                 "of your regular account password.\n"
                 "See http://www.google.com/"
                 "support/accounts/bin/answer.py?answer=185833")
           else:
-            print >>sys.stderr, "Invalid username or password."
+            print >> sys.stderr, "Invalid username or password."
         elif e.reason == "CaptchaRequired":
-          print >>sys.stderr, (
+          print >> sys.stderr, (
               "Please go to\n"
               "https://www.google.com/accounts/DisplayUnlockCaptcha\n"
               "and verify you are a human.  Then try again.\n"
               "If you are using a Google Apps account the URL is:\n"
               "https://www.google.com/a/yourdomain.com/UnlockCaptcha")
         elif e.reason == "NotVerified":
-          print >>sys.stderr, "Account not verified."
+          print >> sys.stderr, "Account not verified."
         elif e.reason == "TermsNotAgreed":
-          print >>sys.stderr, "User has not agreed to TOS."
+          print >> sys.stderr, "User has not agreed to TOS."
         elif e.reason == "AccountDeleted":
-          print >>sys.stderr, "The user account has been deleted."
+          print >> sys.stderr, "The user account has been deleted."
         elif e.reason == "AccountDisabled":
-          print >>sys.stderr, "The user account has been disabled."
+          print >> sys.stderr, "The user account has been disabled."
           break
         elif e.reason == "ServiceDisabled":
-          print >>sys.stderr, ("The user's access to the service has been "
+          print >> sys.stderr, ("The user's access to the service has been "
                                "disabled.")
         elif e.reason == "ServiceUnavailable":
-          print >>sys.stderr, "The service is not available; try again later."
+          print >> sys.stderr, "The service is not available; try again later."
         else:
           # Unknown error.
           raise
-        print >>sys.stderr, ''
+        print >> sys.stderr, ''
         continue
       self._GetAuthCookie(auth_token)
       return
 
   def Send(self, request_path, payload=None,
            content_type="application/octet-stream",
            timeout=None,
            extra_headers=None,
            **kwargs):
     """Sends an RPC and returns the response.
@@ skipping 26 matching lines @@
         args = dict(kwargs)
         url = "%s%s" % (self.host, request_path)
         if args:
           url += "?" + urllib.urlencode(args)
         req = self._CreateRequest(url=url, data=payload)
         req.add_header("Content-Type", content_type)
         if extra_headers:
           for header, value in extra_headers.items():
             req.add_header(header, value)
         try:
-          f = self.opener.open(req)
+          f = self.opener.open(req, timeout=70)
           response = f.read()
           f.close()
           return response
         except urllib2.HTTPError, e:
           if tries > 3:
             raise
           elif e.code == 401 or e.code == 302:
             if not self.auth_function:
               raise
             self._Authenticate()
@@ skipping 60 matching lines @@
       # Always chmod the cookie file
       os.chmod(self.cookie_file, 0600)
     else:
       # Don't save cookies across runs of update.py.
       self.cookie_jar = cookielib.CookieJar()
     opener.add_handler(urllib2.HTTPCookieProcessor(self.cookie_jar))
     return opener
 
 
 class CondensedHelpFormatter(optparse.IndentedHelpFormatter):
-   """Frees more horizontal space by removing indentation from group
-      options and collapsing arguments between short and long, e.g.
-      '-o ARG, --opt=ARG' to -o --opt ARG"""
+  """Frees more horizontal space by removing indentation from group
+     options and collapsing arguments between short and long, e.g.
+     '-o ARG, --opt=ARG' to -o --opt ARG"""
 
-   def format_heading(self, heading):
-     return "%s:\n" % heading
+  def format_heading(self, heading):
+    return "%s:\n" % heading
 
-   def format_option(self, option):
-     self.dedent()
-     res = optparse.HelpFormatter.format_option(self, option)
-     self.indent()
-     return res
+  def format_option(self, option):
+    self.dedent()
+    res = optparse.HelpFormatter.format_option(self, option)
+    self.indent()
+    return res
 
-   def format_option_strings(self, option):
-     self.set_long_opt_delimiter(" ")
-     optstr = optparse.HelpFormatter.format_option_strings(self, option)
-     optlist = optstr.split(", ")
-     if len(optlist) > 1:
-       if option.takes_value():
-         # strip METAVAR from all but the last option
-         optlist = [x.split()[0] for x in optlist[:-1]] + optlist[-1:]
-       optstr = " ".join(optlist)
-     return optstr
+  def format_option_strings(self, option):
+    self.set_long_opt_delimiter(" ")
+    optstr = optparse.HelpFormatter.format_option_strings(self, option)
+    optlist = optstr.split(", ")
+    if len(optlist) > 1:
+      if option.takes_value():
+        # strip METAVAR from all but the last option
+        optlist = [x.split()[0] for x in optlist[:-1]] + optlist[-1:]
+      optstr = " ".join(optlist)
+    return optstr
 
 
 parser = optparse.OptionParser(
-    usage=("%prog [options] [-- diff_options] [path...]\n"
-           "See also: http://code.google.com/p/rietveld/wiki/UploadPyUsage"),
-    add_help_option=False,
-    formatter=CondensedHelpFormatter()
-)
+  usage=("%prog [options] [-- diff_options] [path...]\n"
+         "See also: http://code.google.com/p/rietveld/wiki/UploadPyUsage"),
+  add_help_option=False,
+  formatter=CondensedHelpFormatter()
+  )
 parser.add_option("-h", "--help", action="store_true",
                   help="Show this help message and exit.")
 parser.add_option("-y", "--assume_yes", action="store_true",
                   dest="assume_yes", default=False,
                   help="Assume that the answer to yes/no questions is 'yes'.")
 # Logging
 group = parser.add_option_group("Logging options")
 group.add_option("-q", "--quiet", action="store_const", const=0,
                  dest="verbose", help="Print errors only.")
 group.add_option("-v", "--verbose", action="store_const", const=2,
@@ skipping 87 matching lines @@
                  metavar="VCS", default=None,
                  help=("Explicitly specify version control system (%s)"
                        % ", ".join(VCS_SHORT_NAMES)))
 group.add_option("--emulate_svn_auto_props", action="store_true",
                  dest="emulate_svn_auto_props", default=False,
                  help=("Emulate Subversion's auto properties feature."))
 # Git-specific
 group = parser.add_option_group("Git-specific options")
 group.add_option("--git_similarity", action="store", dest="git_similarity",
                  metavar="SIM", type="int", default=50,
-                 help=("Set the minimum similarity index for detecting renames "
-                       "and copies. See `git diff -C`. (default 50)."))
+                 help=("Set the minimum similarity percentage for detecting "
+                       "renames and copies. See `git diff -C`. (default 50)."))
+group.add_option("--git_only_search_patch", action="store_false", default=True,
+                 dest='git_find_copies_harder',
+                 help="Removes --find-copies-harder when seaching for copies")
 group.add_option("--git_no_find_copies", action="store_false", default=True,
                  dest="git_find_copies",
                  help=("Prevents git from looking for copies (default off)."))
 # Perforce-specific
 group = parser.add_option_group("Perforce-specific options "
                                 "(overrides P4 environment variables)")
 group.add_option("--p4_port", action="store", dest="p4_port",
                  metavar="P4_PORT", default=None,
                  help=("Perforce server and port (optional)"))
 group.add_option("--p4_changelist", action="store", dest="p4_changelist",
@@ skipping 307 matching lines @@
   """Helper to guess the content-type from the filename."""
   return mimetypes.guess_type(filename)[0] or 'application/octet-stream'
 
 
 # Use a shell for subcommands on Windows to get a PATH search.
 use_shell = sys.platform.startswith("win")
 
 def RunShellWithReturnCodeAndStderr(command, print_output=False,
                            universal_newlines=True,
                            env=os.environ):
-  """Executes a command and returns the output from stdout, stderr and the return code.
+  """Run a command and return output from stdout, stderr and the return code.
 
   Args:
     command: Command to execute.
     print_output: If True, the output is printed to stdout.
                   If False, both stdout and stderr are ignored.
     universal_newlines: Use universal_newlines flag (default: True).
 
   Returns:
     Tuple (stdout, stderr, return code)
   """
@@ skipping 10 matching lines @@
       if not line:
         break
       print line.strip("\n")
       output_array.append(line)
     output = "".join(output_array)
   else:
     output = p.stdout.read()
   p.wait()
   errout = p.stderr.read()
   if print_output and errout:
-    print >>sys.stderr, errout
+    print >> sys.stderr, errout
   p.stdout.close()
   p.stderr.close()
   return output, errout, p.returncode
 
 def RunShellWithReturnCode(command, print_output=False,
                            universal_newlines=True,
                            env=os.environ):
-  """Executes a command and returns the output from stdout and the return code."""
+  """Run a command and return output from stdout and the return code."""
   out, err, retcode = RunShellWithReturnCodeAndStderr(command, print_output,
                            universal_newlines, env)
   return out, retcode
 
 def RunShell(command, silent_ok=False, universal_newlines=True,
              print_output=False, env=os.environ):
   data, retcode = RunShellWithReturnCode(command, print_output,
                                          universal_newlines, env)
   if retcode:
     ErrorExit("Got error status from %s:\n%s" % (command, data))
@@ skipping 154 matching lines @@
 
     for t in threads:
       print t.get(timeout=60)
 
 
   def IsImage(self, filename):
     """Returns true if the filename has an image extension."""
     mimetype =  mimetypes.guess_type(filename)[0]
     if not mimetype:
       return False
-    return mimetype.startswith("image/") and not mimetype.startswith("image/svg")
+    return (mimetype.startswith("image/") and
+            not mimetype.startswith("image/svg"))
 
   def IsBinaryData(self, data):
     """Returns true if data contains a null byte."""
     # Derived from how Mercurial's heuristic, see
     # http://selenic.com/hg/file/848a6658069e/mercurial/util.py#l229
     return bool(data and "\0" in data)
 
 
 class SubversionVCS(VersionControlSystem):
   """Implementation of the VersionControlSystem interface for Subversion."""
@@ skipping 25 matching lines @@
 
   def _GuessBase(self, required):
     """Returns base URL for current diff.
 
     Args:
       required: If true, exits if the url can't be guessed, otherwise None is
         returned.
     """
     url = self._GetInfo("URL")
     if url:
-        scheme, netloc, path, params, query, fragment = urlparse.urlparse(url)
-        guess = ""
-        # TODO(anatoli) - repository specific hacks should be handled by server
-        if netloc == "svn.python.org" and scheme == "svn+ssh":
-          path = "projects" + path
-          scheme = "http"
-          guess = "Python "
-        elif netloc.endswith(".googlecode.com"):
-          scheme = "http"
-          guess = "Google Code "
-        path = path + "/"
-        base = urlparse.urlunparse((scheme, netloc, path, params,
-                                    query, fragment))
-        LOGGER.info("Guessed %sbase = %s", guess, base)
-        return base
+      scheme, netloc, path, params, query, fragment = urlparse.urlparse(url)
+      guess = ""
+      # TODO(anatoli) - repository specific hacks should be handled by server
+      if netloc == "svn.python.org" and scheme == "svn+ssh":
+        path = "projects" + path
+        scheme = "http"
+        guess = "Python "
+      elif netloc.endswith(".googlecode.com"):
+        scheme = "http"
+        guess = "Google Code "
+      path = path + "/"
+      base = urlparse.urlunparse((scheme, netloc, path, params,
+                                  query, fragment))
+      LOGGER.info("Guessed %sbase = %s", guess, base)
+      return base
     if required:
       ErrorExit("Can't find URL in output from svn info")
     return None
 
   def _GetInfo(self, key):
     """Parses 'svn info' for current dir. Returns value for key or None"""
     for line in RunShell(["svn", "info"]).splitlines():
       if line.startswith(key + ": "):
         return line.split(":", 1)[1].strip()
 
@@ skipping 34 matching lines @@
       'Id':                  ['Id'],
 
       # Aliases
       'LastChangedDate':     ['LastChangedDate', 'Date'],
       'LastChangedRevision': ['LastChangedRevision', 'Rev', 'Revision'],
       'LastChangedBy':       ['LastChangedBy', 'Author'],
       'URL':                 ['URL', 'HeadURL'],
     }
 
     def repl(m):
-       if m.group(2):
-         return "$%s::%s$" % (m.group(1), " " * len(m.group(3)))
-       return "$%s$" % m.group(1)
+      if m.group(2):
+        return "$%s::%s$" % (m.group(1), " " * len(m.group(3)))
+      return "$%s$" % m.group(1)
+
     keywords = [keyword
                 for name in keyword_str.split(" ")
                 for keyword in svn_keywords.get(name, [])]
     return re.sub(r"\$(%s):(:?)([^\$]+)\$" % '|'.join(keywords), repl, content)
 
   def GetUnknownFiles(self):
     status = RunShell(["svn", "status", "--ignore-externals"], silent_ok=True)
     unknown_files = []
     for line in status.split("\n"):
       if line and line[0] == "?":
@@ skipping 32 matching lines @@
     # the correct status for a file.
     else:
       dirname, relfilename = os.path.split(filename)
       if dirname not in self.svnls_cache:
         cmd = ["svn", "list", "-r", self.rev_start,
                self._EscapeFilename(dirname) or "."]
         out, err, returncode = RunShellWithReturnCodeAndStderr(cmd)
         if returncode:
           # Directory might not yet exist at start revison
           # svn: Unable to find repository location for 'abc' in revision nnn
-          if re.match('^svn: Unable to find repository location for .+ in revision \d+', err):
+          if re.match('^svn: Unable to find repository location '
+                      'for .+ in revision \d+', err):
             old_files = ()
           else:
             ErrorExit("Failed to get status for %s:\n%s" % (filename, err))
         else:
           old_files = out.splitlines()
         args = ["svn", "list"]
         if self.rev_end:
           args += ["-r", self.rev_end]
         cmd = args + [self._EscapeFilename(dirname) or "."]
         out, returncode = RunShellWithReturnCode(cmd)
@@ skipping 201 matching lines @@
     # git config key "diff.external" is used).
     env = os.environ.copy()
     if "GIT_EXTERNAL_DIFF" in env:
       del env["GIT_EXTERNAL_DIFF"]
     # -M/-C will not print the diff for the deleted file when a file is renamed.
     # This is confusing because the original file will not be shown on the
     # review when a file is renamed. So, get a diff with ONLY deletes, then
     # append a diff (with rename detection), without deletes.
     cmd = [
         "git", "diff", "--no-color", "--no-ext-diff", "--full-index",
-        "--ignore-submodules",
+        "--ignore-submodules", "--src-prefix=a/", "--dst-prefix=b/",
     ]
     diff = RunShell(
         cmd + ["--no-renames", "--diff-filter=D"] + extra_args,
         env=env, silent_ok=True)
+    assert 0 <= self.options.git_similarity <= 100
     if self.options.git_find_copies:
-      similarity_options = ["--find-copies-harder", "-l100000",
-                            "-C%s" % self.options.git_similarity ]
+      similarity_options = ["-l100000", "-C%d%%" % self.options.git_similarity]
+      if self.options.git_find_copies_harder:
+        similarity_options.append("--find-copies-harder")
     else:
-      similarity_options = ["-M%s" % self.options.git_similarity ]
+      similarity_options = ["-M%d%%" % self.options.git_similarity ]
     diff += RunShell(
         cmd + ["--diff-filter=AMCRT"] + similarity_options + extra_args,
         env=env, silent_ok=True)
 
     # The CL could be only file deletion or not. So accept silent diff for both
     # commands then check for an empty diff manually.
     if not diff:
       ErrorExit("No output from %s" % (cmd + extra_args))
     return diff
 
@@ skipping 440 matching lines @@
         diffData.base_rev = fstat["headRev"] # Re-adding a deleted file
       else:
         diffData.base_rev = "0" # Brand new file
       diffData.working_copy = False
       rel_path = self.GetLocalFilename(diffData.filename)
       diffData.file_body = open(rel_path, 'r').read()
       # Replicate svn's list of changed lines
       line_count = len(diffData.file_body.splitlines())
       diffData.change_summary = "@@ -0,0 +1"
       if line_count > 1:
-          diffData.change_summary += ",%d" % line_count
+        diffData.change_summary += ",%d" % line_count
       diffData.change_summary += " @@"
       diffData.prefix = "+"
       return diffData
 
     def GenerateDeleteDiff(diffData):
       diffData.base_rev = self.GetBaseRevision(diffData.filename)
       is_base_binary = self.IsBaseBinary(diffData.filename)
       # For deletes, base_filename == filename
       diffData.file_body = self.GetFileContent(diffData.base_filename,
           None,
@@ skipping 606 matching lines @@
     os.environ['LC_ALL'] = 'C'
     RealMain(sys.argv)
   except KeyboardInterrupt:
     print
     StatusUpdate("Interrupted.")
     sys.exit(1)
 
 
 if __name__ == "__main__":
   main()