Update fancy_urllib to work with python-2.7.10

third_party/fancy_urllib/__init__.py

-#!/usr/bin/env python
-#
 # Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007 Python Software
 # Foundation; All Rights Reserved
 
 """A HTTPSConnection/Handler with additional proxy and cert validation features.
 
 In particular, monkey patches in Python r74203 to provide support for CONNECT
 proxies and adds SSL cert validation if the ssl module is present.
 """
 
 __author__ = "{frew,nick.johnson}@google.com (Fred Wulff and Nick Johnson)"
 
 import base64
 import httplib
 import logging
-import re
 import socket
+from urllib import splitpasswd
+from urllib import splittype
+from urllib import splituser
 import urllib2
 
-from urllib import splittype
-from urllib import splituser
-from urllib import splitpasswd
 
 class InvalidCertificateException(httplib.HTTPException):
   """Raised when a certificate is provided with an invalid hostname."""
 
   def __init__(self, host, cert, reason):
     """Constructor.
 
     Args:
       host: The hostname the connection was made to.
       cert: The SSL certificate (as a dictionary) the host returned.
+      reason: user readable error reason.
     """
     httplib.HTTPException.__init__(self)
     self.host = host
     self.cert = cert
     self.reason = reason
 
   def __str__(self):
-    return ('Host %s returned an invalid certificate (%s): %s\n'
-            'To learn more, see '
-            'http://code.google.com/appengine/kb/general.html#rpcssl' %
+    return ("Host %s returned an invalid certificate (%s): %s\n"
+            "To learn more, see "
+            "http://code.google.com/appengine/kb/general.html#rpcssl" %
             (self.host, self.reason, self.cert))
 
+
+try:
+  import ssl
+  _CAN_VALIDATE_CERTS = True
+except ImportError:
+  _CAN_VALIDATE_CERTS = False
+
+
 def can_validate_certs():
   """Return True if we have the SSL package and can validate certificates."""
-  try:
-    import ssl
-    return True
-  except ImportError:
-    return False
+  return _CAN_VALIDATE_CERTS
 
-def _create_fancy_connection(tunnel_host=None, key_file=None,
-                             cert_file=None, ca_certs=None):
+
+# Reexport SSLError so clients don't have to to do their own checking for ssl's
+# existence.
+if can_validate_certs():
+  SSLError = ssl.SSLError
+else:
+  SSLError = None
+
+
+def create_fancy_connection(tunnel_host=None, key_file=None,
+                            cert_file=None, ca_certs=None,
+                            proxy_authorization=None):
   # This abomination brought to you by the fact that
   # the HTTPHandler creates the connection instance in the middle
   # of do_open so we need to add the tunnel host to the class.
 
   class PresetProxyHTTPSConnection(httplib.HTTPSConnection):
     """An HTTPS connection that uses a proxy defined by the enclosing scope."""
 
     def __init__(self, *args, **kwargs):
       httplib.HTTPSConnection.__init__(self, *args, **kwargs)
 
       self._tunnel_host = tunnel_host
       if tunnel_host:
         logging.debug("Creating preset proxy https conn: %s", tunnel_host)
 
       self.key_file = key_file
       self.cert_file = cert_file
       self.ca_certs = ca_certs
-      try:
-        import ssl
+      if can_validate_certs():
         if self.ca_certs:
           self.cert_reqs = ssl.CERT_REQUIRED
         else:
           self.cert_reqs = ssl.CERT_NONE
-      except ImportError:
-        pass
+
+    def _get_hostport(self, host, port):
+      # Python 2.7.7rc1 (hg r90728:568041fd8090), 3.4.1 and 3.5 rename
+      # _set_hostport to _get_hostport and changes it's functionality.  The
+      # Python 2.7.7rc1 version of this method is included here for
+      # compatibility with earlier versions of Python.  Without this, HTTPS over
+      # HTTP CONNECT proxies cannot be used.
+
+      # This method may be removed if compatibility with Python <2.7.7rc1 is not
+      # required.
+
+      # Python bug: http://bugs.python.org/issue7776
+      if port is None:
+        i = host.rfind(":")
+        j = host.rfind("]")         # ipv6 addresses have [...]
+        if i > j:
+          try:
+            port = int(host[i+1:])
+          except ValueError:
+            if host[i+1:] == "":  # http://foo.com:/ == http://foo.com/
+              port = self.default_port
+            else:
+              raise httplib.InvalidURL("nonnumeric port: '%s'" % host[i+1:])
+          host = host[:i]
+        else:
+          port = self.default_port
+        if host and host[0] == "[" and host[-1] == "]":
+          host = host[1:-1]
+
+      return (host, port)
 
     def _tunnel(self):
-      self._set_hostport(self._tunnel_host, None)
+      self.host, self.port = self._get_hostport(self._tunnel_host, None)
       logging.info("Connecting through tunnel to: %s:%d",
                    self.host, self.port)
-      self.send("CONNECT %s:%d HTTP/1.0\r\n\r\n" % (self.host, self.port))
+
+      self.send("CONNECT %s:%d HTTP/1.0\r\n" % (self.host, self.port))
+
+      if proxy_authorization:
+        self.send("Proxy-Authorization: %s\r\n" % proxy_authorization)
+
+      # blank line
+      self.send("\r\n")
+
       response = self.response_class(self.sock, strict=self.strict,
                                      method=self._method)
+      # pylint: disable=protected-access
       (_, code, message) = response._read_status()
 
       if code != 200:
         self.close()
-        raise socket.error, "Tunnel connection failed: %d %s" % (
-            code, message.strip())
+        raise socket.error("Tunnel connection failed: %d %s" %
+                           (code, message.strip()))
 
       while True:
         line = response.fp.readline()
         if line == "\r\n":
           break
 
     def _get_valid_hosts_for_cert(self, cert):
       """Returns a list of valid host globs for an SSL certificate.
 
       Args:
         cert: A dictionary representing an SSL certificate.
       Returns:
         list: A list of valid host globs.
       """
-      if 'subjectAltName' in cert:
-        return [x[1] for x in cert['subjectAltName'] if x[0].lower() == 'dns']
+      if "subjectAltName" in cert:
+        return [x[1] for x in cert["subjectAltName"] if x[0].lower() == "dns"]
       else:
         # Return a list of commonName fields
-        return [x[0][1] for x in cert['subject']
-                if x[0][0].lower() == 'commonname']
+        return [x[0][1] for x in cert["subject"]
+                if x[0][0].lower() == "commonname"]
 
     def _validate_certificate_hostname(self, cert, hostname):
-      """Validates that a given hostname is valid for an SSL certificate.
+      """Perform RFC2818/6125 validation against a cert and hostname.
 
       Args:
         cert: A dictionary representing an SSL certificate.
         hostname: The hostname to test.
       Returns:
         bool: Whether or not the hostname is valid for this certificate.
       """
       hosts = self._get_valid_hosts_for_cert(cert)
       for host in hosts:
-        # Convert the glob-style hostname expression (eg, '*.google.com') into a
-        # valid regular expression.
-        host_re = host.replace('.', '\.').replace('*', '[^.]*')
-        if re.search('^%s$' % (host_re,), hostname, re.I):
+        # Wildcards are only valid when the * exists at the end of the last
+        # (left-most) label, and there are at least 3 labels in the expression.
+        if ("*." in host and host.count("*") == 1 and
+            host.count(".") > 1 and "." in hostname):
+          left_expected, right_expected = host.split("*.")
+          left_hostname, right_hostname = hostname.split(".", 1)
+          if (left_hostname.startswith(left_expected) and
+              right_expected == right_hostname):
+            return True
+        elif host == hostname:
           return True
       return False
 
-
     def connect(self):
       # TODO(frew): When we drop support for <2.6 (in the far distant future),
       # change this to socket.create_connection.
       self.sock = _create_connection((self.host, self.port))
 
       if self._tunnel_host:
         self._tunnel()
 
       # ssl and FakeSocket got deprecated. Try for the new hotness of wrap_ssl,
-      # with fallback.
-      try:
-        import ssl
+      # with fallback. Note: Since can_validate_certs() just checks for the
+      # ssl module, it's equivalent to attempting to import ssl from
+      # the function, but doesn't require a dynamic import, which doesn't
+      # play nicely with dev_appserver.
+      if can_validate_certs():
         self.sock = ssl.wrap_socket(self.sock,
                                     keyfile=self.key_file,
                                     certfile=self.cert_file,
                                     ca_certs=self.ca_certs,
                                     cert_reqs=self.cert_reqs)
 
         if self.cert_reqs & ssl.CERT_REQUIRED:
           cert = self.sock.getpeercert()
-          hostname = self.host.split(':', 0)[0]
+          hostname = self.host.split(":", 0)[0]
           if not self._validate_certificate_hostname(cert, hostname):
             raise InvalidCertificateException(hostname, cert,
-                                              'hostname mismatch')
-      except ImportError:
-        ssl = socket.ssl(self.sock,
-                         keyfile=self.key_file,
-                         certfile=self.cert_file)
-        self.sock = httplib.FakeSocket(self.sock, ssl)
+                                              "hostname mismatch")
+      else:
+        ssl_socket = socket.ssl(self.sock,
+                                keyfile=self.key_file,
+                                certfile=self.cert_file)
+        self.sock = httplib.FakeSocket(self.sock, ssl_socket)
 
   return PresetProxyHTTPSConnection
 
 
 # Here to end of _create_connection copied wholesale from Python 2.6"s socket.py
 _GLOBAL_DEFAULT_TIMEOUT = object()
 
 
 def _create_connection(address, timeout=_GLOBAL_DEFAULT_TIMEOUT):
   """Connect to *address* and return the socket object.
@@ skipping 148 matching lines @@
     # This condition is the change
     if orig_type == "https":
       return None
 
     return urllib2.ProxyHandler.proxy_open(self, req, proxy, type)
 
 
 class FancyHTTPSHandler(urllib2.HTTPSHandler):
   """An HTTPSHandler that works with CONNECT-enabled proxies."""
 
-  def do_open(self, http_class, req):
+  def do_open(self, http_class, req, *args, **kwargs):
+    proxy_authorization = None
+    for header in req.headers:
+      if header.lower() == "proxy-authorization":
+        proxy_authorization = req.headers[header]
+        break
+
     # Intentionally very specific so as to opt for false negatives
     # rather than false positives.
     try:
       return urllib2.HTTPSHandler.do_open(
           self,
-          _create_fancy_connection(req._tunnel_host,
-                                   req._key_file,
-                                   req._cert_file,
-                                   req._ca_certs),
-          req)
+          create_fancy_connection(req._tunnel_host,
+                                  req._key_file,
+                                  req._cert_file,
+                                  req._ca_certs,
+                                  proxy_authorization),
+          req, *args, **kwargs)
     except urllib2.URLError, url_error:
       try:
         import ssl
         if (type(url_error.reason) == ssl.SSLError and
             url_error.reason.args[0] == 1):
           # Display the reason to the user. Need to use args for python2.5
           # compat.
-          raise InvalidCertificateException(req.host, '',
+          raise InvalidCertificateException(req.host, "",
                                             url_error.reason.args[1])
       except ImportError:
         pass
 
       raise url_error
 
 
 # We have to implement this so that we persist the tunneling behavior
 # through redirects.
 class FancyRedirectHandler(urllib2.HTTPRedirectHandler):
@@ skipping 28 matching lines @@
           # req is not proxied, so just make sure _tunnel_host is defined.
           new_req._tunnel_host = None
         new_req.type = "https"
     if hasattr(req, "_key_file") and isinstance(new_req, urllib2.Request):
       # Copy the auxiliary data in case this or any further redirect is https
       new_req._key_file = req._key_file
       new_req._cert_file = req._cert_file
       new_req._ca_certs = req._ca_certs
 
     return new_req