gsutil: Pass proxy information via Boto options.

gsutil.py

 #!/usr/bin/env python
 # Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Run a pinned gsutil."""
 
 
 import argparse
 import base64
 import hashlib
 import json
 import os
 import shutil
 import subprocess
 import sys
 import urllib2
+import urlparse
 import zipfile
 
 
 GSUTIL_URL = 'https://storage.googleapis.com/pub/'
 API_URL = 'https://www.googleapis.com/storage/v1/b/pub/o/'
 
 THIS_DIR = os.path.dirname(os.path.abspath(__file__))
 DEFAULT_BIN_DIR = os.path.join(THIS_DIR, 'external_bin', 'gsutil')
 DEFAULT_FALLBACK_GSUTIL = os.path.join(
     THIS_DIR, 'third_party', 'gsutil', 'gsutil')
@@ skipping 86 matching lines @@
   with zipfile.ZipFile(target_zip_filename, 'r') as target_zip:
     target_zip.extractall(bin_dir)
 
   # Final check that the gsutil bin is okay.  This should never fail.
   if not check_gsutil(gsutil_bin):
     raise InvalidGsutilError()
 
   return gsutil_bin
 
 
+def get_boto_proxy_options_from_environment():
+  """Derive proxy options for gsutil from the environment.
+
+  Parses the contents of the 'http[s]_proxy' (and variations) environment
+  variable and converts it to the corresponding Boto options that gsutil
+  accepts.
+  """
+  proxy_variable = None

[Primiano Tucci (use gerrit), 2015/01/09 17:17:31]

I think you can squeeze lines 132-136 in:
vars = ('https_proxy', 'HTTPS_PROXY', 'http_proxy', 'HTTP_PROXY')
proxy_variable = next((os.environ[v] for v in vars if v in os.environ), None)

+  for env_var in ('https_proxy', 'HTTPS_PROXY', 'http_proxy', 'HTTP_PROXY'):
+    if env_var in os.environ:
+      proxy_variable = env_var
+      break
+  if proxy_variable is None:

[Primiano Tucci (use gerrit), 2015/01/09 17:17:31]

I think here you want to check: if not proxy_variable
If the variable is set but is empty I guess you want to bail out anyways.

[Raphael Kubo da Costa, 2015/01/09 17:54:12]

Doing this works, but if I only store os.environ[v] I can't say which variable
was used in the EnvironmentError message below. And then if the value is empty
the if check down there will fail with a proper message about the format being
wrong anyway.

+    return []
+
+  proxy_url = urlparse.urlsplit(os.environ[proxy_variable])
+  if proxy_url.hostname is None or proxy_url.port is None:
+    raise EnvironmentError(

[Primiano Tucci (use gerrit), 2015/01/09 17:17:31]

Not sure you want to be so strict on the port. What if
http_proxy=http://myhost.corp ?
I'd make it more gentle and narrow the check only for hostname:
if proxy_url.hostname is None: raise ......

and below do something like
'Boto:proxy_port=%d' % (proxy_url.port or 80)

[Raphael Kubo da Costa, 2015/01/09 17:54:12]

As far as I can see, httplib2 (which gsutil uses) requires both the hostname and
the port to be set, otherwise the port will be set to 0 by default.

+        'The "%s" environment is set, but has an invalid format. Its '
+        'format must be "http://[user[:password]@]host:port".' % proxy_variable)
+  boto_options = ['-o', 'Boto:proxy=%s' % proxy_url.hostname,
+                  '-o', 'Boto:proxy_port=%d' % proxy_url.port]
+  if proxy_url.username:
+    boto_options.extend(['-o', 'Boto:proxy_user=%s' % proxy_url.username])
+  if proxy_url.password:
+    boto_options.extend(['-o', 'Boto:proxy_pass=%s' % proxy_url.password])
+  return boto_options
+
+
 def run_gsutil(force_version, fallback, target, args):
   if force_version:
     gsutil_bin = ensure_gsutil(force_version, target)
+    proxy_options = get_boto_proxy_options_from_environment()
   else:
     gsutil_bin = fallback
-  cmd = [sys.executable, gsutil_bin] + args
+    proxy_options = []  # Not available in the fallback gsutil.
+  cmd = [sys.executable, gsutil_bin] + proxy_options + args
   try:
     call(cmd)
   except SubprocessError as e:
     return e.code
   return 0
 
 
 def parse_args():
   parser = argparse.ArgumentParser()
   parser.add_argument('--force-version')
   parser.add_argument('--fallback', default=DEFAULT_FALLBACK_GSUTIL)
   parser.add_argument('--target', default=DEFAULT_BIN_DIR)
   parser.add_argument('args', nargs=argparse.REMAINDER)
 
   args, extras = parser.parse_known_args()
   if args.args and args.args[0] == '--':
     args.args.pop(0)
   if extras:
     args.args = extras + args.args
   return args.force_version, args.fallback, args.target, args.args
 
 
 def main():
   force_version, fallback, target, args = parse_args()
   return run_gsutil(force_version, fallback, target, args)
 
 if __name__ == '__main__':
   sys.exit(main())