Use gsutil.py for download_from_google_storage instead of the builtin one

download_from_google_storage.py

 #!/usr/bin/env python
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Download files from Google Storage based on SHA1 sums."""
 
 
 import hashlib
 import optparse
 import os
 import Queue
 import re
 import stat
 import sys
 import threading
 import time
 
 import subprocess2
 
 
 GSUTIL_DEFAULT_PATH = os.path.join(
-    os.path.dirname(os.path.abspath(__file__)),
-    'third_party', 'gsutil', 'gsutil')
+    os.path.dirname(os.path.abspath(__file__)), 'gsutil.py')
 # Maps sys.platform to what we actually want to call them.
 PLATFORM_MAPPING = {
     'cygwin': 'win',
     'darwin': 'mac',
     'linux2': 'linux',
     'win32': 'win',
 }
 
 
 class FileNotFoundError(IOError):
@@ skipping 13 matching lines @@
   Under cygwin, this will always return "win32" like the native Python."""
   if sys.platform == 'cygwin':
     return 'win32'
   return sys.platform
 
 
 # Common utilities
 class Gsutil(object):
   """Call gsutil with some predefined settings.  This is a convenience object,
   and is also immutable."""
-  def __init__(self, path, boto_path, timeout=None, bypass_prodaccess=False):
+  def __init__(self, path, boto_path, timeout=None):
     if not os.path.exists(path):
       raise FileNotFoundError('GSUtil not found in %s' % path)
     self.path = path
     self.timeout = timeout
     self.boto_path = boto_path
-    self.bypass_prodaccess = bypass_prodaccess
+    self.version = '4.7'

[pgervais, 2014/12/17 01:07:07]

Keyword parameters are good for default values:

def __init__(self, ..., version='4.7'):
   ...
   self.version = version

You get additional flexibility for almost no cost.

[hinoka, 2014/12/17 01:09:41]

Done.

 
   def get_sub_env(self):
     env = os.environ.copy()
     if self.boto_path == os.devnull:
       env['AWS_CREDENTIAL_FILE'] = ''
       env['BOTO_CONFIG'] = ''
     elif self.boto_path:
       env['AWS_CREDENTIAL_FILE'] = self.boto_path
       env['BOTO_CONFIG'] = self.boto_path
     else:
       custompath = env.get('AWS_CREDENTIAL_FILE', '~/.boto') + '.depot_tools'
       custompath = os.path.expanduser(custompath)
       if os.path.exists(custompath):
         env['AWS_CREDENTIAL_FILE'] = custompath
 
     return env
 
   def call(self, *args):
-    cmd = [sys.executable, self.path]
-    if self.bypass_prodaccess:
-      cmd.append('--bypass_prodaccess')
+    cmd = [sys.executable, self.path, '--force-version', self.version]
     cmd.extend(args)
     return subprocess2.call(cmd, env=self.get_sub_env(), timeout=self.timeout)
 
   def check_call(self, *args):
-    cmd = [sys.executable, self.path]
-    if self.bypass_prodaccess:
-      cmd.append('--bypass_prodaccess')
+    cmd = [sys.executable, self.path, '--force-version', self.version]
     cmd.extend(args)
     ((out, err), code) = subprocess2.communicate(
         cmd,
         stdout=subprocess2.PIPE,
         stderr=subprocess2.PIPE,
         env=self.get_sub_env(),
         timeout=self.timeout)
 
     # Parse output.
     status_code_match = re.search('status=([0-9]+)', err)
@@ skipping 127 matching lines @@
           file_url, output_filename)))
       continue
     # Fetch the file.
     out_q.put('%d> Downloading %s...' % (thread_num, output_filename))
     try:
       os.remove(output_filename)  # Delete the file if it exists already.
     except OSError:
       if os.path.exists(output_filename):
         out_q.put('%d> Warning: deleting %s failed.' % (
             thread_num, output_filename))
-    code, _, err = gsutil.check_call('cp', '-q', file_url, output_filename)
+    code, _, err = gsutil.check_call('cp', file_url, output_filename)
     if code != 0:
       out_q.put('%d> %s' % (thread_num, err))
       ret_codes.put((code, err))
 
     # Set executable bit.
     if sys.platform == 'cygwin':
       # Under cygwin, mark all files as executable. The executable flag in
       # Google Storage will not be set when uploading from Windows, so if
       # this script is running under cygwin and we're downloading an
       # executable, it will be unrunnable from inside cygwin without this.
@@ skipping 140 matching lines @@
               options.platform)
       return 0
 
   # Set the boto file to /dev/null if we don't need auth.
   if options.no_auth:
     options.boto = os.devnull
 
   # Make sure gsutil exists where we expect it to.
   if os.path.exists(GSUTIL_DEFAULT_PATH):
     gsutil = Gsutil(GSUTIL_DEFAULT_PATH,
-                    boto_path=options.boto,
-                    bypass_prodaccess=options.no_auth)
+                    boto_path=options.boto)
   else:
     parser.error('gsutil not found in %s, bad depot_tools checkout?' %
                  GSUTIL_DEFAULT_PATH)
 
   # Passing in -g/--config will run our copy of GSUtil, then quit.
   if options.config:
     return gsutil.call('config', '-r', '-o',
                        os.path.expanduser('~/.boto.depot_tools'))
 
   if not args:
@@ skipping 47 matching lines @@
       return code
 
   return download_from_google_storage(
       input_filename, base_url, gsutil, options.num_threads, options.directory,
       options.recursive, options.force, options.output, options.ignore_errors,
       options.sha1_file, options.verbose, options.auto_platform)
 
 
 if __name__ == '__main__':
   sys.exit(main(sys.argv))