Implement git-drover.

download_from_google_storage.py

Index: download_from_google_storage.py
diff --git a/download_from_google_storage.py b/download_from_google_storage.py
index 8370515e6a15879c3ddf5c1e8693efeb595a8ccd..c184b87e8d5cb23c3c24b94cff04327505e12668 100755
--- a/download_from_google_storage.py
+++ b/download_from_google_storage.py
@@ -66,7 +66,7 @@ class Gsutil(object):
   def get_sub_env(self):
     env = os.environ.copy()
     if self.boto_path == os.devnull:
-      env['AWS_CREDENTIAL_FILE'] = ''
+      env.pop('AWS_CREDENTIAL_FILE', None)

[Ryan Tseng, 2014/04/18 00:28:53]

???

[iannucci, 2014/04/28 21:05:28]

Otherwise gsutil freaks out because the env var is set but not a valid path.
This is more-better.

       env['BOTO_CONFIG'] = ''
     elif self.boto_path:
       env['AWS_CREDENTIAL_FILE'] = self.boto_path
@@ -86,17 +86,35 @@ class Gsutil(object):
     cmd.extend(args)
     return subprocess2.call(cmd, env=self.get_sub_env(), timeout=self.timeout)
 
-  def check_call(self, *args):
+  def check_call(self, *args, **kwargs):
+    def tee(stream, buf):
+      def _inner(line):

[Ryan Tseng, 2014/04/18 00:28:53]

"data" or "char" is probably more appropriate vs "line", since subprocess2 will
only ever call this with a single char

[iannucci, 2014/04/28 21:05:28]

What? That's not true...

/me reads subprocess2 implementation

What the... who... why.... arg! :(

+        stream.write(line)
+        buf.append(line)
+      return _inner
+
     cmd = [sys.executable, self.path]
     if self.bypass_prodaccess:
       cmd.append('--bypass_prodaccess')
     cmd.extend(args)
-    ((out, err), code) = subprocess2.communicate(
-        cmd,
-        stdout=subprocess2.PIPE,
-        stderr=subprocess2.PIPE,
-        env=self.get_sub_env(),
-        timeout=self.timeout)
+
+    out = []
+    err = []

[Ryan Tseng, 2014/04/18 00:28:53]

why not a string or cStringIO?

I'd probably prefer cStringIO.

[Ryan Tseng, 2014/04/18 00:37:36]

Disregard that, I ran a test and your list implementation is faster.

I don't understand python.

[iannucci, 2014/04/28 21:05:28]

string allocation n' stuff

+
+    proc = subprocess2.Popen(cmd, env=self.get_sub_env(),
+                             stdout=subprocess2.PIPE,
+                             stderr=subprocess2.PIPE)
+    if kwargs.get('verbose'):
+      proc.stdout_cb = tee(sys.stdout, out)
+      proc.stderr_cb = tee(sys.stderr, err)
+    else:
+      proc.stdout_cb = out.append
+      proc.stderr_cb = err.append
+    proc.communicate(timeout=self.timeout)
+    code = proc.returncode
+
+    out = ''.join(out)
+    err = ''.join(err)
 
     # Parse output.
     status_code_match = re.search('status=([0-9]+)', err)