Add a platform filtering and executable setting abilities to download_from_google_storage.

download_from_google_storage.py

Index: download_from_google_storage.py
===================================================================
--- download_from_google_storage.py	(revision 230811)
+++ download_from_google_storage.py	(working copy)
@@ -11,6 +11,7 @@
 import os
 import Queue
 import re
+import stat
 import sys
 import threading
 import time
@@ -158,7 +159,7 @@
 
 
 def _downloader_worker_thread(thread_num, q, force, base_url,
-                              gsutil, out_q, ret_codes):
+                              gsutil, make_executable, out_q, ret_codes):
   while True:
     input_sha1_sum, output_filename = q.get()
     if input_sha1_sum is None:
@@ -185,6 +186,10 @@
       out_q.put('%d> %s' % (thread_num, err))
       ret_codes.put((code, err))
 
+    # Mark executable if necessary.
+    if make_executable and not sys.platform.startswith('win'):

[Ryan Tseng, 2013/10/24 23:42:24]

We discussed this offline, but posting a comment for posterity.

We should store an executable flag in google storage as metadata:
https://developers.google.com/storage/docs/concepts-techniques
and set executable if the flag is present and active (using "gsutl ls -L")

Another change will need to be made in upload_to_google_storage to test for the
executable bit and set the metadata.

+      st = os.stat(output_filename)
+      os.chmod(output_filename, st.st_mode | stat.S_IEXEC)
 
 def printer_worker(output_queue):
   while True:
@@ -196,8 +201,8 @@
 
 
 def download_from_google_storage(
-    input_filename, base_url, gsutil, num_threads, directory, recursive,
-    force, output, ignore_errors, sha1_file):
+    input_filename, base_url, gsutil, make_executable, num_threads, directory,
+    recursive, force, output, ignore_errors, sha1_file):
   # Start up all the worker threads.
   all_threads = []
   download_start = time.time()
@@ -209,7 +214,7 @@
     t = threading.Thread(
         target=_downloader_worker_thread,
         args=[thread_num, work_queue, force, base_url,
-              gsutil, stdout_queue, ret_codes])
+              gsutil, make_executable, stdout_queue, ret_codes])
     t.daemon = True
     t.start()
     all_threads.append(t)
@@ -282,9 +287,23 @@
                     help='Alias for "gsutil config".  Run this if you want '
                          'to initialize your saved Google Storage '
                          'credentials.')
+  parser.add_option('', '--platform', 

[Ryan Tseng, 2013/10/24 23:42:24]

how about '-p'

+                    help='A regular expression that is compared against '
+                         'Python\'s sys.platform. If this option is specified, '
+                         'the download will happen only if there is a match.')
+  parser.add_option('', '--make_exec', action='store_true',

[Ryan Tseng, 2013/10/24 23:42:24]

how about '-x'

+                    help='Mark the file(s) downloaded executable on Posix '
+                         'systems.')
 
   (options, args) = parser.parse_args()
-  # First, make sure we can find a working instance of gsutil.
+
+  # Make sure we should run at all based on platform matching.
+  if options.platform:
+    platform_regexp = re.compile(options.platform)
+    if not platform_regexp.match(sys.platform):
+      return 0  # Skip rnuning on this platform

[Ryan Tseng, 2013/10/24 23:42:24]

/s/rnuning/running/

Also lets print a helpful message like "Platform <options.platform> does not
match <sys.platform>, exiting".

+
+  # Make sure we can find a working instance of gsutil.
   if os.path.exists(GSUTIL_DEFAULT_PATH):
     gsutil = Gsutil(GSUTIL_DEFAULT_PATH, boto_path=options.boto)
   else:
@@ -343,9 +362,9 @@
     return code
 
   return download_from_google_storage(
-      input_filename, base_url, gsutil, options.num_threads, options.directory,
-      options.recursive, options.force, options.output, options.ignore_errors,
-      options.sha1_file)
+      input_filename, base_url, gsutil, options.make_exec, options.num_threads,
+      options.directory, options.recursive, options.force, options.output,
+      options.ignore_errors, options.sha1_file)
 
 
 if __name__ == '__main__':