Add a platform filtering and executable setting abilities to download_from_google_storage.

download_from_google_storage.py

 #!/usr/bin/env python
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Download files from Google Storage based on SHA1 sums."""
 
 
 import hashlib
 import optparse
 import os
 import Queue
 import re
+import stat
 import sys
 import threading
 import time
 
 import subprocess2
 
 
 GSUTIL_DEFAULT_PATH = os.path.join(
     os.path.dirname(os.path.abspath(__file__)),
     'third_party', 'gsutil', 'gsutil')
@@ skipping 127 matching lines @@
               (sha1_match.groups(1)[0], full_path.replace('.sha1', '')))
           work_queue_size += 1
         else:
           if not ignore_errors:
             raise InvalidFileError('No sha1 sum found in %s.' % filename)
           print >> sys.stderr, 'No sha1 sum found in %s.' % filename
   return work_queue_size
 
 
 def _downloader_worker_thread(thread_num, q, force, base_url,
-                              gsutil, out_q, ret_codes):
+                              gsutil, make_executable, out_q, ret_codes):
   while True:
     input_sha1_sum, output_filename = q.get()
     if input_sha1_sum is None:
       return
     if os.path.exists(output_filename) and not force:
       if get_sha1(output_filename) == input_sha1_sum:
         out_q.put(
             '%d> File %s exists and SHA1 matches. Skipping.' % (
                 thread_num, output_filename))
         continue
     # Check if file exists.
     file_url = '%s/%s' % (base_url, input_sha1_sum)
     if gsutil.check_call('ls', file_url)[0] != 0:
       out_q.put('%d> File %s for %s does not exist, skipping.' % (
           thread_num, file_url, output_filename))
       ret_codes.put((1, 'File %s for %s does not exist.' % (
           file_url, output_filename)))
       continue
     # Fetch the file.
     out_q.put('%d> Downloading %s...' % (
         thread_num, output_filename))
     code, _, err = gsutil.check_call('cp', '-q', file_url, output_filename)
     if code != 0:
       out_q.put('%d> %s' % (thread_num, err))
       ret_codes.put((code, err))
 
+    # Mark executable if necessary.
+    if make_executable and not sys.platform.startswith('win'):

[Ryan Tseng, 2013/10/24 23:42:24]

We discussed this offline, but posting a comment for posterity.

We should store an executable flag in google storage as metadata:
https://developers.google.com/storage/docs/concepts-techniques
and set executable if the flag is present and active (using "gsutl ls -L")

Another change will need to be made in upload_to_google_storage to test for the
executable bit and set the metadata.

+      st = os.stat(output_filename)
+      os.chmod(output_filename, st.st_mode | stat.S_IEXEC)
 
 def printer_worker(output_queue):
   while True:
     line = output_queue.get()
     # Its plausible we want to print empty lines.
     if line is None:
       break
     print line
 
 
 def download_from_google_storage(
-    input_filename, base_url, gsutil, num_threads, directory, recursive,
-    force, output, ignore_errors, sha1_file):
+    input_filename, base_url, gsutil, make_executable, num_threads, directory,
+    recursive, force, output, ignore_errors, sha1_file):
   # Start up all the worker threads.
   all_threads = []
   download_start = time.time()
   stdout_queue = Queue.Queue()
   work_queue = Queue.Queue()
   ret_codes = Queue.Queue()
   ret_codes.put((0, None))
   for thread_num in range(num_threads):
     t = threading.Thread(
         target=_downloader_worker_thread,
         args=[thread_num, work_queue, force, base_url,
-              gsutil, stdout_queue, ret_codes])
+              gsutil, make_executable, stdout_queue, ret_codes])
     t.daemon = True
     t.start()
     all_threads.append(t)
   printer_thread = threading.Thread(target=printer_worker, args=[stdout_queue])
   printer_thread.daemon = True
   printer_thread.start()
 
   # Enumerate our work queue.
   work_queue_size = enumerate_work_queue(
       input_filename, work_queue, directory, recursive,
@@ skipping 52 matching lines @@
   parser.add_option('-d', '--directory', action='store_true',
                     help='The target is a directory.  '
                          'Cannot be used with -s/--sha1_file.')
   parser.add_option('-s', '--sha1_file', action='store_true',
                     help='The target is a file containing a sha1 sum.  '
                          'Cannot be used with -d/--directory.')
   parser.add_option('-g', '--config', action='store_true',
                     help='Alias for "gsutil config".  Run this if you want '
                          'to initialize your saved Google Storage '
                          'credentials.')
+  parser.add_option('', '--platform', 

[Ryan Tseng, 2013/10/24 23:42:24]

how about '-p'

+                    help='A regular expression that is compared against '
+                         'Python\'s sys.platform. If this option is specified, '
+                         'the download will happen only if there is a match.')
+  parser.add_option('', '--make_exec', action='store_true',

[Ryan Tseng, 2013/10/24 23:42:24]

how about '-x'

+                    help='Mark the file(s) downloaded executable on Posix '
+                         'systems.')
 
   (options, args) = parser.parse_args()
-  # First, make sure we can find a working instance of gsutil.
+
+  # Make sure we should run at all based on platform matching.
+  if options.platform:
+    platform_regexp = re.compile(options.platform)
+    if not platform_regexp.match(sys.platform):
+      return 0  # Skip rnuning on this platform

[Ryan Tseng, 2013/10/24 23:42:24]

/s/rnuning/running/

Also lets print a helpful message like "Platform <options.platform> does not
match <sys.platform>, exiting".

+
+  # Make sure we can find a working instance of gsutil.
   if os.path.exists(GSUTIL_DEFAULT_PATH):
     gsutil = Gsutil(GSUTIL_DEFAULT_PATH, boto_path=options.boto)
   else:
     gsutil = None
     for path in os.environ["PATH"].split(os.pathsep):
       if os.path.exists(path) and 'gsutil' in os.listdir(path):
         gsutil = Gsutil(os.path.join(path, 'gsutil'), boto_path=options.boto)
     if not gsutil:
       parser.error('gsutil not found in %s, bad depot_tools checkout?' %
                   GSUTIL_DEFAULT_PATH)
@@ skipping 38 matching lines @@
     if os.path.exists(options.output):
       parser.error('Output file %s exists and --no_resume is specified.'
                    % options.output)
 
   # Check we have a valid bucket with valid permissions.
   base_url, code = check_bucket_permissions(options.bucket, gsutil)
   if code:
     return code
 
   return download_from_google_storage(
-      input_filename, base_url, gsutil, options.num_threads, options.directory,
-      options.recursive, options.force, options.output, options.ignore_errors,
-      options.sha1_file)
+      input_filename, base_url, gsutil, options.make_exec, options.num_threads,
+      options.directory, options.recursive, options.force, options.output,
+      options.ignore_errors, options.sha1_file)
 
 
 if __name__ == '__main__':
   sys.exit(main(sys.argv))