| OLD | NEW | 
|---|
|  | (Empty) | 
| 1 #!/usr/bin/python |  | 
| 2 |  | 
| 3 """ |  | 
| 4 Copyright 2014 Google Inc. |  | 
| 5 |  | 
| 6 Use of this source code is governed by a BSD-style license that can be |  | 
| 7 found in the LICENSE file. |  | 
| 8 |  | 
| 9 Utilities for accessing Google Cloud Storage. |  | 
| 10 |  | 
| 11 TODO(epoger): move this into tools/utils for broader use? |  | 
| 12 """ |  | 
| 13 |  | 
| 14 # System-level imports |  | 
| 15 import os |  | 
| 16 import posixpath |  | 
| 17 import sys |  | 
| 18 |  | 
| 19 # Imports from third-party code |  | 
| 20 TRUNK_DIRECTORY = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) |  | 
| 21 APICLIENT_DIRECTORY = os.path.join( |  | 
| 22     TRUNK_DIRECTORY, 'third_party', 'externals', 'google-api-python-client') |  | 
| 23 if APICLIENT_DIRECTORY not in sys.path: |  | 
| 24   sys.path.append(APICLIENT_DIRECTORY) |  | 
| 25 from googleapiclient.discovery import build as build_service |  | 
| 26 |  | 
| 27 # Local imports |  | 
| 28 import url_utils |  | 
| 29 |  | 
| 30 |  | 
| 31 def download_file(source_bucket, source_path, dest_path, |  | 
| 32                   create_subdirs_if_needed=False): |  | 
| 33   """ Downloads a single file from Google Cloud Storage to local disk. |  | 
| 34 |  | 
| 35   Args: |  | 
| 36     source_bucket: GCS bucket to download the file from |  | 
| 37     source_path: full path (Posix-style) within that bucket |  | 
| 38     dest_path: full path (local-OS-style) on local disk to copy the file to |  | 
| 39     create_subdirs_if_needed: boolean; whether to create subdirectories as |  | 
| 40         needed to create dest_path |  | 
| 41   """ |  | 
| 42   source_http_url = posixpath.join( |  | 
| 43       'http://storage.googleapis.com', source_bucket, source_path) |  | 
| 44   url_utils.copy_contents(source_url=source_http_url, dest_path=dest_path, |  | 
| 45                           create_subdirs_if_needed=create_subdirs_if_needed) |  | 
| 46 |  | 
| 47 |  | 
| 48 def list_bucket_contents(bucket, subdir=None): |  | 
| 49   """ Returns files in the Google Cloud Storage bucket as a (dirs, files) tuple. |  | 
| 50 |  | 
| 51   Uses the API documented at |  | 
| 52   https://developers.google.com/storage/docs/json_api/v1/objects/list |  | 
| 53 |  | 
| 54   Args: |  | 
| 55     bucket: name of the Google Storage bucket |  | 
| 56     subdir: directory within the bucket to list, or None for root directory |  | 
| 57   """ |  | 
| 58   # The GCS command relies on the subdir name (if any) ending with a slash. |  | 
| 59   if subdir and not subdir.endswith('/'): |  | 
| 60     subdir += '/' |  | 
| 61   subdir_length = len(subdir) if subdir else 0 |  | 
| 62 |  | 
| 63   storage = build_service('storage', 'v1') |  | 
| 64   command = storage.objects().list( |  | 
| 65       bucket=bucket, delimiter='/', fields='items(name),prefixes', |  | 
| 66       prefix=subdir) |  | 
| 67   results = command.execute() |  | 
| 68 |  | 
| 69   # The GCS command returned two subdicts: |  | 
| 70   # prefixes: the full path of every directory within subdir, with trailing '/' |  | 
| 71   # items: property dict for each file object within subdir |  | 
| 72   #        (including 'name', which is full path of the object) |  | 
| 73   dirs = [] |  | 
| 74   for dir_fullpath in results.get('prefixes', []): |  | 
| 75     dir_basename = dir_fullpath[subdir_length:] |  | 
| 76     dirs.append(dir_basename[:-1])  # strip trailing slash |  | 
| 77   files = [] |  | 
| 78   for file_properties in results.get('items', []): |  | 
| 79     file_fullpath = file_properties['name'] |  | 
| 80     file_basename = file_fullpath[subdir_length:] |  | 
| 81     files.append(file_basename) |  | 
| 82   return (dirs, files) |  | 
| OLD | NEW | 
|---|