OLD | NEW |
(Empty) | |
| 1 #!/usr/bin/python |
| 2 |
| 3 """ |
| 4 Copyright 2014 Google Inc. |
| 5 |
| 6 Use of this source code is governed by a BSD-style license that can be |
| 7 found in the LICENSE file. |
| 8 |
| 9 Utilities for accessing Google Cloud Storage. |
| 10 """ |
| 11 |
| 12 # System-level imports |
| 13 import os |
| 14 import posixpath |
| 15 import sys |
| 16 |
| 17 # Imports from third-party code |
| 18 TRUNK_DIRECTORY = os.path.abspath(os.path.join( |
| 19 os.path.dirname(__file__), os.pardir, os.pardir)) |
| 20 for import_subdir in ['google-api-python-client', 'httplib2', 'oauth2client', |
| 21 'uritemplate-py']: |
| 22 import_dirpath = os.path.join( |
| 23 TRUNK_DIRECTORY, 'third_party', 'externals', import_subdir) |
| 24 if import_dirpath not in sys.path: |
| 25 # We need to insert at the beginning of the path, to make sure that our |
| 26 # imported versions are favored over others that might be in the path. |
| 27 # Also, the google-api-python-client checkout contains an empty |
| 28 # oauth2client directory, which will confuse things unless we insert |
| 29 # our checked-out oauth2client in front of it in the path. |
| 30 sys.path.insert(0, import_dirpath) |
| 31 try: |
| 32 from googleapiclient.discovery import build as build_service |
| 33 except ImportError: |
| 34 # We should not require any googleapiclient dependencies to be |
| 35 # installed at a system level, but in the meanwhile, if developers run into |
| 36 # trouble they can install those system-level dependencies to get unblocked. |
| 37 print ('We should not require any googleapiclient dependencies to be ' |
| 38 'installed at a system level, but it seems like some are missing. ' |
| 39 'Please install google-api-python-client to get those dependencies; ' |
| 40 'directions can be found at https://developers.google.com/' |
| 41 'api-client-library/python/start/installation . ' |
| 42 'More details in http://skbug.com/2641 ') |
| 43 raise |
| 44 |
| 45 # Local imports |
| 46 import url_utils |
| 47 |
| 48 |
| 49 def download_file(source_bucket, source_path, dest_path, |
| 50 create_subdirs_if_needed=False): |
| 51 """ Downloads a single file from Google Cloud Storage to local disk. |
| 52 |
| 53 Args: |
| 54 source_bucket: GCS bucket to download the file from |
| 55 source_path: full path (Posix-style) within that bucket |
| 56 dest_path: full path (local-OS-style) on local disk to copy the file to |
| 57 create_subdirs_if_needed: boolean; whether to create subdirectories as |
| 58 needed to create dest_path |
| 59 """ |
| 60 source_http_url = posixpath.join( |
| 61 'http://storage.googleapis.com', source_bucket, source_path) |
| 62 url_utils.copy_contents(source_url=source_http_url, dest_path=dest_path, |
| 63 create_subdirs_if_needed=create_subdirs_if_needed) |
| 64 |
| 65 |
| 66 def list_bucket_contents(bucket, subdir=None): |
| 67 """ Returns files in the Google Cloud Storage bucket as a (dirs, files) tuple. |
| 68 |
| 69 Uses the API documented at |
| 70 https://developers.google.com/storage/docs/json_api/v1/objects/list |
| 71 |
| 72 Args: |
| 73 bucket: name of the Google Storage bucket |
| 74 subdir: directory within the bucket to list, or None for root directory |
| 75 """ |
| 76 # The GCS command relies on the subdir name (if any) ending with a slash. |
| 77 if subdir and not subdir.endswith('/'): |
| 78 subdir += '/' |
| 79 subdir_length = len(subdir) if subdir else 0 |
| 80 |
| 81 storage = build_service('storage', 'v1') |
| 82 command = storage.objects().list( |
| 83 bucket=bucket, delimiter='/', fields='items(name),prefixes', |
| 84 prefix=subdir) |
| 85 results = command.execute() |
| 86 |
| 87 # The GCS command returned two subdicts: |
| 88 # prefixes: the full path of every directory within subdir, with trailing '/' |
| 89 # items: property dict for each file object within subdir |
| 90 # (including 'name', which is full path of the object) |
| 91 dirs = [] |
| 92 for dir_fullpath in results.get('prefixes', []): |
| 93 dir_basename = dir_fullpath[subdir_length:] |
| 94 dirs.append(dir_basename[:-1]) # strip trailing slash |
| 95 files = [] |
| 96 for file_properties in results.get('items', []): |
| 97 file_fullpath = file_properties['name'] |
| 98 file_basename = file_fullpath[subdir_length:] |
| 99 files.append(file_basename) |
| 100 return (dirs, files) |
OLD | NEW |