| OLD | NEW |
| (Empty) |
| 1 #!/usr/bin/python | |
| 2 | |
| 3 """ | |
| 4 Copyright 2014 Google Inc. | |
| 5 | |
| 6 Use of this source code is governed by a BSD-style license that can be | |
| 7 found in the LICENSE file. | |
| 8 | |
| 9 Utilities for accessing Google Cloud Storage. | |
| 10 | |
| 11 TODO(epoger): move this into tools/utils for broader use? | |
| 12 """ | |
| 13 | |
| 14 # System-level imports | |
| 15 import os | |
| 16 import posixpath | |
| 17 import sys | |
| 18 | |
| 19 # Imports from third-party code | |
| 20 TRUNK_DIRECTORY = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) | |
| 21 APICLIENT_DIRECTORY = os.path.join( | |
| 22 TRUNK_DIRECTORY, 'third_party', 'externals', 'google-api-python-client') | |
| 23 if APICLIENT_DIRECTORY not in sys.path: | |
| 24 sys.path.append(APICLIENT_DIRECTORY) | |
| 25 from googleapiclient.discovery import build as build_service | |
| 26 | |
| 27 # Local imports | |
| 28 import url_utils | |
| 29 | |
| 30 | |
| 31 def download_file(source_bucket, source_path, dest_path, | |
| 32 create_subdirs_if_needed=False): | |
| 33 """ Downloads a single file from Google Cloud Storage to local disk. | |
| 34 | |
| 35 Args: | |
| 36 source_bucket: GCS bucket to download the file from | |
| 37 source_path: full path (Posix-style) within that bucket | |
| 38 dest_path: full path (local-OS-style) on local disk to copy the file to | |
| 39 create_subdirs_if_needed: boolean; whether to create subdirectories as | |
| 40 needed to create dest_path | |
| 41 """ | |
| 42 source_http_url = posixpath.join( | |
| 43 'http://storage.googleapis.com', source_bucket, source_path) | |
| 44 url_utils.copy_contents(source_url=source_http_url, dest_path=dest_path, | |
| 45 create_subdirs_if_needed=create_subdirs_if_needed) | |
| 46 | |
| 47 | |
| 48 def list_bucket_contents(bucket, subdir=None): | |
| 49 """ Returns files in the Google Cloud Storage bucket as a (dirs, files) tuple. | |
| 50 | |
| 51 Uses the API documented at | |
| 52 https://developers.google.com/storage/docs/json_api/v1/objects/list | |
| 53 | |
| 54 Args: | |
| 55 bucket: name of the Google Storage bucket | |
| 56 subdir: directory within the bucket to list, or None for root directory | |
| 57 """ | |
| 58 # The GCS command relies on the subdir name (if any) ending with a slash. | |
| 59 if subdir and not subdir.endswith('/'): | |
| 60 subdir += '/' | |
| 61 subdir_length = len(subdir) if subdir else 0 | |
| 62 | |
| 63 storage = build_service('storage', 'v1') | |
| 64 command = storage.objects().list( | |
| 65 bucket=bucket, delimiter='/', fields='items(name),prefixes', | |
| 66 prefix=subdir) | |
| 67 results = command.execute() | |
| 68 | |
| 69 # The GCS command returned two subdicts: | |
| 70 # prefixes: the full path of every directory within subdir, with trailing '/' | |
| 71 # items: property dict for each file object within subdir | |
| 72 # (including 'name', which is full path of the object) | |
| 73 dirs = [] | |
| 74 for dir_fullpath in results.get('prefixes', []): | |
| 75 dir_basename = dir_fullpath[subdir_length:] | |
| 76 dirs.append(dir_basename[:-1]) # strip trailing slash | |
| 77 files = [] | |
| 78 for file_properties in results.get('items', []): | |
| 79 file_fullpath = file_properties['name'] | |
| 80 file_basename = file_fullpath[subdir_length:] | |
| 81 files.append(file_basename) | |
| 82 return (dirs, files) | |
| OLD | NEW |