OLD | NEW |
| (Empty) |
1 #!/usr/bin/python | |
2 | |
3 """ | |
4 Copyright 2014 Google Inc. | |
5 | |
6 Use of this source code is governed by a BSD-style license that can be | |
7 found in the LICENSE file. | |
8 | |
9 Utilities for accessing Google Cloud Storage. | |
10 """ | |
11 | |
12 # System-level imports | |
13 import os | |
14 import posixpath | |
15 import sys | |
16 | |
17 # Imports from third-party code | |
18 TRUNK_DIRECTORY = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) | |
19 for import_subdir in ['google-api-python-client', 'httplib2', 'oauth2client', | |
20 'uritemplate-py']: | |
21 import_dirpath = os.path.join( | |
22 TRUNK_DIRECTORY, 'third_party', 'externals', import_subdir) | |
23 if import_dirpath not in sys.path: | |
24 # We need to insert at the beginning of the path, to make sure that our | |
25 # imported versions are favored over others that might be in the path. | |
26 # Also, the google-api-python-client checkout contains an empty | |
27 # oauth2client directory, which will confuse things unless we insert | |
28 # our checked-out oauth2client in front of it in the path. | |
29 sys.path.insert(0, import_dirpath) | |
30 try: | |
31 from googleapiclient.discovery import build as build_service | |
32 except ImportError: | |
33 # TODO(epoger): We are moving toward not needing any dependencies to be | |
34 # installed at a system level, but in the meanwhile, if developers run into | |
35 # trouble they can install those system-level dependencies to get unblocked. | |
36 print ('Missing dependencies of google-api-python-client. Please install ' | |
37 'google-api-python-client to get those dependencies; directions ' | |
38 'can be found at https://developers.google.com/api-client-library/' | |
39 'python/start/installation . More details in http://skbug.com/2641 ') | |
40 raise | |
41 | |
42 # Local imports | |
43 import url_utils | |
44 | |
45 | |
46 def download_file(source_bucket, source_path, dest_path, | |
47 create_subdirs_if_needed=False): | |
48 """ Downloads a single file from Google Cloud Storage to local disk. | |
49 | |
50 Args: | |
51 source_bucket: GCS bucket to download the file from | |
52 source_path: full path (Posix-style) within that bucket | |
53 dest_path: full path (local-OS-style) on local disk to copy the file to | |
54 create_subdirs_if_needed: boolean; whether to create subdirectories as | |
55 needed to create dest_path | |
56 """ | |
57 source_http_url = posixpath.join( | |
58 'http://storage.googleapis.com', source_bucket, source_path) | |
59 url_utils.copy_contents(source_url=source_http_url, dest_path=dest_path, | |
60 create_subdirs_if_needed=create_subdirs_if_needed) | |
61 | |
62 | |
63 def list_bucket_contents(bucket, subdir=None): | |
64 """ Returns files in the Google Cloud Storage bucket as a (dirs, files) tuple. | |
65 | |
66 Uses the API documented at | |
67 https://developers.google.com/storage/docs/json_api/v1/objects/list | |
68 | |
69 Args: | |
70 bucket: name of the Google Storage bucket | |
71 subdir: directory within the bucket to list, or None for root directory | |
72 """ | |
73 # The GCS command relies on the subdir name (if any) ending with a slash. | |
74 if subdir and not subdir.endswith('/'): | |
75 subdir += '/' | |
76 subdir_length = len(subdir) if subdir else 0 | |
77 | |
78 storage = build_service('storage', 'v1') | |
79 command = storage.objects().list( | |
80 bucket=bucket, delimiter='/', fields='items(name),prefixes', | |
81 prefix=subdir) | |
82 results = command.execute() | |
83 | |
84 # The GCS command returned two subdicts: | |
85 # prefixes: the full path of every directory within subdir, with trailing '/' | |
86 # items: property dict for each file object within subdir | |
87 # (including 'name', which is full path of the object) | |
88 dirs = [] | |
89 for dir_fullpath in results.get('prefixes', []): | |
90 dir_basename = dir_fullpath[subdir_length:] | |
91 dirs.append(dir_basename[:-1]) # strip trailing slash | |
92 files = [] | |
93 for file_properties in results.get('items', []): | |
94 file_fullpath = file_properties['name'] | |
95 file_basename = file_fullpath[subdir_length:] | |
96 files.append(file_basename) | |
97 return (dirs, files) | |
OLD | NEW |