OLD | NEW |
---|---|
(Empty) | |
1 #!/usr/bin/python | |
2 | |
3 """ | |
4 Copyright 2014 Google Inc. | |
5 | |
6 Use of this source code is governed by a BSD-style license that can be | |
7 found in the LICENSE file. | |
8 | |
9 Utilities for accessing Google Cloud Storage. | |
10 """ | |
11 | |
12 # System-level imports | |
13 import os | |
14 import posixpath | |
15 import sys | |
16 | |
17 # Imports from third-party code | |
18 TRUNK_DIRECTORY = os.path.abspath(os.path.join( | |
epoger
2014/07/10 17:27:40
This refers to the root directory of the common re
| |
19 os.path.dirname(__file__), os.pardir, os.pardir)) | |
20 for import_subdir in ['google-api-python-client', 'httplib2', 'oauth2client', | |
21 'uritemplate-py']: | |
22 import_dirpath = os.path.join( | |
23 TRUNK_DIRECTORY, 'third_party', 'externals', import_subdir) | |
24 print 'EPOGER %s' % import_dirpath | |
epoger
2014/07/10 17:27:40
Note to self: delete this line in the next patchse
| |
25 if import_dirpath not in sys.path: | |
26 # We need to insert at the beginning of the path, to make sure that our | |
27 # imported versions are favored over others that might be in the path. | |
28 # Also, the google-api-python-client checkout contains an empty | |
29 # oauth2client directory, which will confuse things unless we insert | |
30 # our checked-out oauth2client in front of it in the path. | |
31 sys.path.insert(0, import_dirpath) | |
32 try: | |
33 from googleapiclient.discovery import build as build_service | |
34 except ImportError: | |
35 # We should not require any googleapiclient dependencies to be | |
36 # installed at a system level, but in the meanwhile, if developers run into | |
37 # trouble they can install those system-level dependencies to get unblocked. | |
38 print ('We should not require any googleapiclient dependencies to be ' | |
39 'installed at a system level, but it seems like some are missing. ' | |
40 'Please install google-api-python-client to get those dependencies; ' | |
41 'directions can be found at https://developers.google.com/' | |
42 'api-client-library/python/start/installation . ' | |
43 'More details in http://skbug.com/2641 ') | |
44 raise | |
45 | |
46 # Local imports | |
47 import url_utils | |
48 | |
49 | |
50 def download_file(source_bucket, source_path, dest_path, | |
51 create_subdirs_if_needed=False): | |
52 """ Downloads a single file from Google Cloud Storage to local disk. | |
53 | |
54 Args: | |
55 source_bucket: GCS bucket to download the file from | |
56 source_path: full path (Posix-style) within that bucket | |
57 dest_path: full path (local-OS-style) on local disk to copy the file to | |
58 create_subdirs_if_needed: boolean; whether to create subdirectories as | |
59 needed to create dest_path | |
60 """ | |
61 source_http_url = posixpath.join( | |
62 'http://storage.googleapis.com', source_bucket, source_path) | |
63 url_utils.copy_contents(source_url=source_http_url, dest_path=dest_path, | |
64 create_subdirs_if_needed=create_subdirs_if_needed) | |
65 | |
66 | |
67 def list_bucket_contents(bucket, subdir=None): | |
68 """ Returns files in the Google Cloud Storage bucket as a (dirs, files) tuple. | |
69 | |
70 Uses the API documented at | |
71 https://developers.google.com/storage/docs/json_api/v1/objects/list | |
72 | |
73 Args: | |
74 bucket: name of the Google Storage bucket | |
75 subdir: directory within the bucket to list, or None for root directory | |
76 """ | |
77 # The GCS command relies on the subdir name (if any) ending with a slash. | |
78 if subdir and not subdir.endswith('/'): | |
79 subdir += '/' | |
80 subdir_length = len(subdir) if subdir else 0 | |
81 | |
82 storage = build_service('storage', 'v1') | |
83 command = storage.objects().list( | |
84 bucket=bucket, delimiter='/', fields='items(name),prefixes', | |
85 prefix=subdir) | |
86 results = command.execute() | |
87 | |
88 # The GCS command returned two subdicts: | |
89 # prefixes: the full path of every directory within subdir, with trailing '/' | |
90 # items: property dict for each file object within subdir | |
91 # (including 'name', which is full path of the object) | |
92 dirs = [] | |
93 for dir_fullpath in results.get('prefixes', []): | |
94 dir_basename = dir_fullpath[subdir_length:] | |
95 dirs.append(dir_basename[:-1]) # strip trailing slash | |
96 files = [] | |
97 for file_properties in results.get('items', []): | |
98 file_fullpath = file_properties['name'] | |
99 file_basename = file_fullpath[subdir_length:] | |
100 files.append(file_basename) | |
101 return (dirs, files) | |
OLD | NEW |