OLD | NEW |
---|---|
(Empty) | |
1 #!/usr/bin/python | |
borenet
2014/07/10 18:06:16
I guess this is pure copy/paste?
epoger
2014/07/10 18:17:40
Yup, sorry, copied over from https://skia.googleso
| |
2 | |
3 """ | |
4 Copyright 2014 Google Inc. | |
5 | |
6 Use of this source code is governed by a BSD-style license that can be | |
7 found in the LICENSE file. | |
8 | |
9 Utilities for accessing Google Cloud Storage. | |
10 """ | |
11 | |
12 # System-level imports | |
13 import os | |
14 import posixpath | |
15 import sys | |
16 | |
17 # Imports from third-party code | |
18 TRUNK_DIRECTORY = os.path.abspath(os.path.join( | |
19 os.path.dirname(__file__), os.pardir, os.pardir)) | |
20 for import_subdir in ['google-api-python-client', 'httplib2', 'oauth2client', | |
21 'uritemplate-py']: | |
22 import_dirpath = os.path.join( | |
23 TRUNK_DIRECTORY, 'third_party', 'externals', import_subdir) | |
24 if import_dirpath not in sys.path: | |
25 # We need to insert at the beginning of the path, to make sure that our | |
26 # imported versions are favored over others that might be in the path. | |
27 # Also, the google-api-python-client checkout contains an empty | |
28 # oauth2client directory, which will confuse things unless we insert | |
29 # our checked-out oauth2client in front of it in the path. | |
30 sys.path.insert(0, import_dirpath) | |
31 try: | |
32 from googleapiclient.discovery import build as build_service | |
33 except ImportError: | |
34 # We should not require any googleapiclient dependencies to be | |
35 # installed at a system level, but in the meanwhile, if developers run into | |
36 # trouble they can install those system-level dependencies to get unblocked. | |
37 print ('We should not require any googleapiclient dependencies to be ' | |
38 'installed at a system level, but it seems like some are missing. ' | |
39 'Please install google-api-python-client to get those dependencies; ' | |
40 'directions can be found at https://developers.google.com/' | |
41 'api-client-library/python/start/installation . ' | |
42 'More details in http://skbug.com/2641 ') | |
43 raise | |
44 | |
45 # Local imports | |
46 import url_utils | |
47 | |
48 | |
49 def download_file(source_bucket, source_path, dest_path, | |
50 create_subdirs_if_needed=False): | |
51 """ Downloads a single file from Google Cloud Storage to local disk. | |
52 | |
53 Args: | |
54 source_bucket: GCS bucket to download the file from | |
55 source_path: full path (Posix-style) within that bucket | |
56 dest_path: full path (local-OS-style) on local disk to copy the file to | |
57 create_subdirs_if_needed: boolean; whether to create subdirectories as | |
58 needed to create dest_path | |
59 """ | |
60 source_http_url = posixpath.join( | |
61 'http://storage.googleapis.com', source_bucket, source_path) | |
62 url_utils.copy_contents(source_url=source_http_url, dest_path=dest_path, | |
63 create_subdirs_if_needed=create_subdirs_if_needed) | |
64 | |
65 | |
66 def list_bucket_contents(bucket, subdir=None): | |
67 """ Returns files in the Google Cloud Storage bucket as a (dirs, files) tuple. | |
68 | |
69 Uses the API documented at | |
70 https://developers.google.com/storage/docs/json_api/v1/objects/list | |
71 | |
72 Args: | |
73 bucket: name of the Google Storage bucket | |
74 subdir: directory within the bucket to list, or None for root directory | |
75 """ | |
76 # The GCS command relies on the subdir name (if any) ending with a slash. | |
77 if subdir and not subdir.endswith('/'): | |
78 subdir += '/' | |
79 subdir_length = len(subdir) if subdir else 0 | |
80 | |
81 storage = build_service('storage', 'v1') | |
82 command = storage.objects().list( | |
83 bucket=bucket, delimiter='/', fields='items(name),prefixes', | |
84 prefix=subdir) | |
85 results = command.execute() | |
86 | |
87 # The GCS command returned two subdicts: | |
88 # prefixes: the full path of every directory within subdir, with trailing '/' | |
89 # items: property dict for each file object within subdir | |
90 # (including 'name', which is full path of the object) | |
91 dirs = [] | |
92 for dir_fullpath in results.get('prefixes', []): | |
93 dir_basename = dir_fullpath[subdir_length:] | |
94 dirs.append(dir_basename[:-1]) # strip trailing slash | |
95 files = [] | |
96 for file_properties in results.get('items', []): | |
97 file_fullpath = file_properties['name'] | |
98 file_basename = file_fullpath[subdir_length:] | |
99 files.append(file_basename) | |
100 return (dirs, files) | |
OLD | NEW |