OLD | NEW |
| (Empty) |
1 #!/usr/bin/python | |
2 | |
3 """ | |
4 Copyright 2014 Google Inc. | |
5 | |
6 Use of this source code is governed by a BSD-style license that can be | |
7 found in the LICENSE file. | |
8 | |
9 Utilities for accessing Google Cloud Storage. | |
10 | |
11 TODO(epoger): move this into tools/utils for broader use? | |
12 """ | |
13 | |
14 # System-level imports | |
15 import os | |
16 import posixpath | |
17 import sys | |
18 | |
19 # Imports from third-party code | |
20 TRUNK_DIRECTORY = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) | |
21 APICLIENT_DIRECTORY = os.path.join( | |
22 TRUNK_DIRECTORY, 'third_party', 'externals', 'google-api-python-client') | |
23 if APICLIENT_DIRECTORY not in sys.path: | |
24 sys.path.append(APICLIENT_DIRECTORY) | |
25 from googleapiclient.discovery import build as build_service | |
26 | |
27 # Local imports | |
28 import url_utils | |
29 | |
30 | |
31 def download_file(source_bucket, source_path, dest_path, | |
32 create_subdirs_if_needed=False): | |
33 """ Downloads a single file from Google Cloud Storage to local disk. | |
34 | |
35 Args: | |
36 source_bucket: GCS bucket to download the file from | |
37 source_path: full path (Posix-style) within that bucket | |
38 dest_path: full path (local-OS-style) on local disk to copy the file to | |
39 create_subdirs_if_needed: boolean; whether to create subdirectories as | |
40 needed to create dest_path | |
41 """ | |
42 source_http_url = posixpath.join( | |
43 'http://storage.googleapis.com', source_bucket, source_path) | |
44 url_utils.copy_contents(source_url=source_http_url, dest_path=dest_path, | |
45 create_subdirs_if_needed=create_subdirs_if_needed) | |
46 | |
47 | |
48 def list_bucket_contents(bucket, subdir=None): | |
49 """ Returns files in the Google Cloud Storage bucket as a (dirs, files) tuple. | |
50 | |
51 Uses the API documented at | |
52 https://developers.google.com/storage/docs/json_api/v1/objects/list | |
53 | |
54 Args: | |
55 bucket: name of the Google Storage bucket | |
56 subdir: directory within the bucket to list, or None for root directory | |
57 """ | |
58 # The GCS command relies on the subdir name (if any) ending with a slash. | |
59 if subdir and not subdir.endswith('/'): | |
60 subdir += '/' | |
61 subdir_length = len(subdir) if subdir else 0 | |
62 | |
63 storage = build_service('storage', 'v1') | |
64 command = storage.objects().list( | |
65 bucket=bucket, delimiter='/', fields='items(name),prefixes', | |
66 prefix=subdir) | |
67 results = command.execute() | |
68 | |
69 # The GCS command returned two subdicts: | |
70 # prefixes: the full path of every directory within subdir, with trailing '/' | |
71 # items: property dict for each file object within subdir | |
72 # (including 'name', which is full path of the object) | |
73 dirs = [] | |
74 for dir_fullpath in results.get('prefixes', []): | |
75 dir_basename = dir_fullpath[subdir_length:] | |
76 dirs.append(dir_basename[:-1]) # strip trailing slash | |
77 files = [] | |
78 for file_properties in results.get('items', []): | |
79 file_fullpath = file_properties['name'] | |
80 file_basename = file_fullpath[subdir_length:] | |
81 files.append(file_basename) | |
82 return (dirs, files) | |
OLD | NEW |