Index: tools/pyutils/gs_utils.py |
diff --git a/tools/pyutils/gs_utils.py b/tools/pyutils/gs_utils.py |
deleted file mode 100755 |
index 745276e693d1ef342b55ded02f8f5140caa81fa9..0000000000000000000000000000000000000000 |
--- a/tools/pyutils/gs_utils.py |
+++ /dev/null |
@@ -1,82 +0,0 @@ |
-#!/usr/bin/python |
- |
-""" |
-Copyright 2014 Google Inc. |
- |
-Use of this source code is governed by a BSD-style license that can be |
-found in the LICENSE file. |
- |
-Utilities for accessing Google Cloud Storage. |
- |
-TODO(epoger): move this into tools/utils for broader use? |
-""" |
- |
-# System-level imports |
-import os |
-import posixpath |
-import sys |
- |
-# Imports from third-party code |
-TRUNK_DIRECTORY = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) |
-APICLIENT_DIRECTORY = os.path.join( |
- TRUNK_DIRECTORY, 'third_party', 'externals', 'google-api-python-client') |
-if APICLIENT_DIRECTORY not in sys.path: |
- sys.path.append(APICLIENT_DIRECTORY) |
-from googleapiclient.discovery import build as build_service |
- |
-# Local imports |
-import url_utils |
- |
- |
-def download_file(source_bucket, source_path, dest_path, |
- create_subdirs_if_needed=False): |
- """ Downloads a single file from Google Cloud Storage to local disk. |
- |
- Args: |
- source_bucket: GCS bucket to download the file from |
- source_path: full path (Posix-style) within that bucket |
- dest_path: full path (local-OS-style) on local disk to copy the file to |
- create_subdirs_if_needed: boolean; whether to create subdirectories as |
- needed to create dest_path |
- """ |
- source_http_url = posixpath.join( |
- 'http://storage.googleapis.com', source_bucket, source_path) |
- url_utils.copy_contents(source_url=source_http_url, dest_path=dest_path, |
- create_subdirs_if_needed=create_subdirs_if_needed) |
- |
- |
-def list_bucket_contents(bucket, subdir=None): |
- """ Returns files in the Google Cloud Storage bucket as a (dirs, files) tuple. |
- |
- Uses the API documented at |
- https://developers.google.com/storage/docs/json_api/v1/objects/list |
- |
- Args: |
- bucket: name of the Google Storage bucket |
- subdir: directory within the bucket to list, or None for root directory |
- """ |
- # The GCS command relies on the subdir name (if any) ending with a slash. |
- if subdir and not subdir.endswith('/'): |
- subdir += '/' |
- subdir_length = len(subdir) if subdir else 0 |
- |
- storage = build_service('storage', 'v1') |
- command = storage.objects().list( |
- bucket=bucket, delimiter='/', fields='items(name),prefixes', |
- prefix=subdir) |
- results = command.execute() |
- |
- # The GCS command returned two subdicts: |
- # prefixes: the full path of every directory within subdir, with trailing '/' |
- # items: property dict for each file object within subdir |
- # (including 'name', which is full path of the object) |
- dirs = [] |
- for dir_fullpath in results.get('prefixes', []): |
- dir_basename = dir_fullpath[subdir_length:] |
- dirs.append(dir_basename[:-1]) # strip trailing slash |
- files = [] |
- for file_properties in results.get('items', []): |
- file_fullpath = file_properties['name'] |
- file_basename = file_fullpath[subdir_length:] |
- files.append(file_basename) |
- return (dirs, files) |