Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(288)

Side by Side Diff: tools/pyutils/gs_utils.py

Issue 317783004: Revert "rebaseline_server: download actual-results.json files from GCS instead of SVN" (Closed) Base URL: https://skia.googlesource.com/skia.git@master
Patch Set: Created 6 years, 6 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
« no previous file with comments | « tools/pyutils/__init__.py ('k') | tools/pyutils/url_utils.py » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
(Empty)
1 #!/usr/bin/python
2
3 """
4 Copyright 2014 Google Inc.
5
6 Use of this source code is governed by a BSD-style license that can be
7 found in the LICENSE file.
8
9 Utilities for accessing Google Cloud Storage.
10
11 TODO(epoger): move this into tools/utils for broader use?
12 """
13
14 # System-level imports
15 import os
16 import posixpath
17 import sys
18
19 # Imports from third-party code
20 TRUNK_DIRECTORY = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
21 APICLIENT_DIRECTORY = os.path.join(
22 TRUNK_DIRECTORY, 'third_party', 'externals', 'google-api-python-client')
23 if APICLIENT_DIRECTORY not in sys.path:
24 sys.path.append(APICLIENT_DIRECTORY)
25 from googleapiclient.discovery import build as build_service
26
27 # Local imports
28 import url_utils
29
30
31 def download_file(source_bucket, source_path, dest_path,
32 create_subdirs_if_needed=False):
33 """ Downloads a single file from Google Cloud Storage to local disk.
34
35 Args:
36 source_bucket: GCS bucket to download the file from
37 source_path: full path (Posix-style) within that bucket
38 dest_path: full path (local-OS-style) on local disk to copy the file to
39 create_subdirs_if_needed: boolean; whether to create subdirectories as
40 needed to create dest_path
41 """
42 source_http_url = posixpath.join(
43 'http://storage.googleapis.com', source_bucket, source_path)
44 url_utils.copy_contents(source_url=source_http_url, dest_path=dest_path,
45 create_subdirs_if_needed=create_subdirs_if_needed)
46
47
48 def list_bucket_contents(bucket, subdir=None):
49 """ Returns files in the Google Cloud Storage bucket as a (dirs, files) tuple.
50
51 Uses the API documented at
52 https://developers.google.com/storage/docs/json_api/v1/objects/list
53
54 Args:
55 bucket: name of the Google Storage bucket
56 subdir: directory within the bucket to list, or None for root directory
57 """
58 # The GCS command relies on the subdir name (if any) ending with a slash.
59 if subdir and not subdir.endswith('/'):
60 subdir += '/'
61 subdir_length = len(subdir) if subdir else 0
62
63 storage = build_service('storage', 'v1')
64 command = storage.objects().list(
65 bucket=bucket, delimiter='/', fields='items(name),prefixes',
66 prefix=subdir)
67 results = command.execute()
68
69 # The GCS command returned two subdicts:
70 # prefixes: the full path of every directory within subdir, with trailing '/'
71 # items: property dict for each file object within subdir
72 # (including 'name', which is full path of the object)
73 dirs = []
74 for dir_fullpath in results.get('prefixes', []):
75 dir_basename = dir_fullpath[subdir_length:]
76 dirs.append(dir_basename[:-1]) # strip trailing slash
77 files = []
78 for file_properties in results.get('items', []):
79 file_fullpath = file_properties['name']
80 file_basename = file_fullpath[subdir_length:]
81 files.append(file_basename)
82 return (dirs, files)
OLDNEW
« no previous file with comments | « tools/pyutils/__init__.py ('k') | tools/pyutils/url_utils.py » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698