[telemetry] Add Google Cloud Storage utilities.

tools/telemetry/telemetry/page/cloud_storage.py

Index: tools/telemetry/telemetry/page/cloud_storage.py
diff --git a/tools/telemetry/telemetry/page/cloud_storage.py b/tools/telemetry/telemetry/page/cloud_storage.py
new file mode 100644
index 0000000000000000000000000000000000000000..b3078be9dfa4add1e51cc64e8c3119748c454b35
--- /dev/null
+++ b/tools/telemetry/telemetry/page/cloud_storage.py
@@ -0,0 +1,109 @@
+# Copyright (c) 2013 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+import ConfigParser
+import httplib2
+import json
+import os
+import sys
+
+from telemetry.core import util
+
+sys.path.append(os.path.join(
+    util.GetTelemetryDir(), 'third_party', 'oauth2client'))
+
+import oauth2client
+from oauth2client import client
+from oauth2client import file as oauth2_file
+from oauth2client import tools
+
+
+CLOUD_STORAGE_BASE_URL = 'https://www.googleapis.com/storage/v1beta2'
+CLOUD_STORAGE_INSERT_URL = 'https://www.googleapis.com/upload/storage/v1beta2'
+
+# Client ID and secret from gsutil.
+GSUTIL_CLIENT_ID = '909320924072.apps.googleusercontent.com'
+GSUTIL_CLIENT_NOTSOSECRET = 'p3RlpR10xMFh9ZXBS/ZNLYUu'

[tonyg, 2013/06/25 23:14:37]

What does NOTSOSECRET mean? Is it really safe to check this in?

[dtu, 2013/06/26 00:04:46]

Heh, yes, it has the same name in the gsutil code.
https://code.google.com/p/chromium/codesearch#chromium/tools/depot_tools/thir...

+
+# Read-write permissions for Cloud Storage.
+READ_WRITE_SCOPE = 'https://www.googleapis.com/auth/devstorage.read_write'
+
+# For non-web-based applications.
+OOB_REDIRECT_URI = 'urn:ietf:wg:oauth:2.0:oob'
+
+
+class CloudStorageError(Exception):
+  def __init__(self, content):
+    super(CloudStorageError, self).__init__(
+        json.loads(content)['error']['errors'][0]['message'])
+
+
+class CloudStorage(object):
+  """Simple wrapper for OAuth2 and Cloud Storage commands."""
+
+  def __init__(self, bucket='chromium-wpr'):
+    self.bucket = bucket
+    self._http = self._GetCredentials().authorize(httplib2.Http())
+
+  def _GetCredentials(self):
+    # Read credentials from gsutil/boto config.

[tonyg, 2013/06/25 23:14:37]

Will this work for developers running locally?

[dtu, 2013/06/26 00:04:46]

Yes, that's what this path is for. It uses the same config file as gsutil. I'm
more worried about the bots, because if none of these config files exist, it has
to open a browser to manually authenticate.

+    if 'BOTO_CONFIG' in os.environ:
+      paths = [os.environ['BOTO_CONFIG']]
+    else:
+      paths = ['/etc/boto.cfg', '~/.boto']
+    paths = [os.path.expanduser(os.path.expandvars(path)) for path in paths]
+
+    parser = ConfigParser.RawConfigParser()
+    parser.read(paths)
+
+    if (parser.has_section('Credentials') and
+        parser.has_option('Credentials', 'gs_oauth2_refresh_token')):
+      refresh_token = parser.get('Credentials', 'gs_oauth2_refresh_token')
+      return client.OAuth2Credentials(None, GSUTIL_CLIENT_ID,
+          GSUTIL_CLIENT_NOTSOSECRET, refresh_token, None,
+          oauth2client.GOOGLE_TOKEN_URI, None)
+
+    # Read credentials from our own credentials file.
+    storage = oauth2_file.Storage('.wpr_credentials')
+    credentials = storage.get()
+    if credentials:
+      return credentials
+
+    # Launch a web browser to authenticate.
+    flow = client.OAuth2WebServerFlow(client_id=GSUTIL_CLIENT_ID,
+                                      client_secret=GSUTIL_CLIENT_NOTSOSECRET,
+                                      scope=READ_WRITE_SCOPE,
+                                      redirect_uri=OOB_REDIRECT_URI)
+    return tools.run(flow, storage)
+
+  def Delete(self, name):
+    url = '%s/b/%s/o/%s' % (CLOUD_STORAGE_BASE_URL, self.bucket, name)
+    response, content = self._http.request(url, method='DELETE')
+    if response.status != 204:
+      raise CloudStorageError(content)
+    return content
+
+  def List(self):
+    url = '%s/b/%s/o' % (CLOUD_STORAGE_BASE_URL, self.bucket)
+    response, content = self._http.request(url)
+    if response.status != 200:
+      raise CloudStorageError(content)
+    bucket_info = json.loads(content)
+    return [item['name'] for item in bucket_info['items']]
+
+  def Get(self, name, file_path):
+    url = '%s/b/%s/o/%s?alt=media' % (CLOUD_STORAGE_BASE_URL, self.bucket, name)
+    response, content = self._http.request(url)
+    if response.status != 200:
+      raise CloudStorageError(content)
+    with open(file_path, 'w') as f:
+      f.write(content)
+
+  def Insert(self, name, file_path):
+    with open(file_path, 'r') as f:
+      data = f.read()
+    url = ('%s/b/%s/o?uploadType=media&name=%s' %
+           (CLOUD_STORAGE_INSERT_URL, self.bucket, name))
+    response, content = self._http.request(url, method='POST', body=data)
+    if response.status != 200:
+      raise CloudStorageError(content)