Monitoring proxy for time series data

appengine/chrome_infra_mon_proxy/vm_module.py

Index: appengine/chrome_infra_mon_proxy/vm_module.py
diff --git a/appengine/chrome_infra_mon_proxy/vm_module.py b/appengine/chrome_infra_mon_proxy/vm_module.py
new file mode 100644
index 0000000000000000000000000000000000000000..57a390b7e603af49f97804240b57655ab96f4f3a
--- /dev/null
+++ b/appengine/chrome_infra_mon_proxy/vm_module.py
@@ -0,0 +1,94 @@
+# Copyright 2015 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import googleapiclient.http
+import httplib2
+import json
+import logging
+import os
+import traceback
+import webapp2
+
+from google.appengine.api import app_identity
+from google.appengine.ext import ndb
+
+import common
+
+
+def _get_config_data():
+  data_entity = common.MonAcqData.get_by_id(common.CONFIG_DATA_KEY)
+  logging.info('get_config_data(): entity = %r', data_entity)
+  if not data_entity:
+    return None
+  return data_entity.to_dict()
+
+
+def _get_credentials(credentials_dict, scopes):
+  """Obtain Aquisition API credentials as Credentials object."""
+  from oauth2client.client import SignedJwtAssertionCredentials
+
+  # Ideally, we should have loaded credentials with GoogleCredentials.
+  # However, it insists to load only from a file. So, here's a hack.
+  return SignedJwtAssertionCredentials(
+      service_account_name=credentials_dict['client_email'],
+      private_key=credentials_dict['private_key'],
+      scope=scopes,
+      # Extra **kwargs, just in case.
+      service_account_id=credentials_dict['client_id'],
+      private_key_id=credentials_dict['private_key_id'],
+  )
+
+
+class VMHandler(webapp2.RequestHandler):
+
+  def requester_is_me(self):
+    requester_id = self.request.headers.get('X-Appengine-Inbound-Appid')
+    return requester_id == app_identity.get_application_id()
+
+  def requester_is_task(self):

[agable, 2015/04/23 19:29:37]

This doesn't return a boolean, as the name and usage suggest.

[Sergey Berezin (google), 2015/04/24 19:09:52]

Added bool(...) cast. It is indeed supposed to be a predicate, and worked in
python as is.

[agable, 2015/04/27 20:51:07]

Yeah I know it worked, I just also know that because it didn't actually return a
boolean someday someone would abuse it :P

+    return self.request.headers.get('X-AppEngine-TaskName')
+
+  def post(self):
+    authorized = (
+        common.is_development_server() or
+        self.requester_is_task() or self.requester_is_me())
+    if not authorized:
+      self.abort(403)
+    data = _get_config_data()
+    if not data:
+      self.abort_admin_error('Endpoint data is not set')
+    if not all(f in data for f in ['credentials', 'url']):

[agable, 2015/04/23 19:29:37]

We do checking here to make sure the admin data contains credentials, but we
never do checking on the credentials to make sure they contain the required
fields (we'll just IndexError in _get_credentials if the json is wrong).

[Sergey Berezin (google), 2015/04/24 19:09:52]

Added a try: except around _get_credentials, and print an error log with useful
info if this happens.

+      self.abort_admin_error('Missing required fields: credentials, url')
+
+    url = data['url']
+    http_auth = httplib2.Http()
+    credentials = _get_credentials(data['credentials'], data['scopes'])
+    http_auth = credentials.authorize(http_auth)
+    def callback(response, _content):
+      success = 200 <= response.status and response.status <= 299

[agable, 2015/04/23 19:29:37]

Dammit, I really miss requests.codes.ok.

[Sergey Berezin (google), 2015/04/24 19:09:52]

Acknowledged.

+      if not success:
+        logging.error('Failed to send data to %s: %d %s',
+                      url, response.status, response.reason)
+    # Important: set content-type to binary, otherwise httplib2 mangles it.
+    data.setdefault('headers', {}).update({
+        'content-length': str(len(self.request.body)),
+        'content-type': 'application/x-protobuf',
+    })
+    request = googleapiclient.http.HttpRequest(
+        http_auth, callback, url, method='POST', body=self.request.body,
+        headers=data['headers'])
+    logging.debug('Sending the payload.')
+    request.execute()
+    logging.debug('Done sending the payload.')

[agable, 2015/04/23 19:29:37]

Maybe note that execution of the callback is done synchronously before this line
is executed.

[Sergey Berezin (google), 2015/04/24 19:09:52]

Done.

+
+  def abort_admin_error(self, message):
+      logging.error('%s; please visit https://%s/admin/',
+                    message, app_identity.get_default_version_hostname())
+      self.abort(500)
+
+
+logging.basicConfig(level=logging.DEBUG)
+app = webapp2.WSGIApplication([
+    ('/vm.*', VMHandler),
+    ], debug=True)