gerrit_util: Add GCE metadata server auth.

gerrit_util.py

 # Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """
 Utilities for requesting information for a gerrit server via https.
 
 https://gerrit-review.googlesource.com/Documentation/rest-api.html
 """
 
 import base64
 import httplib
 import json
 import logging
 import netrc
 import os
 import re
+import socket
 import stat
 import sys
 import time
 import urllib
+import urlparse
 from cStringIO import StringIO
 
-_netrc_file = '_netrc' if sys.platform.startswith('win') else '.netrc'
-_netrc_file = os.path.join(os.environ['HOME'], _netrc_file)
-try:
-  NETRC = netrc.netrc(_netrc_file)
-except IOError:
-  print >> sys.stderr, 'WARNING: Could not read netrc file %s' % _netrc_file
-  NETRC = netrc.netrc(os.devnull)
-except netrc.NetrcParseError as e:
-  _netrc_stat = os.stat(e.filename)
-  if _netrc_stat.st_mode & (stat.S_IRWXG | stat.S_IRWXO):
-    print >> sys.stderr, (
-        'WARNING: netrc file %s cannot be used because its file permissions '
-        'are insecure.  netrc file permissions should be 600.' % _netrc_file)
-  else:
-    print >> sys.stderr, ('ERROR: Cannot use netrc file %s due to a parsing '
-                          'error.' % _netrc_file)
-    raise
-  del _netrc_stat
-  NETRC = netrc.netrc(os.devnull)
-del _netrc_file
 
 LOGGER = logging.getLogger()
 TRY_LIMIT = 5
 
+
 # Controls the transport protocol used to communicate with gerrit.
 # This is parameterized primarily to enable GerritTestCase.
 GERRIT_PROTOCOL = 'https'
 
 
 class GerritError(Exception):
   """Exception class for errors commuicating with the gerrit-on-borg service."""
   def __init__(self, http_status, *args, **kwargs):
     super(GerritError, self).__init__(*args, **kwargs)
     self.http_status = http_status
@@ skipping 19 matching lines @@
     protocol = GERRIT_PROTOCOL
   if protocol == 'https':
     return httplib.HTTPSConnection
   elif protocol == 'http':
     return httplib.HTTPConnection
   else:
     raise RuntimeError(
         "Don't know how to work with protocol '%s'" % protocol)
 
 
+class Authenticator(object):
+  """Base authenticator class for authenticator implementations to subclass."""
+
+  def get_auth_header(self, host):
+    raise NotImplementedError()
+
+  @staticmethod
+  def get():
+    """Returns: (Authenticator) The identified Authenticator to use.
+
+    Probes the local system and its environment and identifies the
+    Authenticator instance to use.
+    """
+    if GceAuthenticator.is_gce():
+      return GceAuthenticator()
+    return NetrcAuthenticator()
+
+
+class NetrcAuthenticator(Authenticator):
+  """Authenticator implementation that uses ".netrc" for token.
+  """
+
+  def __init__(self):
+    self.netrc = self._get_netrc()
+
+  @staticmethod
+  def _get_netrc():
+    path = '_netrc' if sys.platform.startswith('win') else '.netrc'
+    path = os.path.join(os.environ['HOME'], path)
+    try:
+      return netrc.netrc(path)
+    except IOError:
+      print >> sys.stderr, 'WARNING: Could not read netrc file %s' % path
+      return netrc.netrc(os.devnull)
+    except netrc.NetrcParseError as e:
+      st = os.stat(e.path)
+      if st.st_mode & (stat.S_IRWXG | stat.S_IRWXO):
+        print >> sys.stderr, (
+            'WARNING: netrc file %s cannot be used because its file '
+            'permissions are insecure.  netrc file permissions should be '
+            '600.' % path)
+      else:
+        print >> sys.stderr, ('ERROR: Cannot use netrc file %s due to a '
+                              'parsing error.' % path)
+        raise
+      return netrc.netrc(os.devnull)
+
+  def get_auth_header(self, host):
+    auth = self.netrc.authenticators(host)
+    if auth:
+      return 'Basic %s' % (base64.b64encode('%s:%s' % (auth[0], auth[2])))
+    return None
+
+
+class GceAuthenticator(Authenticator):

[Sergiy Byelozyorov, 2015/09/22 11:49:49]

I recall writing this code. Can you please reference the source so that I can
compare? I couldn't find it myself:
https://code.google.com/p/chromium/codesearch#search/&q=GceAuthenticator&sq=p...

[dnj, 2015/09/25 17:40:55]

This is standalone, so I don't think referencing the Twisted one in "build/" is
useful here.

+  """Authenticator implementation that uses GCE metadata service for token.
+  """
+
+  _INFO_URL = 'http://metadata.google.internal'
+  _ACQUIRE_URL = ('http://metadata/computeMetadata/v1/instance/'
+                  'service-accounts/default/token')
+  _ACQUIRE_HEADERS = {"Metadata-Flavor": "Google"}
+
+  _cache_is_gce = None
+  _token_cache = None
+  _token_expiration = None
+
+  @classmethod
+  def is_gce(cls):
+    if cls._cache_is_gce is None:
+      cls._cache_is_gce = cls._test_is_gce()

[Sergiy Byelozyorov, 2015/09/25 15:36:20]

Please add 5 retries inside out outside this call. Metadata endpoint may flake
with 500 just like AppEngine does.

[dnj, 2015/09/25 17:40:55]

I'm fairly reluctant to do 5 retries in all circumstances, but I'll throw in
some logic to retry on 500s.

[Sergiy Byelozyorov, 2015/09/25 20:06:34]

If you think 5 is too much, feel free to reduce it to 3 or 2, but not retrying
at all is also not good. Also AFAIU this will run on bots, so users won't be
annoyed too much by this.

+    return cls._cache_is_gce
+
+  @classmethod
+  def _test_is_gce(cls):
+    # Based on https://cloud.google.com/compute/docs/metadata#runninggce
+    try:
+      resp = cls._get(cls._INFO_URL)
+    except socket.error:
+      # Could not resolve URL.
+      return False
+    return resp.getheader('Metadata-Flavor', None) == 'Google'
+
+  @staticmethod
+  def _get(url, **kwargs):
+    p = urlparse.urlparse(url)
+    c = GetConnectionClass(protocol=p.scheme)(p.netloc)
+    c.request('GET', url, **kwargs)
+    return c.getresponse()
+
+  @classmethod
+  def _get_token_dict(cls):
+    if cls._token_cache:
+      # If it expires within 25 seconds, refresh.
+      if cls._token_expiration > time.time() - 25:
+        return cls._token_cache

[Sergiy Byelozyorov, 2015/09/25 15:36:20]

You actually return the token only if it expires within the next 25 seconds...
and otherwise you always refresh it. Change: s/>/</

[dnj, 2015/09/25 17:40:55]

lol good catch

+
+    resp = cls._get(cls._ACQUIRE_URL, headers=cls._ACQUIRE_HEADERS)

[Sergiy Byelozyorov, 2015/09/25 15:36:20]

Please add 5 retries around this. Metadata endpoint may flake with 500 just like
AppEngine does.

[dnj, 2015/09/25 17:40:55]

Done.

+    if resp.status != httplib.OK:
+      return None
+    cls._token_cache = token_dict = json.load(resp)

[Sergiy Byelozyorov, 2015/09/25 15:36:20]

why not just assign to cls._token_cache and use that in the next statement?

[dnj, 2015/09/25 17:40:55]

It avoids one more dictionary getattr hit *shrug*.

Doesn't really matter though, so I'll simplify it since you think it's useful.

+    cls._token_expiration = token_dict['expires_in'] + time.time()
+    return cls._token_cache
+
+  def get_auth_header(self, _host):
+    token_dict = self._get_token_dict()
+    if not token_dict:
+      return None
+    return '%(token_type)s %(access_token)s' % token_dict
+
+
+
 def CreateHttpConn(host, path, reqtype='GET', headers=None, body=None):
   """Opens an https connection to a gerrit service, and sends a request."""
   headers = headers or {}
   bare_host = host.partition(':')[0]
-  auth = NETRC.authenticators(bare_host)
 
+  auth = Authenticator.get().get_auth_header(bare_host)
   if auth:
-    headers.setdefault('Authorization', 'Basic %s' % (
-        base64.b64encode('%s:%s' % (auth[0], auth[2]))))
+    headers.setdefault('Authorization', auth)
   else:
-    LOGGER.debug('No authorization found in netrc for %s.' % bare_host)
+    LOGGER.debug('No authorization found for %s.' % bare_host)
 
   if 'Authorization' in headers and not path.startswith('a/'):
     url = '/a/%s' % path
   else:
     url = '/%s' % path
 
   if body:
     body = json.JSONEncoder().encode(body)
     headers.setdefault('Content-Type', 'application/json')
   if LOGGER.isEnabledFor(logging.DEBUG):
@@ skipping 357 matching lines @@
         username = review.get('email', jmsg.get('name', ''))
         raise GerritError(200, 'Unable to set %s label for user "%s"'
                        ' on change %s.' % (label, username, change))
   jmsg = GetChangeCurrentRevision(host, change)
   if not jmsg:
     raise GerritError(
         200, 'Could not get review information for change "%s"' % change)
   elif jmsg[0]['current_revision'] != revision:
     raise GerritError(200, 'While resetting labels on change "%s", '
                    'a new patchset was uploaded.' % change)