auth_service: Fetch revisions of all configs (for UI) at once.

appengine/auth_service/config.py

 # Copyright 2015 The LUCI Authors. All rights reserved.
 # Use of this source code is governed under the Apache License, Version 2.0
 # that can be found in the LICENSE file.
 
 """Adapter between config service client and the rest of auth_service.
 
 Basically a cron job that each minute refetches config files from config service
 and modifies auth service datastore state if anything changed.
 
 Following files are fetched:
@@ skipping 16 matching lines @@
 from google import protobuf
 from google.appengine.ext import ndb
 
 from components import config
 from components import datastore_utils
 from components import gitiles
 from components import utils
 from components.auth import ipaddr
 from components.auth import model
 from components.config import validation
-from components.config import validation_context
 
 from proto import config_pb2
 import importer
 
 
 # Config file revision number and where it came from.
 Revision = collections.namedtuple('Revision', ['revision', 'url'])
 
 
 class CannotLoadConfigError(Exception):
@@ skipping 13 matching lines @@
 
 
 def get_remote_url():
   """Returns URL of a config service if configured, to display in UI."""
   settings = config.ConfigSettings.cached()
   if settings and settings.service_hostname:
     return 'https://%s' % settings.service_hostname
   return None
 
 
-def get_config_revision(path):
+def get_revisions():
+  """Returns a mapping {config file name => Revision instance or None}."""
+  futures = {p: _get_config_revision_async(p) for p in _CONFIG_SCHEMAS}
+  return {p: f.get_result() for p, f in futures.iteritems()}
+
+
+def _get_config_revision_async(path):
   """Returns tuple with info about last imported config revision."""
+  assert path in _CONFIG_SCHEMAS, path
   schema = _CONFIG_SCHEMAS.get(path)
-  return schema['revision_getter']() if schema else None
+  return schema['revision_getter']()
 
 
 @utils.cache_with_expiration(expiration_sec=60)
 def get_settings():
   """Returns auth service own settings (from settings.cfg) as SettingsCfg proto.
 
   Returns default settings if the ones in the datastore are no longer valid.
   """
   text = _get_service_config('settings.cfg')
   if not text:
@@ skipping 23 matching lines @@
     configs = _fetch_configs(_CONFIG_SCHEMAS)
   except CannotLoadConfigError as exc:
     logging.error('Failed to fetch configs\n%s', exc)
     return
 
   # Figure out what needs to be updated.
   dirty = {}
   dirty_in_authdb = {}
   for path, (new_rev, conf) in sorted(configs.iteritems()):
     assert path in _CONFIG_SCHEMAS, path
-    cur_rev = get_config_revision(path)
+    cur_rev = _get_config_revision_async(path).get_result()
     if cur_rev != new_rev or force:
       if _CONFIG_SCHEMAS[path]['use_authdb_transaction']:
         dirty_in_authdb[path] = (new_rev, conf)
       else:
         dirty[path] = (new_rev, conf)
     else:
       logging.info('Config %s is up-to-date at rev %s', path, cur_rev.revision)
 
   # First update configs that do not touch AuthDB, one by one.
   for path, (rev, conf) in sorted(dirty.iteritems()):
@@ skipping 57 matching lines @@
   Root entity. Key ID is config file name.
   """
   # The body of the config itself.
   config = ndb.TextProperty()
   # Last imported SHA1 revision of the config.
   revision = ndb.StringProperty(indexed=False)
   # URL the config was imported from.
   url = ndb.StringProperty(indexed=False)
 
 
-def _get_service_config_rev(cfg_name):
+@ndb.tasklet
+def _get_service_config_rev_async(cfg_name):
   """Returns last processed Revision of given config."""
-  e = _AuthServiceConfig.get_by_id(cfg_name)
-  return Revision(e.revision, e.url) if e else None
+  e = yield _AuthServiceConfig.get_by_id_async(cfg_name)
+  raise ndb.Return(Revision(e.revision, e.url) if e else None)
 
 
 def _get_service_config(cfg_name):
   """Returns text of given config file or None if missing."""
   e = _AuthServiceConfig.get_by_id(cfg_name)
   return e.config if e else None
 
 
 @ndb.transactional
 def _update_service_config(cfg_name, rev, conf):
   """Stores new config (and its revision).
 
   This function is called only if config has already been validated.
   """
   assert isinstance(conf, basestring)
   e = _AuthServiceConfig.get_by_id(cfg_name) or _AuthServiceConfig(id=cfg_name)
   old = e.config
   e.populate(config=conf, revision=rev.revision, url=rev.url)
   e.put()
   return old != conf
 
 
 ### Group importer config implementation details.
 
 
-def _get_imports_config_revision():
+@ndb.tasklet
+def _get_imports_config_revision_async():
   """Returns Revision of last processed imports.cfg config."""
-  e = importer.config_key().get()
+  e = yield importer.config_key().get_async()
   if not e or not isinstance(e.config_revision, dict):
-    return None
+    raise ndb.Return(None)
   desc = e.config_revision
-  return Revision(desc.get('rev'), desc.get('url'))
+  raise ndb.Return(Revision(desc.get('rev'), desc.get('url')))
 
 
 def _update_imports_config(rev, conf):
   """Applies imports.cfg config."""
   # Rewrite existing config even if it is the same (to update 'rev').
   cur = importer.read_config()
   importer.write_config(conf, {'rev': rev.revision, 'url': rev.url})
   return cur != conf
 
 
 ### Implementation of configs expanded to AuthDB entities.
 
 
 class _ImportedConfigRevisions(ndb.Model):
   """Stores mapping config path -> {'rev': SHA1, 'url': URL}.
 
   Parent entity is AuthDB root (auth.model.root_key()). Updated in a transaction
   when importing configs.
   """
   revisions = ndb.JsonProperty()
 
 
 def _imported_config_revisions_key():
   return ndb.Key(_ImportedConfigRevisions, 'self', parent=model.root_key())
 
 
-def _get_authdb_config_rev(path):
+@ndb.tasklet
+def _get_authdb_config_rev_async(path):
   """Returns Revision of last processed config given its name."""
-  mapping = _imported_config_revisions_key().get()
+  mapping = yield _imported_config_revisions_key().get_async()
   if not mapping or not isinstance(mapping.revisions, dict):
-    return None
+    raise ndb.Return(None)
   desc = mapping.revisions.get(path)
   if not isinstance(desc, dict):
-    return None
-  return Revision(desc.get('rev'), desc.get('url'))
+    raise ndb.Return(None)
+  raise ndb.Return(Revision(desc.get('rev'), desc.get('url')))
 
 
 @datastore_utils.transactional
 def _update_authdb_configs(configs):
   """Pushes new configs to AuthDB entity group.
 
   Args:
     configs: dict {config path -> (Revision tuple, <config>)}.
   """
   revs = _imported_config_revisions_key().get()
@@ skipping 186 matching lines @@
       modified_ts=utils.utcnow(),
       comment='Importing oauth.cfg at rev %s' % rev.revision)
   existing.put()
   return True
 
 
 ### Description of all known config files: how to validate and import them.
 
 # Config file name -> {
 #   'proto_class': protobuf class of the config or None to keep it as text,
-#   'revision_getter': lambda: <latest imported Revision>,
+#   'revision_getter': lambda: ndb.Future with <latest imported Revision>
 #   'validator': lambda config: <raises ValueError on invalid format>
 #   'updater': lambda rev, config: True if applied, False if not.
 #   'use_authdb_transaction': True to call 'updater' in AuthDB transaction.
 #   'default': Default config value to use if the config file is missing.
 # }
 _CONFIG_SCHEMAS = {
   'imports.cfg': {
     'proto_class': None, # importer configs are stored as text
-    'revision_getter': _get_imports_config_revision,
+    'revision_getter': _get_imports_config_revision_async,
     'updater': _update_imports_config,
     'use_authdb_transaction': False,
   },
   'ip_whitelist.cfg': {
     'proto_class': config_pb2.IPWhitelistConfig,
-    'revision_getter': lambda: _get_authdb_config_rev('ip_whitelist.cfg'),
+    'revision_getter': lambda: _get_authdb_config_rev_async('ip_whitelist.cfg'),
     'updater': _update_ip_whitelist_config,
     'use_authdb_transaction': True,
   },
   'oauth.cfg': {
     'proto_class': config_pb2.OAuthConfig,
-    'revision_getter': lambda: _get_authdb_config_rev('oauth.cfg'),
+    'revision_getter': lambda: _get_authdb_config_rev_async('oauth.cfg'),
     'updater': _update_oauth_config,
     'use_authdb_transaction': True,
   },
   'settings.cfg': {
     'proto_class': None, # settings are stored as text in datastore
     'default': '',  # it's fine if config file is not there
-    'revision_getter': lambda: _get_service_config_rev('settings.cfg'),
+    'revision_getter': lambda: _get_service_config_rev_async('settings.cfg'),
     'updater': lambda rev, c: _update_service_config('settings.cfg', rev, c),
     'use_authdb_transaction': False,
   },
 }
 
 
 @utils.memcache('auth_service:get_configs_url', time=300)
 def _get_configs_url():
   """Returns URL where luci-config fetches configs from."""
   url = config.get_config_set_location(config.self_config_set())
@@ skipping 40 matching lines @@
   try:
     location = gitiles.Location.parse(configs_url)
     return str(gitiles.Location(
         hostname=location.hostname,
         project=location.project,
         treeish=rev,
         path=posixpath.join(location.path, path)))
   except ValueError:
     # Not a gitiles URL, return as is.
     return configs_url