config services: services.cfg and validation

appengine/config_service/services.py

+# Copyright 2015 The Swarming Authors. All rights reserved.
+# Use of this source code is governed by the Apache v2.0 license that can be
+# found in the LICENSE file.
+
+"""Provides info about registered luci services."""
+
+from components import config
+from components import net
+from components import utils
+
+from google.appengine.ext import ndb
+
+from proto import service_config_pb2
+import common
+import storage
+import validation
+
+
+class DynamicMetadataError(Exception):
+  """Raised when a service metadata endpoint response is bad."""
+
+
+@ndb.tasklet
+def get_services_async():
+  """Returns a list of registered luci services.
+
+  The list is stored in services/luci-config:services.cfg. Never returns None.
+  Cached.
+  """
+  cfg = yield storage.get_self_config_async(
+      common.SERVICES_REGISTRY_FILENAME, service_config_pb2.ServicesCfg)
+  raise ndb.Return(cfg.services or [])
+
+
+@ndb.tasklet
+def _get_service_async(service_id):
+  services = yield get_services()

[Vadim Sh., 2015/07/08 03:05:58]

I don't think get_services is defined. How did this pass unit tests? ;)

[nodir, 2015/07/08 15:24:47]

Yeah, this file needed a test. Fixed other bugs

+  for service in services:
+    if service.id == service_id:
+      raise ndb.Return(service)
+
+
+def _dict_to_dynamic_metadata(data):

[Vadim Sh., 2015/07/08 03:05:59]

is it the only place where proto message and protorpc message meet? Mention it
somewhere in docs for ServiceDynamicMetadata (or protorpc message), e.g. end
goal is for doc strings to describe how to easily add a new field if needed.

[nodir, 2015/07/08 15:24:47]

Done.

+  validation.validate_service_dynamic_metadata_blob(
+      data,
+      config.validation.Context.raise_on_error(exc_type=DynamicMetadataError))
+
+  metadata = service_config_pb2.ServiceDynamicMetadata()
+  validation_meta = data.get('validation')
+  if validation_meta:
+    metadata.validation.url = validation_meta['url']
+    for p in validation_meta.get('patterns', []):
+      pattern = metadata.validation.patterns.add()
+      pattern.config_set=p['config_set']
+      pattern.path=p['path']
+  return metadata
+
+
+@utils.memcache_async('service_metadata', ['service_id'], time=60)
+def get_metadata_async(service_id):
+  """Returns service dynamic metadata.
+
+  Memcaches results for 1 min. Never returns None.
+
+  Raises:
+    DynamicMetadataError if metadata endpoint response is bad.

[Vadim Sh., 2015/07/08 03:05:59]

also ValueError, make it DynamicMetadataError too?

[nodir, 2015/07/08 15:24:47]

Done.

+  """
+  service = yield _get_service_async(service_id)
+  if service is None:
+    raise ValueError('Service "%s" not found', service_id)
+
+  if not service.metadata_url:
+    raise ndb.Return(service_config_pb2.ServiceDynamicMetadata())
+
+  try:
+    res = net.json_request_async(service.metadata_url)

[Vadim Sh., 2015/07/08 03:05:59]

need auth here? I think yes, because validation configs leak info about
structure of configs.

[nodir, 2015/07/08 15:24:47]

Done.

+  except net.Error as ex:
+    raise DynamicMetadataError('Net error: %s' % ex.message)
+  raise ndb.Return(_dict_to_dynamic_metadata(res))