swarming: read configs from luci-config

appengine/swarming/server/config.py

Index: appengine/swarming/server/config.py
diff --git a/appengine/swarming/server/config.py b/appengine/swarming/server/config.py
index 3f76daa32a967114e37d8c6272146365b9d44e41..e9f5cf36518162c88530ba0b37e3197bbe46c2e0 100644
--- a/appengine/swarming/server/config.py
+++ b/appengine/swarming/server/config.py
@@ -4,32 +4,85 @@
 
 """Instance specific settings."""
 
-from google.appengine.ext import ndb
+import logging
+import posixpath
 
-from components.datastore_utils import config
+from components import config
+from components import gitiles
+from components import net
+from components import utils
+from components.config import validation
 
+from proto import config_pb2
 
-class GlobalConfig(config.GlobalConfig):
-  """Application wide settings."""
-  # id to inject into pages if applicable.
-  google_analytics = ndb.StringProperty(indexed=False, default='')
+SETTINGS_CFG_FILENAME = 'settings.cfg'
 
-  # The number of seconds an old task can be deduped from.
-  reusable_task_age_secs = ndb.IntegerProperty(
-      indexed=False, default=7*24*60*60)
 
-  # The amount of time that has to pass before a machine is considered dead.
-  bot_death_timeout_secs = ndb.IntegerProperty(default=10*60)
+ConfigApi = config.ConfigApi
 
-  # Enable ts_mon based monitoring.
-  enable_ts_monitoring = ndb.BooleanProperty(indexed=False, default=False)
 
+@validation.self_rule(SETTINGS_CFG_FILENAME, config_pb2.SettingsCfg)
+def validate_settings(_cfg, _ctx):
+  """Validates settings.cfg file against proto message schema."""
 
-def settings(fresh=False):
-  """Loads GlobalConfig or a default one if not present.
 
-  If fresh=True, a full fetch from NDB is done.
+@utils.memcache('config:get_configs_url', time=300)

[M-A Ruel, 2016/04/27 17:50:45]

5 minutes is a long time. 1 minute would be more sensible.

[nodir, 2016/04/27 18:37:21]

This value almost never changes.

[M-A Ruel, 2016/04/27 18:51:19]

but when it does, we don't want to wait. Furthermore, getting "settings
toggling" as one gets older, newer, then older again can be potentially
annoying/harmful. 1 minute is more reasonable.

[nodir, 2016/04/27 19:08:52]

changed to 1min

+def _get_configs_url():
+  """Returns URL where luci-config fetches configs from."""
+  url = None
+  try:
+    url = config.get_config_set_location(config.self_config_set())
+  except net.Error:
+    logging.exception('Could not get configs URL')
+  return url or 'about:blank'
+
+
+def _gitiles_url(configs_url, rev, path):
+  """URL to a directory in gitiles -> URL to a file at concrete revision."""
+  try:
+    loc = gitiles.Location.parse(configs_url)
+    return str(loc._replace(
+        treeish=rev or loc.treeish,
+        path=posixpath.join(loc.path, path)))
+  except ValueError:
+    # Not a gitiles URL, return as is.
+    return configs_url
+
+
+def _get_settings():
+  """Returns (rev, cfg) where cfg is a parsed SettingsCfg message.
+
+  The config is cached in the datastore.
   """
-  if fresh:
-    GlobalConfig.clear_cache()
-  return GlobalConfig.cached()
+  rev = None
+  cfg = None
+  try:
+    # store_last_good=True tells config component to update the config file
+    # in a cron job. Here we just read from datastore.
+    rev, cfg = config.get_self_config(
+        SETTINGS_CFG_FILENAME, config_pb2.SettingsCfg, store_last_good=True)
+  except config.CannotLoadConfigError:
+    logging.exception('Could not load settings.cfg; using defaults')
+  if not cfg:
+    cfg = config_pb2.SettingsCfg(
+        reusable_task_age_secs=7*24*60*60,
+        bot_death_timeout_secs=10*60,
+    )

[M-A Ruel, 2016/04/27 17:50:45]

This doesn't match the code style of this project.

[nodir, 2016/04/27 18:37:21]

you mean that the closing bracket must be on the same line?

[M-A Ruel, 2016/04/27 18:51:19]

yes

+  return rev, cfg
+
+
+def settings_info():
+  """Returns (settings, rev, rev_url) of the settings file.
+
+  Where settings is a parsed SettingsCfg message, rev is the revision of the
+  config and rev_url is a URL of a human-consumable page that displays the
+  config.
+  """
+  rev, cfg = _get_settings()
+  rev_url = _gitiles_url(_get_configs_url(), rev, SETTINGS_CFG_FILENAME)
+  return cfg, rev, rev_url
+
+
+def settings():
+  """Loads settings from an NDB-based cache or a default one if not present."""
+  return _get_settings()[1]