Implement config service cache on top of datastore

appengine/gaeconfig/default.go

 // Copyright 2016 The LUCI Authors. All rights reserved.
 // Use of this source code is governed under the Apache License, Version 2.0
 // that can be found in the LICENSE file.
 
 package gaeconfig
 
 import (
         "errors"
         "fmt"
         "os"
         "path/filepath"
         "time"
 
         "github.com/luci/luci-go/common/config/impl/filesystem"
         "github.com/luci/luci-go/server/config"
         "github.com/luci/luci-go/server/config/caching"
         "github.com/luci/luci-go/server/config/erroring"
         "github.com/luci/luci-go/server/config/testconfig"
         "github.com/luci/luci-go/server/config/textproto"
+        "github.com/luci/luci-go/server/router"
 
         "github.com/luci/gae/service/info"
 
         "golang.org/x/net/context"
 )
 
 // ErrNotConfigured is returned by methods of config.Backend object returned
 // by New if config service URL is not set. Usually happens for new apps.
 var ErrNotConfigured = errors.New("config service URL is not set in settings")
 
 // devCfgDir is a name of the directory with config files when running in
 // local dev appserver model. See New for details.
 const devCfgDir = "devcfg"
 
+// InstallCacheCronHandler installs the configuration service datastore caching
+// cron handler. This must be installed, and an associated cron must be set up,
+// if datastore caching is enabled.
+//
+// The cron should be configured to hit the handler at:
+// /admin/config/cache/manager
+func InstallCacheCronHandler(r *router.Router, base router.MiddlewareChain) {
+        installCacheCronHandlerImpl(r, base, nil)
+}
+
+// Install our cache cron handler into the supplied Router.
+//
+// bf is a generator function used to get the primary (service) Backend to build
+// on top of. If nil, the "production" (one used by Use) Backend will be used.
+func installCacheCronHandlerImpl(r *router.Router, base router.MiddlewareChain, backend config.Backend) {
+        base = base.Extend(func(c *router.Context, next router.Handler) {
+                // Install our Backend into our Context.
+                c.Context = installConfigBackend(c.Context, mustFetchCachedSettings(c.Context), backend, true)
+                next(c)
+        })
+
+        dsCache.InstallCronRoute("/admin/config/cache/manager", r, base)
+}
+
 // Use installs the default luci-config client.
 //
 // The client is configured to use luci-config URL specified in the settings,
 // using GAE app service account for authentication.
 //
 // If running in prod, and the settings don't specify luci-config URL, produces
 // an implementation of config.Interface that returns ErrNotConfigured from all
 // methods.
 //
 // If running on devserver, and the settings don't specify luci-config URL,
 // returns a filesystem-based implementation that reads configs from a directory
 // (or a symlink) named 'devcfg' located in the GAE module directory (where
 // app.yaml is) or its immediate parent directory.
 //
 // If such directory can not be located, produces an implementation of
 // config.Interface that returns errors from all methods.
 //
 // Panics if it can't load the settings (should not happen since they are in
 // the local memory cache usually).
 func Use(c context.Context) context.Context { return useImpl(c, nil) }
 
 func useImpl(c context.Context, backend config.Backend) context.Context {
-        return installConfigBackend(c, mustFetchCachedSettings(c), backend)
+        return installConfigBackend(c, mustFetchCachedSettings(c), backend, false)
 }
 
-func installConfigBackend(c context.Context, s *Settings, backend config.Backend) context.Context {
+func installConfigBackend(c context.Context, s *Settings, backend config.Backend, dsCron bool) context.Context {
         if backend == nil {
                 // Non-testing, build a Backend.
                 backend = getPrimaryBackend(c, s)
         }
 
         // Install a FormatRegistry. Register common config service protobufs with it.
         c = withFormatRegistry(c, defaultFormatterRegistry())
 
         backend = &config.FormatBackend{
                 Backend:     backend,
                 GetRegistry: GetFormatterRegistry,
         }
 
         // Apply caching configuration.
         exp := time.Duration(s.CacheExpirationSec) * time.Second
         if exp > 0 {
                 // Add a ProcCache, backed by memcache.
                 backend = caching.ProcCache(backend, exp)
                 backend = memcacheBackend(backend, exp)
+
+                // If our datastore cache is enabled, install a handler for refresh. This
+                // will be loaded by dsCache's "HandlerFunc".
+                if s.DatastoreCacheMode != dsCacheDisabled {
+                        dsc := datastoreCache{
+                                refreshInterval: exp,
+                                failOpen:        s.DatastoreCacheMode == dsCacheEnabled,
+                        }
+
+                        if !dsCron {
+                                // For non-cron, install the datastore cache Backend.
+                                backend = dsc.getBackend(backend)
+                        } else {
+                                // For cron, do not install the datastore cache Backend. Instead,
+                                // install a lasting Handler into the Context to be used for cache
+                                // resolution. This is necessary since resolution calls will not be
+                                // the result of an actual resolution command (e.g., cache.Get).
+                                c = dsc.withHandler(c, datastoreCronLoader(backend), dsRPCDeadline)
+                        }
+                }
         }
 
         c = config.WithBackend(c, backend)
         return c
 }
 
 func getPrimaryBackend(c context.Context, settings *Settings) (backend config.Backend) {
         // Identify our config service Backend (in testing, it will be supplied).
         if settings.ConfigServiceURL == "" {
                 if info.IsDevAppServer(c) {
@@ skipping 39 matching lines @@
 
 func withFormatRegistry(c context.Context, fr *config.FormatterRegistry) context.Context {
         return context.WithValue(c, &formatRegistryKey, fr)
 }
 
 // GetFormatterRegistry is used to retrieve the config service format registry
 // from the supplied Context.
 func GetFormatterRegistry(c context.Context) *config.FormatterRegistry {
         return c.Value(&formatRegistryKey).(*config.FormatterRegistry)
 }