LogDog: Use datastore config cache when available.

logdog/server/service/service.go

Index: logdog/server/service/service.go
diff --git a/logdog/server/service/service.go b/logdog/server/service/service.go
index 8d896a002a4e6667e69402703fcd48f8f857c335..602c85e11cc4129f2858c368c0e37949eee1f4e5 100644
--- a/logdog/server/service/service.go
+++ b/logdog/server/service/service.go
@@ -16,6 +16,7 @@ import (
 	"sync/atomic"
 	"time"
 
+	"github.com/luci/luci-go/appengine/gaesettings"
 	"github.com/luci/luci-go/client/authcli"
 	"github.com/luci/luci-go/common/auth"
 	"github.com/luci/luci-go/common/clock/clockflag"
@@ -40,8 +41,12 @@ import (
 	"github.com/luci/luci-go/luci_config/server/cfgclient/backend/client"
 	"github.com/luci/luci-go/luci_config/server/cfgclient/backend/testconfig"
 	"github.com/luci/luci-go/luci_config/server/cfgclient/textproto"
+	"github.com/luci/luci-go/server/settings"
+
+	"github.com/luci/gae/impl/cloud"
 
 	"cloud.google.com/go/compute/metadata"
+	"cloud.google.com/go/datastore"
 	"golang.org/x/net/context"
 	"golang.org/x/oauth2"
 	"google.golang.org/api/option"
@@ -92,7 +97,8 @@ type Service struct {
 	heapProfilePath           string
 
 	// onGCE is true if we're on GCE. We probe this exactly once.
-	onGCE bool
+	onGCE        bool
+	hasDatastore bool
 
 	killCheckInterval clockflag.Duration
 	configFilePath    string
@@ -127,6 +133,10 @@ func (s *Service) Run(c context.Context, f func(context.Context) error) {
 }
 
 func (s *Service) runImpl(c context.Context, f func(context.Context) error) error {
+	// Probe our environment for default values.
+	s.probeGCEEnvironment(c)

[iannucci, 2017/01/21 00:32:24]

should this have been in the previous CL?

[dnj, 2017/01/21 02:00:02]

Ugh, looks like it was removed when I split. Oh well, it's here now.

+
+	// Install service flags and parse.
 	s.addFlags(c, &s.Flags)
 	if err := s.Flags.Parse(os.Args[1:]); err != nil {
 		log.WithError(err).Errorf(c, "Failed to parse command-line.")
@@ -172,11 +182,32 @@ func (s *Service) runImpl(c context.Context, f func(context.Context) error) erro
 		}()
 	}
 
+	// Cancel our Context after we're done our run loop.
+	c, cancelFunc := context.WithCancel(c)
+	defer cancelFunc()
+
 	// Validate the runtime environment.
 	if s.serviceID == "" {
 		return errors.New("no service ID was configured (-service-id)")
 	}
 
+	// Install a cloud datastore client. This is non-fatal if it fails.
+	dsClient, err := s.initDatastoreClient(c)
+	if err == nil {
+		defer dsClient.Close()
+
+		ccfg := cloud.Config{
+			DS: dsClient,
+		}
+		c = ccfg.Use(c)
+		c = settings.Use(c, settings.New(gaesettings.Storage{}))
+
+		s.hasDatastore = true
+		log.Infof(c, "Enabled cloud datastore access.")
+	} else {
+		log.WithError(err).Debugf(c, "Failed to create cloud datastore client.")

[iannucci, 2017/01/21 00:32:24]

wdyt about reversing the severities? maybe debug for 'enabled' and warning for
'failed to create'? I assume it should be typical that we DO have datastore
access?

[dnj, 2017/01/21 02:00:02]

Okay.

+	}
+
 	// Install a process-wide cache.
 	c = proccache.Use(c, &proccache.Cache{})
 
@@ -214,7 +245,6 @@ func (s *Service) runImpl(c context.Context, f func(context.Context) error) erro
 	defer tsmon.Shutdown(c)
 
 	// Initialize our Client instantiations.
-	var err error
 	if s.coord, err = s.initCoordinatorClient(c); err != nil {
 		log.WithError(err).Errorf(c, "Failed to setup Coordinator client.")
 		return err
@@ -271,12 +301,12 @@ func (s *Service) addFlags(c context.Context, fs *flag.FlagSet) {
 // probeGCEEnvironment fills in any parameters that can be probed from Google
 // Compute Engine metadata.
 //
-// If we're not running on GCE, this will return nil. An error will only be
-// returned if an operation that is expected to work fails.
-func (s *Service) probeGCEEnvironment(c context.Context) error {
+// If we're not running on GCE, this will do nothing. It is non-fatal if any
+// given GCE field fails to be probed.
+func (s *Service) probeGCEEnvironment(c context.Context) {
 	s.onGCE = metadata.OnGCE()
 	if !s.onGCE {
-		return nil
+		return
 	}
 
 	// Determine our service ID from metadata. The service ID will equal the cloud
@@ -284,11 +314,23 @@ func (s *Service) probeGCEEnvironment(c context.Context) error {
 	if s.serviceID == "" {
 		var err error
 		if s.serviceID, err = metadata.ProjectID(); err != nil {
-			log.WithError(err).Errorf(c, "Failed to probe GCE project ID.")
-			return err
+			log.WithError(err).Warningf(c, "Failed to probe GCE project ID.")
 		}
 	}
-	return nil
+}
+
+func (s *Service) initDatastoreClient(c context.Context) (*datastore.Client, error) {
+	// Initialize Storage authentication.
+	tokenSource, err := s.TokenSource(c, func(o *auth.Options) {
+		o.Scopes = []string{datastore.ScopeDatastore}
+	})
+	if err != nil {
+		log.WithError(err).Errorf(c, "Failed to create datastore TokenSource.")
+		return nil, err
+	}
+
+	return datastore.NewClient(c, s.serviceID,
+		option.WithTokenSource(tokenSource))
 }
 
 func (s *Service) initCoordinatorClient(c context.Context) (logdog.ServicesClient, error) {
@@ -360,6 +402,9 @@ func (s *Service) initConfig(c *context.Context) error {
 		p = &client.RemoteProvider{
 			Host: host,
 		}
+
+		// If using a remote config provider, enable datastore access and caching.
+		opts.DatastoreCacheAvailable = s.hasDatastore
 	} else {
 		// Test / Local: use filesystem config path.
 		ci, err := filesystem.New(s.configFilePath)