LogDog: enable services to fetch project configs

server/internal/logdog/service/service.go

 // Copyright 2016 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 package service
 
 import (
         "errors"
         "flag"
         "fmt"
         "net/http"
         "os"
         "os/signal"
         "path/filepath"
         "runtime/pprof"
         "sync/atomic"
+        "time"
 
         "github.com/luci/luci-go/client/authcli"
         "github.com/luci/luci-go/common/api/logdog_coordinator/services/v1"
         "github.com/luci/luci-go/common/auth"
+        luciConfig "github.com/luci/luci-go/common/config"
         "github.com/luci/luci-go/common/gcloud/gs"
         log "github.com/luci/luci-go/common/logging"
         "github.com/luci/luci-go/common/logging/gologger"
         "github.com/luci/luci-go/common/proto/logdog/svcconfig"
         "github.com/luci/luci-go/common/prpc"
         "github.com/luci/luci-go/common/tsmon"
         "github.com/luci/luci-go/common/tsmon/metric"
         "github.com/luci/luci-go/common/tsmon/target"
         "github.com/luci/luci-go/server/internal/logdog/retryServicesClient"
         "github.com/luci/luci-go/server/internal/logdog/service/config"
         "github.com/luci/luci-go/server/logdog/storage"
         "github.com/luci/luci-go/server/logdog/storage/bigtable"
         "golang.org/x/net/context"
         "google.golang.org/cloud"
+        "google.golang.org/cloud/compute/metadata"
 )
 
 var (
         // ErrInvalidConfig is an error that is returned when the supplied
         // configuration is invalid.
         ErrInvalidConfig = errors.New("invalid configuration")
 
         // CoordinatorScopes is the set of OAuth2 scopes to use for the Coordinator
         // client.
         CoordinatorScopes = []string{
                 auth.OAuthScopeEmail,
         }
 
         presenceUpMetric = metric.NewBool("presence/up",
                 "Set to one when service is present and ready. Alert on missing values.")
 )
 
+// projectConfigCacheDuration is the amount of time to cache a project's
+// configuration before reloading.
+const projectConfigCacheDuration = 30 * time.Minute
+
 // Service is a base class full of common LogDog service application parameters.
 type Service struct {
         // Name is the name of this service. It is used for logging, metrics, and
         // user agent string generation.
         //
         // If empty, a service name will be inferred from the command-line arguments.
         Name string
         // Flags is the set of flags that will be used by the Service.
         Flags flag.FlagSet
 
         shutdownFunc atomic.Value
 
         loggingFlags log.Config
         authFlags    authcli.Flags
         configFlags  config.Flags
         tsMonFlags   tsmon.Flags
 
         coordinatorHost           string
         coordinatorInsecure       bool
+        projectID                 string
         storageCredentialJSONPath string
         cpuProfilePath            string
         heapProfilePath           string
 
         coord  logdog.ServicesClient
         config *config.Manager
+
+        onGCE bool
 }
 
 // Run performs service-wide initialization and invokes the specified run
 // function.
 func (s *Service) Run(c context.Context, f func(context.Context) error) {
         c = gologger.StdConfig.Use(c)
 
         // If a service name isn't specified, default to the base of the current
         // executable.
         if s.Name == "" {
@@ skipping 47 matching lines @@
 
                         if err := pprof.WriteHeapProfile(fd); err != nil {
                                 log.Fields{
                                         log.ErrorKey: err,
                                         "path":       p,
                                 }.Warningf(c, "Failed to write heap profile.")
                         }
                 }()
         }
 
+        // Are we running on a GCE intance?
+        if err := s.probeGCEEnvironment(c); err != nil {
+                log.WithError(err).Errorf(c, "Failed to probe GCE environment.")
+                return err
+        }
+
+        // Validate the runtime environment.
+        if s.projectID == "" {
+                return errors.New("no project ID was configured")
+        }
+
         // Configure our signal handler. It will listen for terminating signals and
         // issue a shutdown signal if one is received.
         signalC := make(chan os.Signal)
         go func() {
                 hasShutdownAlready := false
                 for sig := range signalC {
                         if !hasShutdownAlready {
                                 hasShutdownAlready = true
 
                                 log.Warningf(log.SetField(c, "signal", sig), "Received close signal. Send again to terminate immediately.")
@@ skipping 60 matching lines @@
         // Initialize tsmon flags.
         s.tsMonFlags = tsmon.NewFlags()
         s.tsMonFlags.Flush = tsmon.FlushAuto
         s.tsMonFlags.Target.TargetType = target.TaskType
         s.tsMonFlags.Target.TaskJobName = s.Name
         s.tsMonFlags.Register(fs)
 
         s.authFlags.Register(fs, auth.Options{})
         s.configFlags.AddToFlagSet(fs)
 
+        fs.StringVar(&s.projectID, "project-id", "",
+                "Specify the cloud project ID that this instance is supporting. If empty, the project ID "+

[nodir, 2016/05/18 16:25:51]

Please rename to "app-id". Gcp SDK calls it "app" too 

[dnj (Google), 2016/05/18 16:50:31]

I'm using the luci-config concept here, not the GAE one.

[nodir, 2016/05/18 17:02:19]

Then why it says cloud project and why this flag is mapped to a field which is
populated with GCE metadata. I think it is so confusing even you confuse it

+                        "will attempt to be resolved by probing the local environment. This probably will match the "+
+                        "App ID of the Coordinator.")
         fs.StringVar(&s.coordinatorHost, "coordinator", "",
                 "The Coordinator service's [host][:port].")
         fs.BoolVar(&s.coordinatorInsecure, "coordinator-insecure", false,
                 "Connect to Coordinator over HTTP (instead of HTTPS).")
         fs.StringVar(&s.storageCredentialJSONPath, "storage-credential-json-path", "",
                 "If supplied, the path of a JSON credential file to load and use for storage operations.")
         fs.StringVar(&s.cpuProfilePath, "cpu-profile-path", "",
                 "If supplied, enable CPU profiling and write the profile here.")
         fs.StringVar(&s.heapProfilePath, "heap-profile-path", "",
                 "If supplied, enable CPU profiling and write the profile here.")
 }
 
+// probeGCEEnvironment fills in any parameters that can be probed from Google
+// Compute Engine metadata.
+//
+// If we're not running on GCE, this will return nil. An error will only be
+// returned if an operation that is expected to work fails.
+func (s *Service) probeGCEEnvironment(c context.Context) error {
+        s.onGCE = metadata.OnGCE()
+        if !s.onGCE {
+                return nil
+        }
+
+        // Determine our project ID.
+        if s.projectID == "" {
+                var err error
+                if s.projectID, err = metadata.ProjectID(); err != nil {
+                        log.WithError(err).Errorf(c, "Failed to probe GCE project ID.")
+                        return err
+                }
+        }
+        return nil
+}
+
 func (s *Service) initCoordinatorClient(c context.Context) (logdog.ServicesClient, error) {
         if s.coordinatorHost == "" {
                 log.Errorf(c, "Missing Coordinator URL (-coordinator).")
                 return nil, ErrInvalidConfig
         }
 
         httpClient, err := s.AuthenticatedClient(c, func(o *auth.Options) {
                 o.Scopes = CoordinatorScopes
         })
         if err != nil {
@@ skipping 22 matching lines @@
                 log.WithError(err).Errorf(c, "Failed to create config client.")
                 return nil, err
         }
 
         s.configFlags.RoundTripper = rt
         o, err := s.configFlags.CoordinatorOptions(c, s.coord)
         if err != nil {
                 log.WithError(err).Errorf(c, "Failed to load configuration parameters.")
                 return nil, err
         }
+        o.ProjectID = s.projectID
+        o.ProjectConfigCacheDuration = projectConfigCacheDuration
         o.KillFunc = s.shutdown
 
         return config.NewManager(c, *o)
 }
 
 // SetShutdownFunc sets the service shutdown function.
 func (s *Service) SetShutdownFunc(f func()) {
         s.shutdownFunc.Store(f)
 }
 
 func (s *Service) shutdown() {
         v := s.shutdownFunc.Load()
         if f, ok := v.(func()); ok {
                 f()
         } else {
                 s.shutdownImmediately()
         }
 }
 
 func (s *Service) shutdownImmediately() {
         os.Exit(1)
 }
 
 // Config returns the cached service configuration.
 func (s *Service) Config() *svcconfig.Config {
         return s.config.Config()
 }
 
+// ProjectConfig returns the cached project configuration.
+//
+// If the project configuration is not available, nil will be returned.
+func (s *Service) ProjectConfig(c context.Context, proj luciConfig.ProjectName) (*svcconfig.ProjectConfig, error) {
+        return s.config.ProjectConfig(c, proj)
+}
+
 // Coordinator returns the cached Coordinator client.
 func (s *Service) Coordinator() logdog.ServicesClient {
         return s.coord
 }
 
+// ProjectID returns the project ID.
+//
+// This is synonymous with the AppEngine "app ID".
+func (s *Service) ProjectID() string {
+        return s.projectID
+}
+
 // IntermediateStorage instantiates the configured intermediate Storage
 // instance.
 func (s *Service) IntermediateStorage(c context.Context) (storage.Storage, error) {
         cfg := s.config.Config()
         if cfg.GetStorage() == nil {
                 log.Errorf(c, "Missing storage configuration.")
                 return nil, ErrInvalidConfig
         }
 
         btcfg := cfg.GetStorage().GetBigtable()
@@ skipping 81 matching lines @@
 //
 // An optional permutation functon can be provided to modify those Options
 // before the Authenticator is created.
 func (s *Service) AuthenticatedClient(c context.Context, f func(o *auth.Options)) (*http.Client, error) {
         a, err := s.Authenticator(c, f)
         if err != nil {
                 return nil, err
         }
         return a.Client()
 }