Milo: Move instance configuration to luci-config

milo/appengine/buildbot/pubsub.go

 // Copyright 2016 The LUCI Authors. All rights reserved.
 // Use of this source code is governed under the Apache License, Version 2.0
 // that can be found in the LICENSE file.
 
 package buildbot
 
 import (
         "bytes"
         "compress/gzip"
         "compress/zlib"
         "encoding/base64"
         "encoding/json"
         "fmt"
         "net/http"
         "strings"
         "time"
 
         ds "github.com/luci/gae/service/datastore"
         "github.com/luci/luci-go/common/clock"
         "github.com/luci/luci-go/common/iotools"
         "github.com/luci/luci-go/common/logging"
+        "github.com/luci/luci-go/milo/appengine/common"
         "github.com/luci/luci-go/server/router"
 
         "golang.org/x/net/context"
 
         "github.com/luci/luci-go/common/tsmon/field"
         "github.com/luci/luci-go/common/tsmon/metric"
 )
 
 var (
-        // publicSubName is the name of the pubsub subscription that milo is expecting.
-        // TODO(hinoka): This should be read from luci-config.
-        publicSubName   = "projects/luci-milo/subscriptions/buildbot-public"
-        internalSubName = "projects/luci-milo/subscriptions/buildbot-private"
-
         // Metrics
         buildCounter = metric.NewCounter(
                 "luci/milo/buildbot_pubsub/builds",
                 "The number of buildbot builds received by Milo from PubSub",
                 nil,
                 field.Bool("internal"),
                 field.String("master"),
                 field.String("builder"),
                 field.Bool("finished"),
                 // Status can be one of 3 options.  "New", "Replaced", "Rejected".
@@ skipping 175 matching lines @@
 
 func doMaster(c context.Context, master *buildbotMaster, internal bool) int {
         // Store the master json into the datastore.
         err := putDSMasterJSON(c, master, internal)
         fullname := fmt.Sprintf("master.%s", master.Name)
         if err != nil {
                 logging.WithError(err).Errorf(
                         c, "Could not save master in datastore %s", err)
                 masterCounter.Add(c, 1, internal, fullname, "failure")
                 // This is transient, we do want PubSub to retry.
-                return 500
+                return http.StatusInternalServerError
         }
         masterCounter.Add(c, 1, internal, fullname, "success")
 
         // Extract current builds data out of the master json, and use it to
         // clean up expired builds.
         q := ds.NewQuery("buildbotBuild").
                 Eq("finished", false).
                 Eq("master", master.Name)
         builds := []*buildbotBuild{}
         err = getBuildQueryBatcher(c).GetAll(c, q, &builds)
         if err != nil {
                 logging.WithError(err).Errorf(c, "Could not load current builds from master %s",
                         master.Name)
-                return 500
+                return http.StatusInternalServerError
         }
         for _, b := range builds {
                 builder, ok := master.Builders[b.Buildername]
                 if !ok {
                         // Mark this build due to builder being removed.
                         buildCounter.Add(
                                 c, 1, internal, b.Master, b.Buildername, b.Finished, "Expired")
                         logging.Infof(c, "Expiring %s/%s/%d due to builder being removed",
                                 master.Name, b.Buildername, b.Number)
                         err = expireBuild(c, b)
                         if err != nil {
                                 logging.WithError(err).Errorf(c, "Could not expire build")
-                                return 500
+                                return http.StatusInternalServerError
                         }
                         continue
                 }
 
                 found := false
                 for _, bnum := range builder.CurrentBuilds {
                         if b.Number == bnum {
                                 found = true
                                 break
                         }
                 }
                 if !found {
                         now := int(clock.Now(c).Unix())
                         if b.TimeStamp == nil || ((*b.TimeStamp)+20*60 < now) {
                                 // Expire builds after 20 minutes of not getting data.
                                 // Mark this build due to build not current anymore.
                                 buildCounter.Add(
                                         c, 1, internal, b.Master, b.Buildername, b.Finished, "Expired")
                                 logging.Infof(c, "Expiring %s/%s/%d due to build not current",
                                         master.Name, b.Buildername, b.Number)
                                 err = expireBuild(c, b)
                                 if err != nil {
                                         logging.WithError(err).Errorf(c, "Could not expire build")
-                                        return 500
+                                        return http.StatusInternalServerError
                                 }
                         }
                 }
         }
         return 0
 }
 
 // PubSubHandler is a webhook that stores the builds coming in from pubsub.
 func PubSubHandler(ctx *router.Context) {
         statusCode := pubSubHandlerImpl(ctx.Context, ctx.Request)
         ctx.Writer.WriteHeader(statusCode)
 }
 
 // This is the actual implementation of the pubsub handler.  Returns
-// a status code.  200 for okay (ACK implied, don't retry).  Anything else
-// will signal to pubsub to retry.
+// a status code.  StatusOK (200) for okay (ACK implied, don't retry).
+// Anything else will signal to pubsub to retry.
 func pubSubHandlerImpl(c context.Context, r *http.Request) int {
         msg := pubSubSubscription{}
         now := int(clock.Now(c).Unix())
         defer r.Body.Close()
         dec := json.NewDecoder(r.Body)
         if err := dec.Decode(&msg); err != nil {
                 logging.WithError(err).Errorf(
                         c, "Could not decode message.  %s", err)
-                return 200 // This is a hard failure, we don't want PubSub to retry.
+                return http.StatusOK // This is a hard failure, we don't want PubSub to retry.
         }
         internal := true
+        // Get the name of the subscription on luci-config
+        settings, err := common.GetSettings(c)
+        if err != nil {
+                logging.WithError(err).Errorf(c,
+                        "Cannot load settings to check subscription name.")
+                // This is a configuration error. Tell PubSub to retry until we fix our
+                // configs.
+                return http.StatusInternalServerError
+        }
         switch msg.Subscription {
-        // TODO(hinoka): Move these names to luci-config
-        case publicSubName, publicSubName + "-dev":
+        case settings.Buildbot.PublicTopic:
                 internal = false
-        case internalSubName, internalSubName + "-dev":
+        case settings.Buildbot.InternalTopic:
                 // internal = true, but that's already set.
         default:
                 logging.Errorf(
                         c, "Subscription name %s does not match %s or %s",
-                        msg.Subscription, publicSubName, internalSubName)
-                return 200
+                        msg.Subscription, settings.Buildbot.PublicTopic,
+                        settings.Buildbot.InternalTopic)
+                // This is a configuration error. Tell PubSub to retry until we fix our
+                // configs.
+                return http.StatusInternalServerError
         }
         logging.Infof(
                 c, "Message ID \"%s\" from subscription %s is %d bytes long",
                 msg.Message.MessageID, msg.Subscription, r.ContentLength)
         bbMsg, err := msg.GetData()
         if err != nil {
                 logging.WithError(err).Errorf(c, "Could not base64 decode message %s", err)
-                return 200
+                return http.StatusOK
         }
         builds, master, err := unmarshal(c, bbMsg)
         if err != nil {
                 logging.WithError(err).Errorf(c, "Could not unmarshal message %s", err)
-                return 200
+                return http.StatusOK
         }
         logging.Infof(c, "There are %d builds", len(builds))
         if master != nil {
                 logging.Infof(c, "The master name is %s", master.Name)
         } else {
                 logging.Infof(c, "No master in this message")
         }
         // This is used to cache the master used for extracting OS information.
         cachedMaster := buildbotMaster{}
         // Do not use PutMulti because we might hit the 1MB limit.
         for _, build := range builds {
                 if build.Master == "" {
                         logging.Errorf(c, "Invalid message, missing master name")
-                        return 200
+                        return http.StatusOK
                 }
                 existingBuild := &buildbotBuild{
                         Master:      build.Master,
                         Buildername: build.Buildername,
                         Number:      build.Number,
                 }
                 buildExists := false
                 if err := ds.Get(c, existingBuild); err == nil {
                         if existingBuild.Finished {
                                 // Never replace a completed build.
@@ skipping 17 matching lines @@
                 build.Internal = internal
                 // Try to get the OS information on a best-effort basis.  This assumes that all
                 // builds come from one master.
                 build.OSFamily, build.OSVersion = getOSInfo(c, build, &cachedMaster)
                 err = ds.Put(c, build)
                 if err != nil {
                         if _, ok := err.(errTooBig); ok {
                                 // This will never work, we don't want PubSub to retry.
                                 logging.WithError(err).Errorf(
                                         c, "Could not save build to datastore, failing permanently")
-                                return 200
+                                return http.StatusOK
                         }
                         // This is transient, we do want PubSub to retry.
                         logging.WithError(err).Errorf(c, "Could not save build in datastore")
-                        return 500
+                        return http.StatusInternalServerError
                 }
                 if buildExists {
                         buildCounter.Add(
                                 c, 1, false, build.Master, build.Buildername, build.Finished, "Replaced")
                 } else {
                         buildCounter.Add(
                                 c, 1, false, build.Master, build.Buildername, build.Finished, "New")
                 }
 
         }
         if master != nil {
                 code := doMaster(c, master, internal)
                 if code != 0 {
                         return code
                 }
         }
-        return 200
+        return http.StatusOK
 }