Milo: Create build summaries for buildbot in the pubsub pipeline

milo/buildsource/buildbot/pubsub.go

 // Copyright 2016 The LUCI Authors.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
 //      http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
 package buildbot
 
 import (
         "bytes"
         "compress/gzip"
         "compress/zlib"
         "encoding/json"
         "fmt"
         "net/http"
         "strings"
         "time"
 
-        ds "github.com/luci/gae/service/datastore"
+        "github.com/luci/gae/service/datastore"
         "github.com/luci/luci-go/common/clock"
         "github.com/luci/luci-go/common/iotools"
         "github.com/luci/luci-go/common/logging"
         "github.com/luci/luci-go/milo/common"
+        "github.com/luci/luci-go/milo/common/model"
         "github.com/luci/luci-go/server/router"
 
         "golang.org/x/net/context"
 
         "github.com/luci/luci-go/common/tsmon/field"
         "github.com/luci/luci-go/common/tsmon/metric"
 )
 
 var (
         // Metrics
@@ skipping 58 matching lines @@
         entry := buildbotMasterEntry{
                 Name:     master.Name,
                 Internal: internal,
                 Modified: clock.Now(c).UTC(),
         }
         toPut := []interface{}{&entry}
         publicTag := &buildbotMasterPublic{master.Name}
         if internal {
                 // do the deletion immediately so that the 'public' bit is removed from
                 // datastore before any internal details are actually written to datastore.
-                if err := ds.Delete(c, publicTag); err != nil && err != ds.ErrNoSuchEntity {
+                if err := datastore.Delete(c, publicTag); err != nil && err != datastore.ErrNoSuchEntity {
                         return err
                 }
         } else {
                 toPut = append(toPut, publicTag)
         }
         gzbs := bytes.Buffer{}
         gsw := gzip.NewWriter(&gzbs)
         cw := iotools.CountingWriter{Writer: gsw}
         e := json.NewEncoder(&cw)
         if err := e.Encode(master); err != nil {
                 return err
         }
         gsw.Close()
         entry.Data = gzbs.Bytes()
         logging.Debugf(c, "Length of json data: %d", cw.Count)
         logging.Debugf(c, "Length of gzipped data: %d", len(entry.Data))
-        return ds.Put(c, toPut)
+        return datastore.Put(c, toPut)
 }
 
 // unmarshal a gzipped byte stream into a list of buildbot builds and masters.
 func unmarshal(
         c context.Context, msg []byte) ([]*buildbotBuild, *buildbotMaster, error) {
         bm := buildMasterMsg{}
         if len(msg) == 0 {
                 return bm.Builds, bm.Master, nil
         }
         reader, err := zlib.NewReader(bytes.NewReader(msg))
@@ skipping 26 matching lines @@
 }
 
 // getOSInfo fetches the os family and version of the slave the build was
 // running on from the master json on a best-effort basis.
 func getOSInfo(c context.Context, b *buildbotBuild, m *buildbotMaster) (
         family, version string) {
         // Fetch the master info from datastore if not provided.
         if m.Name == "" {
                 logging.Infof(c, "Fetching info for master %s", b.Master)
                 entry := buildbotMasterEntry{Name: b.Master}
-                err := ds.Get(c, &entry)
+                err := datastore.Get(c, &entry)
                 if err != nil {
                         logging.WithError(err).Errorf(
                                 c, "Encountered error while fetching entry for %s", b.Master)
                         return
                 }
                 err = decodeMasterEntry(c, &entry, m)
                 if err != nil {
                         logging.WithError(err).Warningf(
                                 c, "Failed to decode master information for OS info on master %s", b.Master)
                         return
@@ skipping 30 matching lines @@
         finished := float64(clock.Now(c).Unix())
         if b.TimeStamp != nil {
                 finished = float64(*b.TimeStamp)
         }
         results := int(4) // Exception
         b.Times[1] = &finished
         b.Finished = true
         b.Results = &results
         b.Currentstep = nil
         b.Text = append(b.Text, "Build expired on Milo")
-        return ds.Put(c, b)
+        return datastore.Put(c, b)
+}
+
+// saveBuildSummary summerizes a build into a model.BuildSummary and then saves it.
+func saveBuildSummary(c context.Context, b *buildbotBuild) error {
+        resp := renderBuild(c, b)
+        bs := model.BuildSummary{
+                BuildKey:  datastore.KeyForObj(c, b),
+                BuilderID: fmt.Sprintf("buildbot/%s/%s", b.Master, b.Buildername),
+        }
+        resp.SummarizeTo(&bs)
+        return datastore.Put(c, &bs)
 }
 
 func doMaster(c context.Context, master *buildbotMaster, internal bool) int {
         // Store the master json into the datastore.
         err := putDSMasterJSON(c, master, internal)
         fullname := fmt.Sprintf("master.%s", master.Name)
         if err != nil {
                 logging.WithError(err).Errorf(
                         c, "Could not save master in datastore %s", err)
                 masterCounter.Add(c, 1, internal, fullname, "failure")
                 // This is transient, we do want PubSub to retry.
                 return http.StatusInternalServerError
         }
         masterCounter.Add(c, 1, internal, fullname, "success")
 
         // Extract current builds data out of the master json, and use it to
         // clean up expired builds.
-        q := ds.NewQuery("buildbotBuild").
+        q := datastore.NewQuery("buildbotBuild").
                 Eq("finished", false).
                 Eq("master", master.Name)
         builds := []*buildbotBuild{}
         err = getBuildQueryBatcher(c).GetAll(c, q, &builds)
         if err != nil {
                 logging.WithError(err).Errorf(c, "Could not load current builds from master %s",
                         master.Name)
                 return http.StatusInternalServerError
         }
         for _, b := range builds {
@@ skipping 101 matching lines @@
                 if build.Master == "" {
                         logging.Errorf(c, "Invalid message, missing master name")
                         return http.StatusOK
                 }
                 existingBuild := &buildbotBuild{
                         Master:      build.Master,
                         Buildername: build.Buildername,
                         Number:      build.Number,
                 }
                 buildExists := false
-                if err := ds.Get(c, existingBuild); err == nil {
+                if err := datastore.Get(c, existingBuild); err == nil {
                         if existingBuild.Finished {
                                 // Never replace a completed build.
                                 buildCounter.Add(
                                         c, 1, false, build.Master, build.Buildername, false, "Rejected")
                                 continue
                         }
                         buildExists = true
                 }
                 // Also set the finished, timestamp, and internal bit.
                 build.Finished = false
                 if build.TimeStamp == nil {
                         build.TimeStamp = &now
                 }
                 if len(build.Times) == 2 && build.Times[1] != nil {
                         build.Finished = true
                         logging.Infof(
                                 c, "Recording finished build %s/%s/%d", build.Master,
                                 build.Buildername, build.Number)
                 }
                 build.Internal = internal
                 // Try to get the OS information on a best-effort basis.  This assumes that all
                 // builds come from one master.
                 build.OSFamily, build.OSVersion = getOSInfo(c, build, &cachedMaster)
-                err = ds.Put(c, build)
+                err = datastore.Put(c, build)
                 if err != nil {
                         if _, ok := err.(errTooBig); ok {
                                 // This will never work, we don't want PubSub to retry.
                                 logging.WithError(err).Errorf(
                                         c, "Could not save build to datastore, failing permanently")
                                 return http.StatusOK
                         }
                         // This is transient, we do want PubSub to retry.
                         logging.WithError(err).Errorf(c, "Could not save build in datastore")
                         return http.StatusInternalServerError
                 }
+                err = saveBuildSummary(c, build)
+                if err != nil {
+                        logging.WithError(err).Errorf(c, "could not save build summary into datastore")
+                        return http.StatusInternalServerError
+                }
                 if buildExists {
                         buildCounter.Add(
                                 c, 1, false, build.Master, build.Buildername, build.Finished, "Replaced")
                 } else {
                         buildCounter.Add(
                                 c, 1, false, build.Master, build.Buildername, build.Finished, "New")
                 }
 
         }
         if master != nil {
                 code := doMaster(c, master, internal)
                 if code != 0 {
                         return code
                 }
         }
         return http.StatusOK
 }