Milo: Clean up builds that are no longer current.

milo/appengine/buildbot/pubsub.go

Index: milo/appengine/buildbot/pubsub.go
diff --git a/milo/appengine/buildbot/pubsub.go b/milo/appengine/buildbot/pubsub.go
index c021f0e26ae02f14f9271b6b107a46d2f6b2761f..f61f39b1c55a3f55dfdc6eec7861e5b5011a3549 100644
--- a/milo/appengine/buildbot/pubsub.go
+++ b/milo/appengine/buildbot/pubsub.go
@@ -180,6 +180,19 @@ func getOSInfo(c context.Context, b *buildbotBuild, m *buildbotMaster) (
 	return
 }
 
+// Marks a build as finished and expired.
+func expireBuild(c context.Context, b *buildbotBuild) error {
+	finished := float64(clock.Now(c).Unix())
+	if b.TimeStamp != nil {
+		finished = float64(*b.TimeStamp)
+	}
+	results := int(2)
+	b.Times[1] = &finished
+	b.Finished = true
+	b.Results = &results
+	return ds.Put(c, b)
+}
+
 // PubSubHandler is a webhook that stores the builds coming in from pubsub.
 func PubSubHandler(ctx *router.Context) {
 	c, h, r := ctx.Context, ctx.Writer, ctx.Request
@@ -283,6 +296,7 @@ func PubSubHandler(ctx *router.Context) {
 
 	}
 	if master != nil {
+		// Store the master json into the datastore.
 		err = putDSMasterJSON(c, master, internal)
 		if err != nil {
 			logging.WithError(err).Errorf(
@@ -291,6 +305,53 @@ func PubSubHandler(ctx *router.Context) {
 			h.WriteHeader(500)
 			return
 		}
+
+		// Extract current builds data out of the master json, and use it to

[estaab, 2016/10/20 03:30:39]

This function is pretty long by this point. Is there another logical block you
can take out of it?

[hinoka, 2016/10/26 01:28:04]

Done.

+		// clean up expired builds.
+		q := ds.NewQuery("buildbotBuild").
+			Eq("finished", false).
+			Eq("master", master.Name)
+		builds := []*buildbotBuild{}
+		err := ds.GetAll(c, q, &builds)
+		if err != nil {
+			logging.WithError(err).Errorf(c, "Could not load current builds from master %s",
+				master.Name)
+			h.WriteHeader(500)
+			return
+		}
+		for _, b := range builds {
+			builder, ok := master.Builders[b.Buildername]
+			if !ok {
+				// Mark this build due to builder being removed.
+				logging.Infof(c, "Expiring %s/%s/%d due to builder being removed",
+					master.Name, b.Buildername, b.Number)
+				err = expireBuild(c, b)
+				if err != nil {
+					logging.WithError(err).Errorf(c, "Could not expire build")
+					h.WriteHeader(500)
+					return
+				}
+			}
+
+			found := false
+			for _, bnum := range builder.CurrentBuilds {
+				if b.Number == bnum {
+					found = true
+					break
+				}
+			}
+			if !found {
+				// Mark this build due to build not current anymore.
+				logging.Infof(c, "Expiring %s/%s/%d due to build not current",
+					master.Name, b.Buildername, b.Number)
+				err = expireBuild(c, b)
+				if err != nil {
+					logging.WithError(err).Errorf(c, "Could not expire build")
+					h.WriteHeader(500)
+					return
+				}
+			}
+		}
 	}
 	h.WriteHeader(200)
 }