[milo] {buildbucket,buildbot,swarming,logdog} -> backends/*.

milo/appengine/buildbot/pubsub_test.go

Index: milo/appengine/buildbot/pubsub_test.go
diff --git a/milo/appengine/buildbot/pubsub_test.go b/milo/appengine/buildbot/pubsub_test.go
deleted file mode 100644
index b261f225b8335f488459c908c07827fcf6501497..0000000000000000000000000000000000000000
--- a/milo/appengine/buildbot/pubsub_test.go
+++ /dev/null
@@ -1,494 +0,0 @@
-// Copyright 2016 The LUCI Authors. All rights reserved.
-// Use of this source code is governed under the Apache License, Version 2.0
-// that can be found in the LICENSE file.
-
-package buildbot
-
-import (
-	"bytes"
-	"compress/zlib"
-	"encoding/base64"
-	"encoding/json"
-	"io"
-	"io/ioutil"
-	"math/rand"
-	"net/http"
-	"net/http/httptest"
-	"testing"
-	"time"
-
-	"github.com/luci/gae/impl/memory"
-	ds "github.com/luci/gae/service/datastore"
-	"github.com/luci/luci-go/common/clock/testclock"
-	memcfg "github.com/luci/luci-go/common/config/impl/memory"
-	"github.com/luci/luci-go/common/logging/gologger"
-	"github.com/luci/luci-go/luci_config/server/cfgclient/backend/testconfig"
-	"github.com/luci/luci-go/milo/appengine/common"
-	"github.com/luci/luci-go/server/auth"
-	"github.com/luci/luci-go/server/auth/authtest"
-	"github.com/luci/luci-go/server/auth/identity"
-	"github.com/luci/luci-go/server/router"
-
-	"github.com/julienschmidt/httprouter"
-	"golang.org/x/net/context"
-
-	. "github.com/luci/luci-go/common/testing/assertions"
-	. "github.com/smartystreets/goconvey/convey"
-)
-
-var (
-	fakeTime    = time.Date(2001, time.February, 3, 4, 5, 6, 7, time.UTC)
-	letterRunes = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
-)
-
-func RandStringRunes(n int) string {
-	b := make([]rune, n)
-	for i := range b {
-		b[i] = letterRunes[rand.Intn(len(letterRunes))]
-	}
-	return string(b)
-}
-
-func buildbotTimesFinished(start, end float64) []*float64 {
-	return []*float64{&start, &end}
-}
-
-func buildbotTimesPending(start float64) []*float64 {
-	return []*float64{&start, nil}
-}
-
-func newCombinedPsBody(bs []*buildbotBuild, m *buildbotMaster, internal bool) io.ReadCloser {
-	bmsg := buildMasterMsg{
-		Master: m,
-		Builds: bs,
-	}
-	bm, _ := json.Marshal(bmsg)
-	var b bytes.Buffer
-	zw := zlib.NewWriter(&b)
-	zw.Write(bm)
-	zw.Close()
-	sub := "projects/luci-milo/subscriptions/buildbot-public"
-	if internal {
-		sub = "projects/luci-milo/subscriptions/buildbot-private"
-	}
-	msg := pubSubSubscription{
-		Subscription: sub,
-		Message: pubSubMessage{
-			Data: base64.StdEncoding.EncodeToString(b.Bytes()),
-		},
-	}
-	jmsg, _ := json.Marshal(msg)
-	return ioutil.NopCloser(bytes.NewReader(jmsg))
-}
-
-func TestPubSub(t *testing.T) {
-	Convey(`A test Environment`, t, func() {
-		c := memory.UseWithAppID(context.Background(), "dev~luci-milo")
-		c = gologger.StdConfig.Use(c)
-		c, _ = testclock.UseTime(c, fakeTime)
-		c = testconfig.WithCommonClient(c, memcfg.New(bbAclConfigs))
-		c = auth.WithState(c, &authtest.FakeState{
-			Identity:       identity.AnonymousIdentity,
-			IdentityGroups: []string{"all"},
-		})
-		// Update the service config so that the settings are loaded.
-		err := common.UpdateServiceConfig(c)
-		So(err, ShouldBeNil)
-
-		rand.Seed(5)
-
-		Convey("Remove source changes", func() {
-			m := &buildbotMaster{
-				Name: "fake",
-				Builders: map[string]*buildbotBuilder{
-					"fake builder": {
-						PendingBuildStates: []*buildbotPending{
-							{
-								Source: buildbotSourceStamp{
-									Changes: []buildbotChange{{Comments: "foo"}},
-								},
-							},
-						},
-					},
-				},
-			}
-			So(putDSMasterJSON(c, m, false), ShouldBeNil)
-			lm, _, _, err := getMasterJSON(c, "fake")
-			So(err, ShouldBeNil)
-			So(lm.Builders["fake builder"].PendingBuildStates[0].Source.Changes[0].Comments, ShouldResemble, "")
-		})
-
-		Convey("Save build entry", func() {
-			build := &buildbotBuild{
-				Master:      "Fake Master",
-				Buildername: "Fake buildername",
-				Number:      1234,
-				Currentstep: "this is a string",
-				Finished:    true,
-			}
-			err := ds.Put(c, build)
-			ds.GetTestable(c).CatchupIndexes()
-
-			So(err, ShouldBeNil)
-			Convey("Load build entry", func() {
-				loadB := &buildbotBuild{
-					Master:      "Fake Master",
-					Buildername: "Fake buildername",
-					Number:      1234,
-				}
-				err = ds.Get(c, loadB)
-				So(err, ShouldBeNil)
-				So(loadB.Master, ShouldEqual, "Fake Master")
-				So(loadB.Internal, ShouldEqual, false)
-				So(loadB.Currentstep.(string), ShouldEqual, "this is a string")
-				So(loadB.Finished, ShouldEqual, true)
-			})
-
-			Convey("Query build entry", func() {
-				q := ds.NewQuery("buildbotBuild")
-				buildbots := []*buildbotBuild{}
-				err = ds.GetAll(c, q, &buildbots)
-				So(err, ShouldBeNil)
-
-				So(len(buildbots), ShouldEqual, 1)
-				So(buildbots[0].Currentstep.(string), ShouldEqual, "this is a string")
-				Convey("Query for finished entries should be 1", func() {
-					q = q.Eq("finished", true)
-					buildbots = []*buildbotBuild{}
-					err = ds.GetAll(c, q, &buildbots)
-					So(err, ShouldBeNil)
-					So(len(buildbots), ShouldEqual, 1)
-				})
-				Convey("Query for unfinished entries should be 0", func() {
-					q = q.Eq("finished", false)
-					buildbots = []*buildbotBuild{}
-					err = ds.GetAll(c, q, &buildbots)
-					So(err, ShouldBeNil)
-					So(len(buildbots), ShouldEqual, 0)
-				})
-			})
-
-			Convey("Save a few more entries", func() {
-				ds.GetTestable(c).Consistent(true)
-				ds.GetTestable(c).AutoIndex(true)
-				for i := 1235; i < 1240; i++ {
-					build := &buildbotBuild{
-						Master:      "Fake Master",
-						Buildername: "Fake buildername",
-						Number:      i,
-						Currentstep: "this is a string",
-						Finished:    i%2 == 0,
-					}
-					err := ds.Put(c, build)
-					So(err, ShouldBeNil)
-				}
-				q := ds.NewQuery("buildbotBuild")
-				q = q.Eq("finished", true)
-				q = q.Eq("master", "Fake Master")
-				q = q.Eq("builder", "Fake buildername")
-				q = q.Order("-number")
-				buildbots := []*buildbotBuild{}
-				err = ds.GetAll(c, q, &buildbots)
-				So(err, ShouldBeNil)
-				So(len(buildbots), ShouldEqual, 3) // 1235, 1237, 1239
-			})
-		})
-
-		Convey("Build panics on invalid query", func() {
-			build := &buildbotBuild{
-				Master: "Fake Master",
-			}
-			So(func() { ds.Put(c, build) }, ShouldPanicLike, "No Master or Builder found")
-		})
-
-		ts := 555
-		b := &buildbotBuild{
-			Master:      "Fake Master",
-			Buildername: "Fake buildername",
-			Number:      1234,
-			Currentstep: "this is a string",
-			Times:       buildbotTimesPending(123.0),
-			TimeStamp:   &ts,
-		}
-
-		Convey("Basic master + build pusbsub subscription", func() {
-			h := httptest.NewRecorder()
-			slaves := map[string]*buildbotSlave{}
-			ft := 1234.0
-			slaves["testslave"] = &buildbotSlave{
-				Name:      "testslave",
-				Connected: true,
-				Runningbuilds: []*buildbotBuild{
-					{
-						Master:      "Fake Master",
-						Buildername: "Fake buildername",
-						Number:      2222,
-						Times:       []*float64{&ft, nil},
-					},
-				},
-			}
-			ms := buildbotMaster{
-				Name:     "Fake Master",
-				Project:  buildbotProject{Title: "some title"},
-				Slaves:   slaves,
-				Builders: map[string]*buildbotBuilder{},
-			}
-
-			ms.Builders["Fake buildername"] = &buildbotBuilder{
-				CurrentBuilds: []int{1234},
-			}
-			r := &http.Request{
-				Body: newCombinedPsBody([]*buildbotBuild{b}, &ms, false),
-			}
-			p := httprouter.Params{}
-			PubSubHandler(&router.Context{
-				Context: c,
-				Writer:  h,
-				Request: r,
-				Params:  p,
-			})
-			So(h.Code, ShouldEqual, 200)
-			Convey("And stores correctly", func() {
-				loadB := &buildbotBuild{
-					Master:      "Fake Master",
-					Buildername: "Fake buildername",
-					Number:      1234,
-				}
-				err := ds.Get(c, loadB)
-				So(err, ShouldBeNil)
-				So(loadB.Master, ShouldEqual, "Fake Master")
-				So(loadB.Currentstep.(string), ShouldEqual, "this is a string")
-				m, _, t, err := getMasterJSON(c, "Fake Master")
-				So(err, ShouldBeNil)
-				So(t.Unix(), ShouldEqual, 981173106)
-				So(m.Name, ShouldEqual, "Fake Master")
-				So(m.Project.Title, ShouldEqual, "some title")
-				So(m.Slaves["testslave"].Name, ShouldEqual, "testslave")
-				So(len(m.Slaves["testslave"].Runningbuilds), ShouldEqual, 0)
-				So(len(m.Slaves["testslave"].RunningbuildsMap), ShouldEqual, 1)
-				So(m.Slaves["testslave"].RunningbuildsMap["Fake buildername"][0],
-					ShouldEqual, 2222)
-			})
-
-			Convey("And a new master overwrites", func() {
-				c, _ = testclock.UseTime(c, fakeTime.Add(time.Duration(1*time.Second)))
-				ms.Project.Title = "some other title"
-				h = httptest.NewRecorder()
-				r := &http.Request{
-					Body: newCombinedPsBody([]*buildbotBuild{b}, &ms, false)}
-				p = httprouter.Params{}
-				PubSubHandler(&router.Context{
-					Context: c,
-					Writer:  h,
-					Request: r,
-					Params:  p,
-				})
-				So(h.Code, ShouldEqual, 200)
-				m, _, t, err := getMasterJSON(c, "Fake Master")
-				So(err, ShouldBeNil)
-				So(m.Project.Title, ShouldEqual, "some other title")
-				So(t.Unix(), ShouldEqual, 981173107)
-				So(m.Name, ShouldEqual, "Fake Master")
-			})
-			Convey("And a new build overwrites", func() {
-				b.Times = buildbotTimesFinished(123.0, 124.0)
-				h = httptest.NewRecorder()
-				r = &http.Request{
-					Body: newCombinedPsBody([]*buildbotBuild{b}, &ms, false),
-				}
-				p = httprouter.Params{}
-				PubSubHandler(&router.Context{
-					Context: c,
-					Writer:  h,
-					Request: r,
-					Params:  p,
-				})
-				So(h.Code, ShouldEqual, 200)
-				loadB := &buildbotBuild{
-					Master:      "Fake Master",
-					Buildername: "Fake buildername",
-					Number:      1234,
-				}
-				err := ds.Get(c, loadB)
-				So(err, ShouldBeNil)
-				So(*loadB.Times[0], ShouldEqual, 123.0)
-				So(*loadB.Times[1], ShouldEqual, 124.0)
-				Convey("And another pending build is rejected", func() {
-					b.Times = buildbotTimesPending(123.0)
-					h = httptest.NewRecorder()
-					r = &http.Request{
-						Body: newCombinedPsBody([]*buildbotBuild{b}, &ms, false),
-					}
-					p = httprouter.Params{}
-					PubSubHandler(&router.Context{
-						Context: c,
-						Writer:  h,
-						Request: r,
-						Params:  p,
-					})
-					So(h.Code, ShouldEqual, 200)
-					loadB := &buildbotBuild{
-						Master:      "Fake Master",
-						Buildername: "Fake buildername",
-						Number:      1234,
-					}
-					err := ds.Get(c, loadB)
-					So(err, ShouldBeNil)
-					So(*loadB.Times[0], ShouldEqual, 123.0)
-					So(*loadB.Times[1], ShouldEqual, 124.0)
-				})
-			})
-			Convey("Don't Expire non-existant current build under 20 min", func() {
-				b.Number = 1235
-				ts := int(fakeTime.Unix()) - 1000
-				b.TimeStamp = &ts
-				h = httptest.NewRecorder()
-				r = &http.Request{
-					Body: newCombinedPsBody([]*buildbotBuild{b}, &ms, false),
-				}
-				p = httprouter.Params{}
-				ds.GetTestable(c).Consistent(true)
-				PubSubHandler(&router.Context{
-					Context: c,
-					Writer:  h,
-					Request: r,
-					Params:  p,
-				})
-				So(h.Code, ShouldEqual, 200)
-				loadB := &buildbotBuild{
-					Master:      "Fake Master",
-					Buildername: "Fake buildername",
-					Number:      1235,
-				}
-				err := ds.Get(c, loadB)
-				So(err, ShouldBeNil)
-				So(loadB.Finished, ShouldEqual, false)
-				So(*loadB.Times[0], ShouldEqual, 123.0)
-				So(loadB.Times[1], ShouldBeNil)
-			})
-			Convey("Expire non-existant current build", func() {
-				b.Number = 1235
-				ts := int(fakeTime.Unix()) - 1201
-				b.TimeStamp = &ts
-				h = httptest.NewRecorder()
-				r = &http.Request{
-					Body: newCombinedPsBody([]*buildbotBuild{b}, &ms, false),
-				}
-				p = httprouter.Params{}
-				ds.GetTestable(c).Consistent(true)
-				PubSubHandler(&router.Context{
-					Context: c,
-					Writer:  h,
-					Request: r,
-					Params:  p,
-				})
-				So(h.Code, ShouldEqual, 200)
-				loadB := &buildbotBuild{
-					Master:      "Fake Master",
-					Buildername: "Fake buildername",
-					Number:      1235,
-				}
-				err := ds.Get(c, loadB)
-				So(err, ShouldBeNil)
-				So(loadB.Finished, ShouldEqual, true)
-				So(*loadB.Times[0], ShouldEqual, 123.0)
-				So(loadB.Times[1], ShouldNotEqual, nil)
-				So(*loadB.Times[1], ShouldEqual, ts)
-				So(*loadB.Results, ShouldEqual, 4)
-			})
-			Convey("Large pubsub message", func() {
-				// This has to be a random string, so that after gzip compresses it
-				// it remains larger than 950KB
-				b.Text = append(b.Text, RandStringRunes(1500000))
-				h := httptest.NewRecorder()
-				r := &http.Request{
-					Body: newCombinedPsBody([]*buildbotBuild{b}, &ms, false),
-				}
-				p := httprouter.Params{}
-				PubSubHandler(&router.Context{
-					Context: c,
-					Writer:  h,
-					Request: r,
-					Params:  p,
-				})
-				So(h.Code, ShouldEqual, 200)
-			})
-
-		})
-
-		Convey("Empty pubsub message", func() {
-			h := httptest.NewRecorder()
-			r := &http.Request{Body: ioutil.NopCloser(bytes.NewReader([]byte{}))}
-			p := httprouter.Params{}
-			PubSubHandler(&router.Context{
-				Context: c,
-				Writer:  h,
-				Request: r,
-				Params:  p,
-			})
-			So(h.Code, ShouldEqual, 200)
-		})
-
-		Convey("Internal master + build pusbsub subscription", func() {
-			h := httptest.NewRecorder()
-			slaves := map[string]*buildbotSlave{}
-			ft := 1234.0
-			slaves["testslave"] = &buildbotSlave{
-				Name:      "testslave",
-				Connected: true,
-				Runningbuilds: []*buildbotBuild{
-					{
-						Master:      "Fake Master",
-						Buildername: "Fake buildername",
-						Number:      2222,
-						Times:       []*float64{&ft, nil},
-					},
-				},
-			}
-			ms := buildbotMaster{
-				Name:    "Fake Master",
-				Project: buildbotProject{Title: "some title"},
-				Slaves:  slaves,
-			}
-			r := &http.Request{
-				Body: newCombinedPsBody([]*buildbotBuild{b}, &ms, true),
-			}
-			p := httprouter.Params{}
-			PubSubHandler(&router.Context{
-				Context: c,
-				Writer:  h,
-				Request: r,
-				Params:  p,
-			})
-			So(h.Code, ShouldEqual, 200)
-			Convey("And stores correctly", func() {
-				err := common.UpdateProjectConfigs(c)
-				So(err, ShouldBeNil)
-				c = auth.WithState(c, &authtest.FakeState{
-					Identity:       "user:alicebob@google.com",
-					IdentityGroups: []string{"googlers", "all"},
-				})
-				loadB := &buildbotBuild{
-					Master:      "Fake Master",
-					Buildername: "Fake buildername",
-					Number:      1234,
-				}
-				err = ds.Get(c, loadB)
-				So(err, ShouldBeNil)
-				So(loadB.Master, ShouldEqual, "Fake Master")
-				So(loadB.Internal, ShouldEqual, true)
-				So(loadB.Currentstep.(string), ShouldEqual, "this is a string")
-				m, _, t, err := getMasterJSON(c, "Fake Master")
-				So(err, ShouldBeNil)
-				So(t.Unix(), ShouldEqual, 981173106)
-				So(m.Name, ShouldEqual, "Fake Master")
-				So(m.Project.Title, ShouldEqual, "some title")
-				So(m.Slaves["testslave"].Name, ShouldEqual, "testslave")
-				So(len(m.Slaves["testslave"].Runningbuilds), ShouldEqual, 0)
-				So(len(m.Slaves["testslave"].RunningbuildsMap), ShouldEqual, 1)
-				So(m.Slaves["testslave"].RunningbuildsMap["Fake buildername"][0],
-					ShouldEqual, 2222)
-			})
-		})
-	})
-}