Remove the milo/buildbot backfiller

appengine/cmd/milo/collectors/buildbot/collector.go

Index: appengine/cmd/milo/collectors/buildbot/collector.go
diff --git a/appengine/cmd/milo/collectors/buildbot/collector.go b/appengine/cmd/milo/collectors/buildbot/collector.go
deleted file mode 100644
index 1378ebeb9902cf4cc5efe6b37c0dd8f7dc3e9b01..0000000000000000000000000000000000000000
--- a/appengine/cmd/milo/collectors/buildbot/collector.go
+++ /dev/null
@@ -1,255 +0,0 @@
-// Copyright 2015 The LUCI Authors. All rights reserved.
-// Use of this source code is governed under the Apache License, Version 2.0
-// that can be found in the LICENSE file.
-
-package buildbot
-
-import (
-	"encoding/json"
-	"fmt"
-	"net/http"
-	"strings"
-	"time"
-
-	"github.com/luci/gae/service/datastore"
-	"github.com/luci/luci-go/appengine/cmd/milo/model"
-	log "github.com/luci/luci-go/common/logging"
-	"github.com/luci/luci-go/common/parallel"
-	"golang.org/x/net/context"
-)
-
-// BuildExtractURL is the URL of Chrome Build extract, which caches buildbot
-// json.
-const BuildExtractURL = "https://chrome-build-extract.appspot.com"
-
-const workPoolSizeMultiplier = 2
-
-type changeJSON struct {
-	Repository string `json:"repository"`
-	Digest     string `json:"revision"`
-	UnixTime   int64  `json:"when"`
-}
-
-type sourceJSON struct {
-	Changes []changeJSON `json:"changes"`
-}
-
-type buildJSON struct {
-	// Times elements are a float64 unix time that the build was started. It's
-	// an array because buildbot :)
-	Times       []float64  `json:"times"`
-	Builder     string     `json:"builderName"`
-	Master      string     `json:"masterName"`
-	Error       string     `json:"error"`
-	Logs        [][]string `json:"logs"`
-	SourceStamp sourceJSON `json:"sourceStamp"`
-	Number      int        `json:"number"`
-	Text        []string   `json:"text"`
-}
-
-func getRevForChange(c context.Context, change changeJSON) model.RevisionInfo {
-	repo := model.GetRepository(c, change.Repository)
-
-	generation := -1
-	if rev, err := repo.GetRevision(c, change.Digest); err == nil {
-		generation = rev.Generation
-	}
-
-	return model.RevisionInfo{
-		Repository: change.Repository,
-		Digest:     change.Digest,
-		Generation: generation,
-	}
-}
-
-func (b *buildJSON) toDS(c context.Context) *model.Build {
-	perRepo := make(map[string]*changeJSON)
-	for _, change := range b.SourceStamp.Changes {
-		current := perRepo[change.Repository]
-		if current != nil {
-			if current.UnixTime < change.UnixTime {
-				perRepo[change.Repository] = &change
-			}
-		} else {
-			perRepo[change.Repository] = &change
-		}
-	}
-
-	revisions := make([]model.RevisionInfo, 0, len(perRepo))
-	for _, change := range perRepo {
-		revisions = append(revisions, getRevForChange(c, *change))
-	}
-
-	logKey := ""
-	if len(b.Logs) > 1 && len(b.Logs[1]) > 1 {
-		logKey = b.Logs[1][1]
-	} else {
-		log.Warningf(c, "build %d on %s master %s had weird logs\n", b.Number, b.Builder, b.Master)
-	}
-
-	userStatus := "UNKNOWN"
-	if len(b.Text) < 2 {
-		log.Warningf(c, "build %d on %s master %s had strange text\n", b.Number, b.Builder, b.Master)
-	} else {
-		if b.Text[0] == "exception" {
-			userStatus = "EXCEPTION"
-		} else if b.Text[0] == "failed" {
-			userStatus = "FAILURE"
-		} else if b.Text[1] == "successful" {
-			userStatus = "SUCCESS"
-		}
-		if userStatus == "UNKNOWN" {
-			log.Warningf(c, "build %d on %s master %s had unknown text %v\n", b.Number, b.Builder, b.Master, b.Text)
-		}
-	}
-
-	return &model.Build{
-		ExecutionTime: model.TimeID{time.Unix(int64(b.Times[0]), 0).UTC()},
-		BuildRoot:     model.GetBuildRoot(c, b.Master, b.Builder).Key,
-		BuildLogKey:   logKey,
-		Revisions:     revisions,
-		UserStatus:    userStatus,
-	}
-}
-
-type builderJSON struct {
-	Builds []int `json:"cachedBuilds"`
-}
-
-type masterJSON struct {
-	Builders map[string]builderJSON `json:"builders"`
-}
-
-func getBuild(master, builder string, build int, useCBE bool) (*buildJSON, error) {
-	resp := (*http.Response)(nil)
-	err := (error)(nil)
-	if useCBE {
-		resp, err = http.Get(strings.Join(
-			[]string{BuildExtractURL, "p", master, "builders", builder, "builds", fmt.Sprintf("%d", build)},
-			"/") + "?json=1")
-	} else {
-		resp, err = http.Get(strings.Join(
-			[]string{"https://build.chromium.org", "p", master, "json", "builders", builder, "builds", fmt.Sprintf("%d", build)},
-			"/"))
-	}
-
-	if err != nil {
-		return nil, err
-	}
-	defer resp.Body.Close()
-
-	dec := json.NewDecoder(resp.Body)
-	doneBuild := &buildJSON{}
-	err = dec.Decode(doneBuild)
-	if err != nil {
-		return nil, err
-	}
-
-	if doneBuild.Error != "" {
-		return nil, fmt.Errorf("error while getting Build for master %s builder %s build number %d: %s", master, builder, build, doneBuild.Error)
-	}
-	return doneBuild, nil
-}
-
-// PopulateMaster puts the data for a master into the datastore.
-//
-// It hits CBE to grab the json, and then creates corresponding entities and
-// puts them into the datastore. It assumes that a datastore impl exists in the
-// supplied context.
-func PopulateMaster(c context.Context, master string, dryRun, buildbotFallback bool) error {
-	masterJSON := &masterJSON{}
-	resp, err := http.Get(strings.Join([]string{BuildExtractURL, "get_master", master}, "/"))
-	if err != nil {
-		return err
-	}
-
-	defer resp.Body.Close()
-	dec := json.NewDecoder(resp.Body)
-	err = dec.Decode(masterJSON)
-	if err != nil {
-		return err
-	}
-
-	buildsToPut := make(map[string][]*model.Build)
-	log.Infof(c, "Getting builds for master %s\n", master)
-
-	errors := parallel.WorkPool(workPoolSizeMultiplier*len(masterJSON.Builders), func(ch chan<- func() error) {
-		for builderName, builder := range masterJSON.Builders {
-			if len(builder.Builds) > 0 {
-				buildsToPut[builderName] = make([]*model.Build, len(builder.Builds))
-
-				for ind, builderNum := range builder.Builds {
-					ind := ind
-					builderNum := builderNum
-					builderName := builderName
-
-					ch <- func() error {
-						res, err := getBuild(master, builderName, builderNum, true)
-						if err != nil {
-							if !buildbotFallback {
-								log.Warningf(c, "got %s, giving up", err)
-								return err
-							}
-
-							log.Warningf(c, "got %s, retrying without CBE", err)
-							res, err = getBuild(master, builderName, builderNum, false)
-
-							if err != nil {
-								log.Warningf(c, "got %s, giving up", err)
-								return err
-							}
-						}
-
-						conv := res.toDS(c)
-						buildsToPut[builderName][ind] = conv
-						return nil
-					}
-				}
-				log.Infof(c, "Queued builds for builder %s.", builderName)
-			}
-		}
-	})
-
-	// We ignore these warnings because, for some reason, CBE sometimes returns
-	// 404s for some builds. So we ignore those, and just move on. They're
-	// printed to the user above, so if something is really wrong all those
-	// errors will show up.
-	log.Warningf(c, "Done fetching builds. errors: %s.\n", errors)
-
-	for name, builds := range buildsToPut {
-		num := 0
-		for _, b := range builds {
-			if b != nil {
-				num++
-			}
-		}
-
-		if num == 0 {
-			buildsToPut[name] = nil
-			continue
-		}
-
-		newBuilds := make([]*model.Build, num)
-		ind := 0
-		for _, b := range builds {
-			if b != nil {
-				newBuilds[ind] = b
-				ind++
-			}
-		}
-		buildsToPut[name] = newBuilds
-	}
-
-	ds := datastore.Get(c)
-	for builderName, toPut := range buildsToPut {
-		log.Infof(c, "%d to put for builder %s\n", len(toPut), builderName)
-		if toPut != nil {
-			if dryRun {
-				log.Infof(c, "dry run, not putting any modifications")
-			} else {
-				ds.Put(toPut)
-			}
-		}
-	}
-	return nil
-}