milo: buildbucket builer view

appengine/cmd/milo/buildbucket/builder.go

Index: appengine/cmd/milo/buildbucket/builder.go
diff --git a/appengine/cmd/milo/buildbucket/builder.go b/appengine/cmd/milo/buildbucket/builder.go
new file mode 100644
index 0000000000000000000000000000000000000000..afc53a16f2be882bdf94ee5b841b94003abf87fa
--- /dev/null
+++ b/appengine/cmd/milo/buildbucket/builder.go
@@ -0,0 +1,278 @@
+// Copyright 2016 The LUCI Authors. All rights reserved.
+// Use of this source code is governed under the Apache License, Version 2.0
+// that can be found in the LICENSE file.
+
+package buildbucket
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/url"
+	"os"
+	"path"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"time"
+
+	"golang.org/x/net/context"
+	"google.golang.org/api/googleapi"
+
+	"github.com/luci/luci-go/appengine/cmd/milo/resp"
+	"github.com/luci/luci-go/common/api/buildbucket/buildbucket/v1"
+	"github.com/luci/luci-go/common/clock"
+	"github.com/luci/luci-go/common/errors"
+	log "github.com/luci/luci-go/common/logging"
+	"github.com/luci/luci-go/common/parallel"
+	"github.com/luci/luci-go/common/retry"
+)
+
+// search executes the search request with retries and exponential back-off.
+func search(c context.Context, client *buildbucket.Service, req *buildbucket.SearchCall) (
+	*buildbucket.ApiSearchResponseMessage, error) {
+
+	var res *buildbucket.ApiSearchResponseMessage
+	err := retry.Retry(
+		c,
+		retry.TransientOnly(retry.Default),
+		func() error {
+			var err error
+			res, err = req.Do()
+			if apiErr, ok := err.(*googleapi.Error); ok && apiErr.Code >= 500 {
+				err = errors.WrapTransient(apiErr)
+			}
+			return err
+		},
+		func(err error, wait time.Duration) {
+			log.WithError(err).Warningf(c, "buildbucket search request failed transiently, will retry in %s", wait)
+		})
+	return res, err
+}
+
+// fetchBuilds fetches builds given a criteria.
+// The returned builds are sorted by build creation descending.
+// count defines maximum number of builds to fetch; if <0, defaults to 100.
+func fetchBuilds(c context.Context, client *buildbucket.Service, bucket, builder,
+	status string, count int) ([]*buildbucket.ApiBuildMessage, error) {
+
+	req := client.Search()
+	req.Bucket(bucket)
+	req.Status(status)
+	req.Tag("builder:" + builder)
+
+	if count < 0 {
+		count = 100
+	}
+
+	fetched := make([]*buildbucket.ApiBuildMessage, 0, count)
+	start := clock.Now(c)
+	for len(fetched) < count {
+		req.MaxBuilds(int64(count - len(fetched)))
+
+		res, err := search(c, client, req)
+		switch {
+		case err != nil:
+			return fetched, err
+		case res.Error != nil:
+			return fetched, fmt.Errorf(res.Error.Message)
+		}
+
+		fetched = append(fetched, res.Builds...)
+
+		if len(res.Builds) == 0 || res.NextCursor == "" {
+			break
+		}
+		req.StartCursor(res.NextCursor)
+	}
+	log.Debugf(c, "Fetched %d %s builds in %s", len(fetched), status, clock.Since(c, start))
+	return fetched, nil
+}
+
+// toMiloBuild converts a buildbucket build to a milo build.
+// In case of an error, returns a build with a description of the error
+// and logs the error.
+func toMiloBuild(c context.Context, build *buildbucket.ApiBuildMessage) *resp.BuildSummary {
+	// Parsing of parameters and result details is best effort.
+	var params buildParameters
+	if err := json.NewDecoder(strings.NewReader(build.ParametersJson)).Decode(&params); err != nil {
+		log.Errorf(c, "Could not parse parameters of build %d: %s", build.Id, err)
+	}
+	var resultDetails resultDetails
+	if err := json.NewDecoder(strings.NewReader(build.ResultDetailsJson)).Decode(&resultDetails); err != nil {
+		log.Errorf(c, "Could not parse result details of build %d: %s", build.Id, err)
+	}
+
+	result := &resp.BuildSummary{
+		Text:     []string{fmt.Sprintf("buildbucket id %d", build.Id)},
+		Revision: resultDetails.Properties.GotRevision,
+	}
+	if result.Revision == "" {
+		result.Revision = params.Properties.Revision
+	}
+
+	var err error
+	result.Status, err = parseStatus(build)
+	if err != nil {
+		// almost never happens
+		log.WithError(err).Errorf(c, "could not convert status of build %d", build.Id)
+		result.Status = resp.InfraFailure
+		result.Text = append(result.Text, fmt.Sprintf("invalid build: %s", err))
+	}
+
+	result.PendingTime.Started = parseTimestamp(build.CreatedTs)
+	switch build.Status {
+	case "SCHEDULED":
+		result.PendingTime.Duration = clock.Since(c, result.PendingTime.Started)
+
+	case "STARTED":
+		result.ExecutionTime.Started = parseTimestamp(build.StatusChangedTs)
+		result.ExecutionTime.Duration = clock.Since(c, result.PendingTime.Started)
+		result.PendingTime.Finished = result.ExecutionTime.Started
+		result.PendingTime.Duration = result.PendingTime.Finished.Sub(result.PendingTime.Started)
+
+	case "COMPLETED":
+		// buildbucket does not provide build start time or execution duration.
+		result.ExecutionTime.Finished = parseTimestamp(build.CompletedTs)
+	}
+
+	cl := getChangeList(build, &params, &resultDetails)
+	if cl != nil {
+		result.Blame = []*resp.Commit{cl}
+	}
+
+	tags := ParseTags(build.Tags)
+
+	if build.Url != "" {
+		u := build.Url
+		parsed, err := url.Parse(u)
+
+		// map milo links to itself
+		switch {
+		case err != nil:
+			log.Errorf(c, "invalid URL in build %d: %s", build.Id, err)
+		case parsed.Host == "luci-milo.appspot.com":
+			parsed.Host = ""
+			parsed.Scheme = ""
+			u = parsed.String()
+		}
+
+		result.Link = &resp.Link{
+			URL:   u,
+			Label: strconv.FormatInt(build.Id, 10),
+		}
+
+		// compute the best link label
+		if taskID := tags["swarming_task_id"]; taskID != "" {
+			result.Link.Label = taskID
+		} else if resultDetails.Properties.BuildNumber != 0 {
+			result.Link.Label = strconv.Itoa(resultDetails.Properties.BuildNumber)
+		} else if parsed != nil {
+			// does the URL look like a buildbot build URL?
+			pattern := fmt.Sprintf(
+				`/%s/builders/%s/builds/`,
+				strings.TrimPrefix(build.Bucket, "master."), params.BuilderName)
+			beforeBuildNumber, buildNumberStr := path.Split(parsed.Path)
+			_, err := strconv.Atoi(buildNumberStr)
+			if strings.HasSuffix(beforeBuildNumber, pattern) && err == nil {
+				result.Link.Label = buildNumberStr
+			}
+		}
+	}
+
+	return result
+}
+
+func getDebugBuilds(c context.Context, bucket, builder string, maxCompletedBuilds int, target *resp.Builder) error {
+	resFile, err := os.Open(filepath.Join("testdata", "buildbucket", bucket, builder+".json"))
+	if err != nil {
+		return err
+	}
+	defer resFile.Close()
+
+	res := &buildbucket.ApiSearchResponseMessage{}
+	if err := json.NewDecoder(resFile).Decode(res); err != nil {
+		return err
+	}
+
+	for _, bb := range res.Builds {
+		mb := toMiloBuild(c, bb)
+		switch mb.Status {
+		case resp.NotRun:
+			target.PendingBuilds = append(target.PendingBuilds, mb)
+
+		case resp.Running:
+			target.CurrentBuilds = append(target.CurrentBuilds, mb)
+
+		case resp.Success, resp.Failure, resp.InfraFailure, resp.Warning:
+			if len(target.FinishedBuilds) < maxCompletedBuilds {
+				target.FinishedBuilds = append(target.FinishedBuilds, mb)
+			}
+
+		default:
+			panic("impossible")
+		}
+	}
+	return nil
+}
+
+// builderImpl is the implementation for getting a milo builder page from buildbucket.
+// if maxCompletedBuilds < 0, 25 is used.
+func builderImpl(c context.Context, server, bucket, builder string, maxCompletedBuilds int) (*resp.Builder, error) {
+	if maxCompletedBuilds < 0 {
+		maxCompletedBuilds = 20
+	}
+
+	result := &resp.Builder{
+		Name: builder,
+	}
+	if server == "debug" {
+		return result, getDebugBuilds(c, bucket, builder, maxCompletedBuilds, result)
+	}
+	client, err := newClient(c, server)
+	if err != nil {
+		return nil, err
+	}
+
+	fetch := func(target *[]*resp.BuildSummary, status string, count int) error {
+		builds, err := fetchBuilds(c, client, bucket, builder, status, count)
+		if err != nil {
+			log.Errorf(c, "Could not fetch builds with status %s: %s", status, err)
+			return err
+		}
+		*target = make([]*resp.BuildSummary, len(builds))
+		for i, bb := range builds {
+			(*target)[i] = toMiloBuild(c, bb)
+		}
+		return nil
+	}
+	// fetch pending, current and finished builds concurrently.
+	// Why not a single request? Because we need different build number
+	// limits for different statuses.
+	return result, parallel.FanOutIn(func(work chan<- func() error) {
+		work <- func() error {
+			return fetch(&result.PendingBuilds, StatusScheduled, -1)
+		}
+		work <- func() error {
+			return fetch(&result.CurrentBuilds, StatusStarted, -1)
+		}
+		work <- func() error {
+			return fetch(&result.FinishedBuilds, StatusCompleted, maxCompletedBuilds)
+		}
+	})
+}
+
+// parseTimestamp converts buildbucket timestamp in microseconds to time.Time
+func parseTimestamp(microseconds int64) time.Time {
+	if microseconds == 0 {
+		return time.Time{}
+	}
+	return time.Unix(microseconds/1e6, microseconds%1e6*1000).UTC()
+}
+
+type newBuildsFirst []*resp.BuildSummary
+
+func (a newBuildsFirst) Len() int      { return len(a) }
+func (a newBuildsFirst) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
+func (a newBuildsFirst) Less(i, j int) bool {
+	return a[i].PendingTime.Started.After(a[j].PendingTime.Started)
+}