milo: Add BuildInfo implementation for BuildBot.

milo/appengine/buildbot/buildinfo.go

Index: milo/appengine/buildbot/buildinfo.go
diff --git a/milo/appengine/buildbot/buildinfo.go b/milo/appengine/buildbot/buildinfo.go
new file mode 100644
index 0000000000000000000000000000000000000000..84d6b063cc21428dcc885d68e54e4ced78dc2821
--- /dev/null
+++ b/milo/appengine/buildbot/buildinfo.go
@@ -0,0 +1,240 @@
+// Copyright 2017 The LUCI Authors. All rights reserved.
+// Use of this source code is governed under the Apache License, Version 2.0
+// that can be found in the LICENSE file.
+
+package buildbot
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+	"unicode/utf8"
+
+	"github.com/luci/luci-go/common/data/stringset"
+	"github.com/luci/luci-go/common/logging"
+	miloProto "github.com/luci/luci-go/common/proto/milo"
+	"github.com/luci/luci-go/grpc/grpcutil"
+	"github.com/luci/luci-go/logdog/client/coordinator"
+	"github.com/luci/luci-go/logdog/common/types"
+	"github.com/luci/luci-go/luci_config/common/cfgtypes"
+	milo "github.com/luci/luci-go/milo/api/proto"
+	"github.com/luci/luci-go/milo/appengine/logdog"
+
+	"google.golang.org/grpc/codes"
+
+	"golang.org/x/net/context"
+)
+
+// BuildInfoProvider is a configuration that provides build information.
+//
+// In a production system, this will be completely defaults. For testing, the
+// various services and data sources may be substituted for testing stubs.
+type BuildInfoProvider struct {
+	// LogdogClientFunc returns a coordinator Client instance for the supplied
+	// parameters.
+	//
+	// If nil, a production client will be generated.
+	LogdogClientFunc func(c context.Context) (*coordinator.Client, error)
+}
+
+func (p *BuildInfoProvider) newLogdogClient(c context.Context) (*coordinator.Client, error) {
+	if p.LogdogClientFunc != nil {
+		return p.LogdogClientFunc(c)
+	}
+	return logdog.NewClient(c, "")
+}
+
+// GetBuildInfo resolves a Milo protobuf Step for a given BuildBot build.
+//
+// On failure, it returns a (potentially-wrapped) gRPC error.
+//
+// This:
+//
+//	1) Fetches the BuildBot build JSON from datastore.
+//	2) Resolves the LogDog annotation stream path from the BuildBot state.
+//	3) Fetches the LogDog annotation stream and resolves it into a Step.
+//	4) Merges some operational BuildBot build information into the Step.
+func (p *BuildInfoProvider) GetBuildInfo(c context.Context, req *milo.BuildInfoRequest_BuildBot,
+	projectHint cfgtypes.ProjectName) (*milo.BuildInfoResponse, error) {
+
+	logging.Infof(c, "Loading build info for master %q, builder %q, build #%d",
+		req.MasterName, req.BuilderName, req.BuildNumber)
+
+	// Load the BuildBot build from datastore.
+	build, err := getBuild(c, req.MasterName, req.BuilderName, int(req.BuildNumber))
+	if err != nil {
+		if err == errBuildNotFound {
+			return nil, grpcutil.Errf(codes.NotFound, "Build #%d for master %q, builder %q was not found",
+				req.BuildNumber, req.MasterName, req.BuilderName)
+		}
+
+		logging.WithError(err).Errorf(c, "Failed to load build info.")
+		return nil, grpcutil.Internal
+	}
+
+	// Create a new LogDog client.
+	client, err := p.newLogdogClient(c)
+	if err != nil {
+		logging.WithError(err).Errorf(c, "Failed to create LogDog client.")
+		return nil, grpcutil.Internal
+	}
+
+	// Identify the LogDog annotation stream from the build.
+	//
+	// This will return a gRPC error on failure.
+	as, err := getLogDogAnnotations(c, client, build, projectHint)
+	if err != nil {
+		return nil, err
+	}
+	logging.Infof(c, "Resolved annotation stream: %s / %s", as.Project, as.Path)
+
+	// Load the annotation protobuf.
+	as.Client = client
+	if err := as.Normalize(); err != nil {
+		logging.WithError(err).Errorf(c, "Failed to normalize annotation stream.")
+		return nil, grpcutil.Internal
+	}
+
+	step, err := as.Load(c)
+	if err != nil {
+		logging.WithError(err).Errorf(c, "Failed to load annotation stream.")
+		return nil, grpcutil.Errf(codes.Internal, "failed to load LogDog annotation stream from: %s", as.Path)
+	}
+
+	// Merge the information together.
+	if err := mergeBuildIntoAnnotation(c, step, build); err != nil {
+		logging.WithError(err).Errorf(c, "Failed to merge annotation with build.")
+		return nil, grpcutil.Errf(codes.Internal, "failed to merge annotation and build data")
+	}
+
+	prefix, name := as.Path.Split()
+	return &milo.BuildInfoResponse{
+		Project: string(as.Project),
+		Step:    step,
+		AnnotationStream: &miloProto.LogdogStream{
+			Server: client.Host,
+			Prefix: string(prefix),
+			Name:   string(name),
+		},
+	}, nil
+}
+
+// Resolve BuildBot and LogDog build information. We do this
+//
+// This returns an AnnotationStream instance with its project and path
+// populated.
+//
+// This function is messy and implementation-specific. That's the point of this
+// endpoint, though. All of the nastiness here should be replaced with something
+// more elegant once that becomes available. In the meantime...
+func getLogDogAnnotations(c context.Context, client *coordinator.Client, build *buildbotBuild,
+	projectHint cfgtypes.ProjectName) (*logdog.AnnotationStream, error) {
+
+	// Modern builds will have this information in their build properties.
+	var as logdog.AnnotationStream
+	prefix, _ := build.getPropertyValue("logdog_prefix").(string)
+	project, _ := build.getPropertyValue("logdog_project").(string)
+	if prefix != "" && project != "" {
+		// Construct the full annotation path.
+		as.Project = cfgtypes.ProjectName(project)
+		as.Path = types.StreamName(prefix).Join("annotations")
+
+		logging.Debugf(c, "Resolved path/project from build properties.")
+		return &as, nil
+	}
+
+	// From here on out, we will need a project hint.
+	if projectHint == "" {
+		return nil, grpcutil.Errf(codes.NotFound, "annotation stream not found")
+	}
+	as.Project = projectHint
+
+	// Execute a LogDog service query to see if we can identify the stream.
+	err := func() error {
+		var annotationStream *coordinator.LogStream
+		err := client.Query(c, as.Project, "", coordinator.QueryOptions{
+			Tags: map[string]string{
+				"buildbot.master":      build.Master,
+				"buildbot.builder":     build.Buildername,
+				"buildbot.buildnumber": strconv.Itoa(build.Number),
+			},
+			ContentType: miloProto.ContentTypeAnnotations,
+		}, func(ls *coordinator.LogStream) bool {
+			// Only need the first (hopefully only?) result.
+			annotationStream = ls
+			return false
+		})
+		if err != nil {
+			logging.WithError(err).Errorf(c, "Failed to issue log stream query.")
+			return grpcutil.Internal
+		}
+
+		if annotationStream != nil {
+			as.Path = annotationStream.Path
+		}
+		return nil
+	}()
+	if err != nil {
+		return nil, err
+	}
+	if as.Path != "" {
+		logging.Debugf(c, "Resolved path/project via tag query.")
+		return &as, nil
+	}
+
+	// Last-ditch effort: generate a prefix based on the build properties. This
+	// re-implements the "_build_prefix" function in:
+	// https://chromium.googlesource.com/chromium/tools/build/+/2d23e5284cc31f31c6bc07aa1d3fc5b1c454c3b4/scripts/slave/logdog_bootstrap.py#363
+	isAlnum := func(r rune) bool {
+		return ((r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9'))
+	}
+	normalize := func(v string) string {
+		v = strings.Map(func(r rune) rune {
+			if isAlnum(r) {
+				return r
+			}
+			switch r {
+			case ':', '_', '-', '.':
+				return r
+			default:
+				return '_'
+			}
+		}, v)
+		if r, _ := utf8.DecodeRuneInString(v); r == utf8.RuneError || !isAlnum(r) {
+			v = "s_" + v
+		}
+		return v
+	}
+	as.Path = types.StreamPath(fmt.Sprintf("bb/%s/%s/%s/+/annotations",
+		normalize(build.Master), normalize(build.Buildername), normalize(strconv.Itoa(build.Number))))
+
+	logging.Debugf(c, "Generated path/project algorithmically.")
+	return &as, nil
+}
+
+// mergeBuildInfoIntoAnnotation merges BuildBot-specific build informtion into
+// a LogDog annotation protobuf.
+//
+// This consists of augmenting the Step's properties with BuildBot's properties,
+// favoring the Step's version of the properties if there are two with the same
+// name.
+func mergeBuildIntoAnnotation(c context.Context, step *miloProto.Step, build *buildbotBuild) error {
+	allProps := stringset.New(len(step.Property) + len(build.Properties))
+	for _, prop := range step.Property {
+		allProps.Add(prop.Name)
+	}
+	for _, prop := range build.Properties {
+		// Annotation protobuf overrides BuildBot properties.
+		if allProps.Has(prop.Name) {
+			continue
+		}
+		allProps.Add(prop.Name)
+
+		step.Property = append(step.Property, &miloProto.Step_Property{
+			Name:  prop.Name,
+			Value: fmt.Sprintf("%v", prop.Value),
+		})
+	}
+
+	return nil
+}