milo: Add BuildInfo implementation for BuildBot.

milo/appengine/buildbot/buildinfo.go

+// Copyright 2017 The LUCI Authors. All rights reserved.
+// Use of this source code is governed under the Apache License, Version 2.0
+// that can be found in the LICENSE file.
+
+package buildbot
+
+import (
+        "fmt"
+        "strconv"
+        "strings"
+        "unicode/utf8"
+
+        "github.com/luci/luci-go/common/data/stringset"
+        "github.com/luci/luci-go/common/logging"
+        miloProto "github.com/luci/luci-go/common/proto/milo"
+        "github.com/luci/luci-go/grpc/grpcutil"
+        "github.com/luci/luci-go/logdog/client/coordinator"
+        "github.com/luci/luci-go/logdog/common/types"
+        "github.com/luci/luci-go/luci_config/common/cfgtypes"
+        milo "github.com/luci/luci-go/milo/api/proto"
+        "github.com/luci/luci-go/milo/appengine/logdog"
+
+        "google.golang.org/grpc/codes"
+
+        "golang.org/x/net/context"
+)
+
+// BuildInfoProvider is a configuration that provides build information.
+//
+// In a production system, this will be completely defaults. For testing, the
+// various services and data sources may be substituted for testing stubs.
+type BuildInfoProvider struct {
+        // LogdogClientFunc returns a coordinator Client instance for the supplied
+        // parameters.
+        //
+        // If nil, a production client will be generated.
+        LogdogClientFunc func(c context.Context) (*coordinator.Client, error)
+}
+
+func (p *BuildInfoProvider) newLogdogClient(c context.Context) (*coordinator.Client, error) {
+        if p.LogdogClientFunc != nil {
+                return p.LogdogClientFunc(c)
+        }
+        return logdog.NewClient(c, "")
+}
+
+// GetBuildInfo resolves a Milo protobuf Step for a given BuildBot build.

[hinoka, 2017/02/03 02:16:44]

Add to comment:
1. Fetches the buildbot build json from datastore
2. Fetches the logdog stream from the logdog server.
3. Extract the milo step proto from the logdog stream
4. Merge some information from the buildbot build json (what info? Just
properties?) into the final milo step proto.

[dnj, 2017/02/03 23:54:04]

Done.

+//
+// On failure, it returns a (potentially-wrapped) gRPC error.
+func (p *BuildInfoProvider) GetBuildInfo(c context.Context, req *milo.BuildInfoRequest_BuildBot,
+        projectHint cfgtypes.ProjectName) (*milo.BuildInfoResponse, error) {
+
+        logging.Infof(c, "Loading build info for master %q, builder %q, build #%d",
+                req.MasterName, req.BuilderName, req.BuildNumber)
+
+        // Load the BuildBot build from datastore.
+        build, err := getBuild(c, req.MasterName, req.BuilderName, int(req.BuildNumber))
+        if err != nil {
+                if err == errBuildNotFound {
+                        return nil, grpcutil.Errf(codes.NotFound, "Build #%d for master %q, builder %q was not found",
+                                req.BuildNumber, req.MasterName, req.BuilderName)
+                }
+
+                logging.WithError(err).Errorf(c, "Failed to load build info.")
+                return nil, grpcutil.Internal
+        }
+
+        // Create a new LogDog client.
+        client, err := p.newLogdogClient(c)
+        if err != nil {
+                logging.WithError(err).Errorf(c, "Failed to create LogDog client.")
+                return nil, grpcutil.Internal
+        }
+
+        // Identify the LogDog annotation stream from the build.
+        //
+        // This will return a gRPC error on failure.
+        as, err := getLogDogAnnotations(c, client, build, projectHint)
+        if err != nil {
+                return nil, err

[hinoka, 2017/02/03 02:16:44]

We should resolve 404s and give a better error message if Milo knows about a
build but Logdog doesn't.

[dnj, 2017/02/03 23:53:38]

This here will error if, given a build, we can't figure out the LogDog
annotation proto. I'll accompany the failures with some more info, though.

+        }
+        logging.Infof(c, "Resolved annotation stream: %s / %s", as.Project, as.Path)
+
+        // Load the annotation protobuf.
+        as.Client = client
+        if err := as.Normalize(); err != nil {
+                logging.WithError(err).Errorf(c, "Failed to normalize annotation stream.")
+                return nil, grpcutil.Internal
+        }
+        if err := as.Load(c); err != nil {
+                logging.WithError(err).Errorf(c, "Failed to load annotation stream.")
+                return nil, grpcutil.Internal
+        }
+
+        // Merge the information together.
+        step := as.Step()
+        if err := mergeBuildIntoAnnotation(c, step, build); err != nil {
+                logging.WithError(err).Errorf(c, "Failed to merge annotation with build.")
+                return nil, grpcutil.Internal
+        }
+
+        prefix, name := as.Path.Split()
+        return &milo.BuildInfoResponse{
+                Project: string(as.Project),
+                Step:    step,
+                AnnotationStream: &miloProto.LogdogStream{
+                        Server: client.Host,
+                        Prefix: string(prefix),
+                        Name:   string(name),
+                },
+        }, nil
+}
+
+// Resolve BuildBot and LogDog build information. We do this
+//
+// This returns an AnnotationStream instance with its project and path
+// populated.
+//
+// This function is messy and implementation-specific. That's the point of this
+// endpoint, though. All of the nastiness here should be replaced with something
+// more elegant once that becomes available. In the meantime...
+func getLogDogAnnotations(c context.Context, client *coordinator.Client, build *buildbotBuild, projectHint cfgtypes.ProjectName) (

[hinoka, 2017/02/03 02:16:44]

Try to fit this in under 100char

[dnj, 2017/02/03 23:53:38]

Done.

+        *logdog.AnnotationStream, error) {
+
+        // Modern builds will have this information in their build properties.
+        var as logdog.AnnotationStream
+        prefix, _ := build.getPropertyValue("logdog_prefix").(string)
+        project, _ := build.getPropertyValue("logdog_project").(string)
+        if prefix != "" && project != "" {
+                // Construct the full annotation path.
+                as.Project = cfgtypes.ProjectName(project)
+                as.Path = types.StreamName(prefix).Join("annotations")
+
+                logging.Debugf(c, "Resolved path/project from build properties.")
+                return &as, nil
+        }
+
+        // From here on out, we will need a project hint.
+        if projectHint == "" {

[hinoka, 2017/02/03 02:16:44]

I thought this was optional, will buildbot fetches fail if this isn't provided?

[dnj, 2017/02/03 23:53:38]

It is optional, but some resolution paths require a hint to succeed.

+                return nil, grpcutil.Errf(codes.NotFound, "annotation stream not found")
+        }
+        as.Project = projectHint
+
+        // Execute a LogDog service query to see if we can identify the stream.
+        err := func() error {
+                var annotationStream *coordinator.LogStream
+                err := client.Query(c, as.Project, "", coordinator.QueryOptions{
+                        Tags: map[string]string{
+                                "buildbot.master":      build.Master,
+                                "buildbot.builder":     build.Buildername,
+                                "buildbot.buildnumber": strconv.Itoa(build.Number),
+                        },
+                        ContentType: miloProto.ContentTypeAnnotations,
+                }, func(ls *coordinator.LogStream) bool {
+                        // Only need the first (hopefully only?) result.
+                        annotationStream = ls
+                        return false
+                })
+                if err != nil {
+                        logging.WithError(err).Errorf(c, "Failed to issue log stream query.")
+                        return grpcutil.Internal
+                }
+
+                if annotationStream != nil {
+                        as.Path = annotationStream.Path
+                }
+                return nil
+        }()
+        if err != nil {
+                return nil, err
+        }
+        if as.Path != "" {
+                logging.Debugf(c, "Resolved path/project via tag query.")

[hinoka, 2017/02/03 02:16:44]

Resolved path/project into %s .....

[dnj, 2017/02/03 23:53:38]

The calling site for getLogDogAnnotations follows up a successful return value
with this information.

+                return &as, nil
+        }
+
+        // Last-ditch effort: generate a prefix based on the build properties. This
+        // re-implements the "_build_prefix" function in:
+        // https://chromium.googlesource.com/chromium/tools/build/+/2d23e5284cc31f31c6bc07aa1d3fc5b1c454c3b4/scripts/slave/logdog_bootstrap.py#363
+        isAlnum := func(r rune) bool {
+                return ((r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9'))
+        }
+        normalize := func(v string) string {
+                v = strings.Map(func(r rune) rune {
+                        if isAlnum(r) {
+                                return r
+                        }
+                        switch r {
+                        case ':', '_', '-', '.':
+                                return r
+                        default:
+                                return '_'
+                        }
+                }, v)
+                if r, _ := utf8.DecodeRuneInString(v); r == utf8.RuneError || !isAlnum(r) {
+                        v = "s_" + v
+                }
+                return v
+        }
+        as.Path = types.StreamPath(fmt.Sprintf("bb/%s/%s/%s/+/annotations",
+                normalize(build.Master), normalize(build.Buildername), normalize(strconv.Itoa(build.Number))))
+
+        logging.Debugf(c, "Generated path/project algorithmically.")
+        return &as, nil
+}
+
+func mergeBuildIntoAnnotation(c context.Context, step *miloProto.Step, build *buildbotBuild) error {

[hinoka, 2017/02/03 02:16:44]

Add comment.  This looks like it just extract build properties from the buildbot
build struct and adds it to the step.  Given the function name I expected it to
do more.

[dnj, 2017/02/03 23:53:38]

I was thinking that this might be expanded in the future if there is additional
data that is identified that we want to merge; the function name reflects this.

I updated the comment to be more detailed.

+        allProps := stringset.New(len(step.Property) + len(build.Properties))
+        for _, prop := range step.Property {
+                allProps.Add(prop.Name)
+        }
+        for _, prop := range build.Properties {
+                // Annotation protobuf overrides BuildBot properties.
+                if allProps.Has(prop.Name) {
+                        continue
+                }
+                allProps.Add(prop.Name)
+
+                step.Property = append(step.Property, &miloProto.Step_Property{
+                        Name:  prop.Name,
+                        Value: fmt.Sprintf("%v", prop.Value),
+                })
+        }
+
+        return nil
+}