milo: Enable Swarming LogDog log loading.

milo/appengine/buildbot/buildinfo.go

 // Copyright 2017 The LUCI Authors. All rights reserved.
 // Use of this source code is governed under the Apache License, Version 2.0
 // that can be found in the LICENSE file.
 
 package buildbot
 
 import (
         "fmt"
         "strconv"
         "strings"
@@ skipping 64 matching lines @@
         // Create a new LogDog client.
         client, err := p.newLogdogClient(c)
         if err != nil {
                 logging.WithError(err).Errorf(c, "Failed to create LogDog client.")
                 return nil, grpcutil.Internal
         }
 
         // Identify the LogDog annotation stream from the build.
         //
         // This will return a gRPC error on failure.
-        as, err := getLogDogAnnotations(c, client, build, projectHint)
+        addr, err := getLogDogAnnotationAddr(c, client, build, projectHint)
         if err != nil {
                 return nil, err
         }
-        logging.Infof(c, "Resolved annotation stream: %s / %s", as.Project, as.Path)
+        logging.Infof(c, "Resolved annotation stream: %s / %s", addr.Project, addr.Path)
 
         // Load the annotation protobuf.
-        as.Client = client
+        as := logdog.AnnotationStream{
+                Client:  client,
+                Path:    addr.Path,
+                Project: addr.Project,
+        }
         if err := as.Normalize(); err != nil {
                 logging.WithError(err).Errorf(c, "Failed to normalize annotation stream.")
                 return nil, grpcutil.Internal
         }
 
-        step, err := as.Load(c)
+        step, err := as.Fetch(c)
         if err != nil {
                 logging.WithError(err).Errorf(c, "Failed to load annotation stream.")
                 return nil, grpcutil.Errf(codes.Internal, "failed to load LogDog annotation stream from: %s", as.Path)
         }
 
         // Merge the information together.
         if err := mergeBuildIntoAnnotation(c, step, build); err != nil {
                 logging.WithError(err).Errorf(c, "Failed to merge annotation with build.")
                 return nil, grpcutil.Errf(codes.Internal, "failed to merge annotation and build data")
         }
@@ skipping 11 matching lines @@
 }
 
 // Resolve BuildBot and LogDog build information. We do this
 //
 // This returns an AnnotationStream instance with its project and path
 // populated.
 //
 // This function is messy and implementation-specific. That's the point of this
 // endpoint, though. All of the nastiness here should be replaced with something
 // more elegant once that becomes available. In the meantime...
-func getLogDogAnnotations(c context.Context, client *coordinator.Client, build *buildbotBuild,
-        projectHint cfgtypes.ProjectName) (*logdog.AnnotationStream, error) {
+func getLogDogAnnotationAddr(c context.Context, client *coordinator.Client, build *buildbotBuild,
+        projectHint cfgtypes.ProjectName) (*types.StreamAddr, error) {
 
         // Modern builds will have this information in their build properties.
-        var as logdog.AnnotationStream
+        var addr types.StreamAddr
         prefix, _ := build.getPropertyValue("logdog_prefix").(string)
         project, _ := build.getPropertyValue("logdog_project").(string)
         if prefix != "" && project != "" {
                 // Construct the full annotation path.
-                as.Project = cfgtypes.ProjectName(project)
-                as.Path = types.StreamName(prefix).Join("annotations")
+                addr.Project = cfgtypes.ProjectName(project)
+                addr.Path = types.StreamName(prefix).Join("annotations")
 
                 logging.Debugf(c, "Resolved path/project from build properties.")
-                return &as, nil
+                return &addr, nil
         }
 
         // From here on out, we will need a project hint.
         if projectHint == "" {
                 return nil, grpcutil.Errf(codes.NotFound, "annotation stream not found")
         }
-        as.Project = projectHint
+        addr.Project = projectHint
 
         // Execute a LogDog service query to see if we can identify the stream.
         err := func() error {
                 var annotationStream *coordinator.LogStream
-                err := client.Query(c, as.Project, "", coordinator.QueryOptions{
+                err := client.Query(c, addr.Project, "", coordinator.QueryOptions{
                         Tags: map[string]string{
                                 "buildbot.master":      build.Master,
                                 "buildbot.builder":     build.Buildername,
                                 "buildbot.buildnumber": strconv.Itoa(build.Number),
                         },
                         ContentType: miloProto.ContentTypeAnnotations,
                 }, func(ls *coordinator.LogStream) bool {
                         // Only need the first (hopefully only?) result.
                         annotationStream = ls
                         return false
                 })
                 if err != nil {
                         logging.WithError(err).Errorf(c, "Failed to issue log stream query.")
                         return grpcutil.Internal
                 }
 
                 if annotationStream != nil {
-                        as.Path = annotationStream.Path
+                        addr.Path = annotationStream.Path
                 }
                 return nil
         }()
         if err != nil {
                 return nil, err
         }
-        if as.Path != "" {
+        if addr.Path != "" {
                 logging.Debugf(c, "Resolved path/project via tag query.")
-                return &as, nil
+                return &addr, nil
         }
 
         // Last-ditch effort: generate a prefix based on the build properties. This
         // re-implements the "_build_prefix" function in:
         // https://chromium.googlesource.com/chromium/tools/build/+/2d23e5284cc31f31c6bc07aa1d3fc5b1c454c3b4/scripts/slave/logdog_bootstrap.py#363
         isAlnum := func(r rune) bool {
                 return ((r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9'))
         }
         normalize := func(v string) string {
                 v = strings.Map(func(r rune) rune {
                         if isAlnum(r) {
                                 return r
                         }
                         switch r {
                         case ':', '_', '-', '.':
                                 return r
                         default:
                                 return '_'
                         }
                 }, v)
                 if r, _ := utf8.DecodeRuneInString(v); r == utf8.RuneError || !isAlnum(r) {
                         v = "s_" + v
                 }
                 return v
         }
-        as.Path = types.StreamPath(fmt.Sprintf("bb/%s/%s/%s/+/annotations",
+        addr.Path = types.StreamPath(fmt.Sprintf("bb/%s/%s/%s/+/annotations",
                 normalize(build.Master), normalize(build.Buildername), normalize(strconv.Itoa(build.Number))))
 
         logging.Debugf(c, "Generated path/project algorithmically.")
-        return &as, nil
+        return &addr, nil
 }
 
 // mergeBuildInfoIntoAnnotation merges BuildBot-specific build informtion into
 // a LogDog annotation protobuf.
 //
 // This consists of augmenting the Step's properties with BuildBot's properties,
 // favoring the Step's version of the properties if there are two with the same
 // name.
 func mergeBuildIntoAnnotation(c context.Context, step *miloProto.Step, build *buildbotBuild) error {
         allProps := stringset.New(len(step.Property) + len(build.Properties))
         for _, prop := range step.Property {
                 allProps.Add(prop.Name)
         }
         for _, prop := range build.Properties {
                 // Annotation protobuf overrides BuildBot properties.
                 if allProps.Has(prop.Name) {
                         continue
                 }
                 allProps.Add(prop.Name)
 
                 step.Property = append(step.Property, &miloProto.Step_Property{
                         Name:  prop.Name,
                         Value: fmt.Sprintf("%v", prop.Value),
                 })
         }
 
         return nil
 }