Milo: Add LogDog annotation stream support.

appengine/cmd/milo/logdog/logDogBuild.go

 // Copyright 2016 The LUCI Authors. All rights reserved.
 // Use of this source code is governed under the Apache License, Version 2.0
 // that can be found in the LICENSE file.
 
 package logdog
 
 import (
         "fmt"
-        "strings"
         "time"
 
         "golang.org/x/net/context"
 
         "github.com/luci/luci-go/appengine/cmd/milo/resp"
         "github.com/luci/luci-go/common/clock"
-        "github.com/luci/luci-go/common/logging"
         miloProto "github.com/luci/luci-go/common/proto/milo"
 )
 
+// URLBuilder constructs URLs for various link types.
+type URLBuilder interface {
+        // LinkURL returns the URL associated with the supplied Link.
+        //
+        // If no URL could be built for that Link, nil will be returned.
+        BuildLink(l *miloProto.Link) *resp.Link
+}
+
 // miloBuildStep converts a logdog/milo step to a BuildComponent struct.
 // buildCompletedTime must be zero if build did not complete yet.
-func miloBuildStep(c context.Context, linkBase string, anno *miloProto.Step, buildCompletedTime time.Time) *resp.BuildComponent {
-        linkBase = strings.TrimSuffix(linkBase, "/")
+func miloBuildStep(c context.Context, ub URLBuilder, anno *miloProto.Step, now time.Time) *resp.BuildComponent {
         comp := &resp.BuildComponent{Label: anno.Name}
         switch anno.Status {
         case miloProto.Status_RUNNING:
                 comp.Status = resp.Running
 
         case miloProto.Status_SUCCESS:
                 comp.Status = resp.Success
 
         case miloProto.Status_FAILURE:
-                if anno.GetFailureDetails() != nil {
-                        switch anno.GetFailureDetails().Type {
+                if fd := anno.GetFailureDetails(); fd != nil {
+                        switch fd.Type {
                         case miloProto.FailureDetails_EXCEPTION, miloProto.FailureDetails_INFRA:
                                 comp.Status = resp.InfraFailure
 
                         case miloProto.FailureDetails_DM_DEPENDENCY_FAILED:
                                 comp.Status = resp.DependencyFailure
 
                         default:
                                 comp.Status = resp.Failure
                         }
+
+                        if fd.Text != "" {
+                                comp.Text = append(comp.Text, fd.Text)
+                        }
                 } else {
                         comp.Status = resp.Failure
                 }
 
+        case miloProto.Status_PENDING:
+                comp.Status = resp.NotRun
+
                 // Missing the case of waiting on unfinished dependency...
         default:
                 comp.Status = resp.NotRun
         }
 
-        if !buildCompletedTime.IsZero() && !comp.Status.Terminal() {
+        if !anno.Ended.Time().IsZero() && !comp.Status.Terminal() {
                 // we cannot have unfinished steps in finished builds.
                 comp.Status = resp.InfraFailure
         }
 
-        // Sub link is for one link per log that isn't stdout.
-        for _, link := range anno.GetOtherLinks() {
-                lds := link.GetLogdogStream()
-                if lds == nil {
-                        logging.Warningf(c, "Warning: %v of %v has an empty logdog stream.", link, anno)
-                        continue // DNE???
-                }
-                newLink := &resp.Link{
-                        Label: lds.Name,
-                        URL:   linkBase + "/" + lds.Name,
-                }
-                comp.SubLink = append(comp.SubLink, newLink)
-        }
-
         // Main link is a link to the stdout.
+        var stdoutLink *miloProto.Link
         if anno.StdoutStream != nil {
-                comp.MainLink = &resp.Link{
+                stdoutLink = &miloProto.Link{
                         Label: "stdout",
-                        URL:   linkBase + "/" + anno.StdoutStream.Name,
+                        Value: &miloProto.Link_LogdogStream{
+                                LogdogStream: anno.StdoutStream,
+                        },
                 }
         }
 
+        if anno.Link != nil {
+                comp.MainLink = ub.BuildLink(anno.Link)
+
+                // If we also have a STDOUT stream, add it to our OtherLinks.
+                if stdoutLink != nil {
+                        anno.OtherLinks = append([]*miloProto.Link{stdoutLink}, anno.OtherLinks...)
+                }
+        } else if stdoutLink != nil {
+                comp.MainLink = ub.BuildLink(stdoutLink)
+        }
+
+        // Sub link is for one link per log that isn't stdout.
+        for _, link := range anno.GetOtherLinks() {
+                if l := ub.BuildLink(link); l != nil {
+                        comp.SubLink = append(comp.SubLink, l)
+                }
+        }
+
         // This should always be a step.
         comp.Type = resp.Step
 
         // This should always be 0
         comp.LevelsDeep = 0
 
         // Timestamps
         comp.Started = anno.Started.Time()
         comp.Finished = anno.Ended.Time()
 
+        latestTime := comp.Finished
+        if latestTime.IsZero() {
+                latestTime = now

[nodir, 2016/07/29 18:42:06]

it should not be now, but time of build completion. we had a case that step
proto never had Finished time (because step hung) and thus its duration was days
since step started till now

[dnj, 2016/07/29 19:57:35]

Done.

+        }
+        if !comp.Started.IsZero() {
+                if d := latestTime.Sub(comp.Started); d >= 0 {
+                        comp.Duration = d
+                }
+        }
+

[nodir, 2016/07/29 18:42:06]

this block of added code solves the same problem as the code below. please
revisit

[dnj, 2016/07/29 19:57:35]

Acknowledged.

         var till time.Time
         switch {
         case comp.Status == resp.Running:
                 till = clock.Now(c)
         case !comp.Finished.IsZero():
                 till = comp.Finished
-        default:
-                till = buildCompletedTime
         }
         if !comp.Started.IsZero() && !till.IsZero() {
                 comp.Duration = till.Sub(comp.Started)
         }
 
         // This should be the exact same thing.
-        comp.Text = anno.Text
+        comp.Text = append(comp.Text, anno.Text...)
 
         return comp
 }
 
 // AddLogDogToBuild takes a set of logdog streams and populate a milo build.
 // build.Summary.Finished must be set.
-func AddLogDogToBuild(c context.Context, linkBase string, s *Streams, build *resp.MiloBuild) {
+func AddLogDogToBuild(c context.Context, ub URLBuilder, s *Streams, build *resp.MiloBuild) {
         if s.MainStream == nil {
                 panic("missing main stream")
         }
         // Now Fetch the main annotation of the build.
-        mainAnno := s.MainStream.Data
+        var (
+                mainAnno = s.MainStream.Data
+                now      = clock.Now(c)
+        )
 
         // Now fill in each of the step components.
         // TODO(hinoka): This is totes cachable.
+        build.Summary = *(miloBuildStep(c, ub, mainAnno, now))
         for _, substepContainer := range mainAnno.Substep {
                 anno := substepContainer.GetStep()
                 if anno == nil {
                         // TODO: We ignore non-embedded substeps for now.
                         continue
                 }
 
-                bs := miloBuildStep(c, linkBase, anno, build.Summary.Finished)
-                if bs.Status != resp.Success && bs.Status != resp.NotRun {
+                bs := miloBuildStep(c, ub, anno, now)
+                if bs.Status != resp.Success {
                         build.Summary.Text = append(
                                 build.Summary.Text, fmt.Sprintf("%s %s", bs.Status, bs.Label))
                 }
                 build.Components = append(build.Components, bs)
                 propGroup := &resp.PropertyGroup{GroupName: bs.Label}
                 for _, prop := range anno.Property {
                         propGroup.Property = append(propGroup.Property, &resp.Property{
                                 Key:   prop.Name,
                                 Value: prop.Value,
                         })
                 }
                 build.PropertyGroup = append(build.PropertyGroup, propGroup)
         }
 
         // Take care of properties
         propGroup := &resp.PropertyGroup{GroupName: "Main"}
         for _, prop := range mainAnno.Property {
                 propGroup.Property = append(propGroup.Property, &resp.Property{
                         Key:   prop.Name,
                         Value: prop.Value,
                 })
         }
         build.PropertyGroup = append(build.PropertyGroup, propGroup)
 
         return
 }