| OLD | NEW |
|---|
| (Empty) |
| 1 // Copyright 2017 The LUCI Authors. All rights reserved. | |
| 2 // Use of this source code is governed under the Apache License, Version 2.0 | |
| 3 // that can be found in the LICENSE file. | |
| 4 | |
| 5 package buildbot | |
| 6 | |
| 7 import ( | |
| 8 "fmt" | |
| 9 "strconv" | |
| 10 "strings" | |
| 11 "unicode/utf8" | |
| 12 | |
| 13 "github.com/luci/luci-go/common/data/stringset" | |
| 14 "github.com/luci/luci-go/common/logging" | |
| 15 miloProto "github.com/luci/luci-go/common/proto/milo" | |
| 16 "github.com/luci/luci-go/grpc/grpcutil" | |
| 17 "github.com/luci/luci-go/logdog/client/coordinator" | |
| 18 "github.com/luci/luci-go/logdog/common/types" | |
| 19 "github.com/luci/luci-go/luci_config/common/cfgtypes" | |
| 20 milo "github.com/luci/luci-go/milo/api/proto" | |
| 21 "github.com/luci/luci-go/milo/appengine/logdog" | |
| 22 | |
| 23 "google.golang.org/grpc/codes" | |
| 24 | |
| 25 "golang.org/x/net/context" | |
| 26 ) | |
| 27 | |
| 28 // BuildInfoProvider is a configuration that provides build information. | |
| 29 // | |
| 30 // In a production system, this will be completely defaults. For testing, the | |
| 31 // various services and data sources may be substituted for testing stubs. | |
| 32 type BuildInfoProvider struct { | |
| 33 // LogdogClientFunc returns a coordinator Client instance for the suppli
ed | |
| 34 // parameters. | |
| 35 // | |
| 36 // If nil, a production client will be generated. | |
| 37 LogdogClientFunc func(c context.Context) (*coordinator.Client, error) | |
| 38 } | |
| 39 | |
| 40 func (p *BuildInfoProvider) newLogdogClient(c context.Context) (*coordinator.Cli
ent, error) { | |
| 41 if p.LogdogClientFunc != nil { | |
| 42 return p.LogdogClientFunc(c) | |
| 43 } | |
| 44 return logdog.NewClient(c, "") | |
| 45 } | |
| 46 | |
| 47 // GetBuildInfo resolves a Milo protobuf Step for a given BuildBot build. | |
| 48 // | |
| 49 // On failure, it returns a (potentially-wrapped) gRPC error. | |
| 50 // | |
| 51 // This: | |
| 52 // | |
| 53 // 1) Fetches the BuildBot build JSON from datastore. | |
| 54 // 2) Resolves the LogDog annotation stream path from the BuildBot state. | |
| 55 // 3) Fetches the LogDog annotation stream and resolves it into a Step. | |
| 56 // 4) Merges some operational BuildBot build information into the Step. | |
| 57 func (p *BuildInfoProvider) GetBuildInfo(c context.Context, req *milo.BuildInfoR
equest_BuildBot, | |
| 58 projectHint cfgtypes.ProjectName) (*milo.BuildInfoResponse, error) { | |
| 59 | |
| 60 logging.Infof(c, "Loading build info for master %q, builder %q, build #%
d", | |
| 61 req.MasterName, req.BuilderName, req.BuildNumber) | |
| 62 | |
| 63 // Load the BuildBot build from datastore. | |
| 64 build, err := getBuild(c, req.MasterName, req.BuilderName, int(req.Build
Number)) | |
| 65 switch err { | |
| 66 case errBuildNotFound: | |
| 67 return nil, grpcutil.Errf(codes.NotFound, "Build #%d for master
%q, builder %q was not found", | |
| 68 req.BuildNumber, req.MasterName, req.BuilderName) | |
| 69 case errNotAuth: | |
| 70 return nil, grpcutil.Unauthenticated | |
| 71 case nil: | |
| 72 // continue | |
| 73 default: | |
| 74 logging.WithError(err).Errorf(c, "Failed to load build info.") | |
| 75 return nil, grpcutil.Internal | |
| 76 } | |
| 77 | |
| 78 // Create a new LogDog client. | |
| 79 client, err := p.newLogdogClient(c) | |
| 80 if err != nil { | |
| 81 logging.WithError(err).Errorf(c, "Failed to create LogDog client
.") | |
| 82 return nil, grpcutil.Internal | |
| 83 } | |
| 84 | |
| 85 // Identify the LogDog annotation stream from the build. | |
| 86 // | |
| 87 // This will return a gRPC error on failure. | |
| 88 addr, err := getLogDogAnnotationAddr(c, client, build, projectHint) | |
| 89 if err != nil { | |
| 90 return nil, err | |
| 91 } | |
| 92 logging.Infof(c, "Resolved annotation stream: %s / %s", addr.Project, ad
dr.Path) | |
| 93 | |
| 94 // Load the annotation protobuf. | |
| 95 as := logdog.AnnotationStream{ | |
| 96 Client: client, | |
| 97 Path: addr.Path, | |
| 98 Project: addr.Project, | |
| 99 } | |
| 100 if err := as.Normalize(); err != nil { | |
| 101 logging.WithError(err).Errorf(c, "Failed to normalize annotation
stream.") | |
| 102 return nil, grpcutil.Internal | |
| 103 } | |
| 104 | |
| 105 step, err := as.Fetch(c) | |
| 106 if err != nil { | |
| 107 logging.WithError(err).Errorf(c, "Failed to load annotation stre
am.") | |
| 108 return nil, grpcutil.Errf(codes.Internal, "failed to load LogDog
annotation stream from: %s", as.Path) | |
| 109 } | |
| 110 | |
| 111 // Merge the information together. | |
| 112 if err := mergeBuildIntoAnnotation(c, step, build); err != nil { | |
| 113 logging.WithError(err).Errorf(c, "Failed to merge annotation wit
h build.") | |
| 114 return nil, grpcutil.Errf(codes.Internal, "failed to merge annot
ation and build data") | |
| 115 } | |
| 116 | |
| 117 prefix, name := as.Path.Split() | |
| 118 return &milo.BuildInfoResponse{ | |
| 119 Project: string(as.Project), | |
| 120 Step: step, | |
| 121 AnnotationStream: &miloProto.LogdogStream{ | |
| 122 Server: client.Host, | |
| 123 Prefix: string(prefix), | |
| 124 Name: string(name), | |
| 125 }, | |
| 126 }, nil | |
| 127 } | |
| 128 | |
| 129 // Resolve BuildBot and LogDog build information. We do this | |
| 130 // | |
| 131 // This returns an AnnotationStream instance with its project and path | |
| 132 // populated. | |
| 133 // | |
| 134 // This function is messy and implementation-specific. That's the point of this | |
| 135 // endpoint, though. All of the nastiness here should be replaced with something | |
| 136 // more elegant once that becomes available. In the meantime... | |
| 137 func getLogDogAnnotationAddr(c context.Context, client *coordinator.Client, buil
d *buildbotBuild, | |
| 138 projectHint cfgtypes.ProjectName) (*types.StreamAddr, error) { | |
| 139 | |
| 140 if v, ok := build.getPropertyValue("log_location").(string); ok && v !=
"" { | |
| 141 addr, err := types.ParseURL(v) | |
| 142 if err == nil { | |
| 143 return addr, nil | |
| 144 } | |
| 145 | |
| 146 logging.Fields{ | |
| 147 logging.ErrorKey: err, | |
| 148 "log_location": v, | |
| 149 }.Debugf(c, "'log_location' property did not parse as LogDog URL
.") | |
| 150 } | |
| 151 | |
| 152 // logdog_annotation_url (if present, must be valid) | |
| 153 if v, ok := build.getPropertyValue("logdog_annotation_url").(string); ok
&& v != "" { | |
| 154 addr, err := types.ParseURL(v) | |
| 155 if err != nil { | |
| 156 logging.Fields{ | |
| 157 logging.ErrorKey: err, | |
| 158 "url": v, | |
| 159 }.Errorf(c, "Failed to parse 'logdog_annotation_url' pro
perty.") | |
| 160 return nil, grpcutil.Errf(codes.FailedPrecondition, "bui
ld has invalid annotation URL") | |
| 161 } | |
| 162 | |
| 163 return addr, nil | |
| 164 } | |
| 165 | |
| 166 // Modern builds will have this information in their build properties. | |
| 167 var addr types.StreamAddr | |
| 168 prefix, _ := build.getPropertyValue("logdog_prefix").(string) | |
| 169 project, _ := build.getPropertyValue("logdog_project").(string) | |
| 170 if prefix != "" && project != "" { | |
| 171 // Construct the full annotation path. | |
| 172 addr.Project = cfgtypes.ProjectName(project) | |
| 173 addr.Path = types.StreamName(prefix).Join("annotations") | |
| 174 | |
| 175 logging.Debugf(c, "Resolved path/project from build properties."
) | |
| 176 return &addr, nil | |
| 177 } | |
| 178 | |
| 179 // From here on out, we will need a project hint. | |
| 180 if projectHint == "" { | |
| 181 return nil, grpcutil.Errf(codes.NotFound, "annotation stream not
found") | |
| 182 } | |
| 183 addr.Project = projectHint | |
| 184 | |
| 185 // Execute a LogDog service query to see if we can identify the stream. | |
| 186 err := func() error { | |
| 187 var annotationStream *coordinator.LogStream | |
| 188 err := client.Query(c, addr.Project, "", coordinator.QueryOption
s{ | |
| 189 Tags: map[string]string{ | |
| 190 "buildbot.master": build.Master, | |
| 191 "buildbot.builder": build.Buildername, | |
| 192 "buildbot.buildnumber": strconv.Itoa(build.Numbe
r), | |
| 193 }, | |
| 194 ContentType: miloProto.ContentTypeAnnotations, | |
| 195 }, func(ls *coordinator.LogStream) bool { | |
| 196 // Only need the first (hopefully only?) result. | |
| 197 annotationStream = ls | |
| 198 return false | |
| 199 }) | |
| 200 if err != nil { | |
| 201 logging.WithError(err).Errorf(c, "Failed to issue log st
ream query.") | |
| 202 return grpcutil.Internal | |
| 203 } | |
| 204 | |
| 205 if annotationStream != nil { | |
| 206 addr.Path = annotationStream.Path | |
| 207 } | |
| 208 return nil | |
| 209 }() | |
| 210 if err != nil { | |
| 211 return nil, err | |
| 212 } | |
| 213 if addr.Path != "" { | |
| 214 logging.Debugf(c, "Resolved path/project via tag query.") | |
| 215 return &addr, nil | |
| 216 } | |
| 217 | |
| 218 // Last-ditch effort: generate a prefix based on the build properties. T
his | |
| 219 // re-implements the "_build_prefix" function in: | |
| 220 // https://chromium.googlesource.com/chromium/tools/build/+/2d23e5284cc3
1f31c6bc07aa1d3fc5b1c454c3b4/scripts/slave/logdog_bootstrap.py#363 | |
| 221 isAlnum := func(r rune) bool { | |
| 222 return ((r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >
= '0' && r <= '9')) | |
| 223 } | |
| 224 normalize := func(v string) string { | |
| 225 v = strings.Map(func(r rune) rune { | |
| 226 if isAlnum(r) { | |
| 227 return r | |
| 228 } | |
| 229 switch r { | |
| 230 case ':', '_', '-', '.': | |
| 231 return r | |
| 232 default: | |
| 233 return '_' | |
| 234 } | |
| 235 }, v) | |
| 236 if r, _ := utf8.DecodeRuneInString(v); r == utf8.RuneError || !i
sAlnum(r) { | |
| 237 v = "s_" + v | |
| 238 } | |
| 239 return v | |
| 240 } | |
| 241 addr.Path = types.StreamPath(fmt.Sprintf("bb/%s/%s/%s/+/annotations", | |
| 242 normalize(build.Master), normalize(build.Buildername), normalize
(strconv.Itoa(build.Number)))) | |
| 243 | |
| 244 logging.Debugf(c, "Generated path/project algorithmically.") | |
| 245 return &addr, nil | |
| 246 } | |
| 247 | |
| 248 // mergeBuildInfoIntoAnnotation merges BuildBot-specific build informtion into | |
| 249 // a LogDog annotation protobuf. | |
| 250 // | |
| 251 // This consists of augmenting the Step's properties with BuildBot's properties, | |
| 252 // favoring the Step's version of the properties if there are two with the same | |
| 253 // name. | |
| 254 func mergeBuildIntoAnnotation(c context.Context, step *miloProto.Step, build *bu
ildbotBuild) error { | |
| 255 allProps := stringset.New(len(step.Property) + len(build.Properties)) | |
| 256 for _, prop := range step.Property { | |
| 257 allProps.Add(prop.Name) | |
| 258 } | |
| 259 for _, prop := range build.Properties { | |
| 260 // Annotation protobuf overrides BuildBot properties. | |
| 261 if allProps.Has(prop.Name) { | |
| 262 continue | |
| 263 } | |
| 264 allProps.Add(prop.Name) | |
| 265 | |
| 266 step.Property = append(step.Property, &miloProto.Step_Property{ | |
| 267 Name: prop.Name, | |
| 268 Value: fmt.Sprintf("%v", prop.Value), | |
| 269 }) | |
| 270 } | |
| 271 | |
| 272 return nil | |
| 273 } | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 2944983003:
[milo] {buildbucket,buildbot,swarming,logdog} -> backends/*. (Closed)
