Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(150)

Issues Search
    My Issues | Starred     Open | Closed | All

Side by Side Diff: milo/build_source/buildbot/buildinfo.go

Issue 2974793002: [milo] build_source -> buildSource. (Closed)
Patch Set: reupload Created 3 years, 5 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
« no previous file with comments | « milo/build_source/buildbot/builder_test.go ('k') | milo/build_source/buildbot/buildinfo_test.go » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
(Empty)
1 // Copyright 2017 The LUCI Authors. All rights reserved.
2 // Use of this source code is governed under the Apache License, Version 2.0
3 // that can be found in the LICENSE file.
4
5 package buildbot
6
7 import (
8 "fmt"
9 "strconv"
10 "strings"
11 "unicode/utf8"
12
13 "github.com/luci/luci-go/common/data/stringset"
14 "github.com/luci/luci-go/common/logging"
15 miloProto "github.com/luci/luci-go/common/proto/milo"
16 "github.com/luci/luci-go/grpc/grpcutil"
17 "github.com/luci/luci-go/logdog/client/coordinator"
18 "github.com/luci/luci-go/logdog/common/types"
19 "github.com/luci/luci-go/luci_config/common/cfgtypes"
20 milo "github.com/luci/luci-go/milo/api/proto"
21 "github.com/luci/luci-go/milo/build_source/raw_presentation"
22
23 "google.golang.org/grpc/codes"
24
25 "golang.org/x/net/context"
26 )
27
28 // BuildInfoProvider is a configuration that provides build information.
29 //
30 // In a production system, this will be completely defaults. For testing, the
31 // various services and data sources may be substituted for testing stubs.
32 type BuildInfoProvider struct {
33 // LogdogClientFunc returns a coordinator Client instance for the suppli ed
34 // parameters.
35 //
36 // If nil, a production client will be generated.
37 LogdogClientFunc func(c context.Context) (*coordinator.Client, error)
38 }
39
40 func (p *BuildInfoProvider) newLogdogClient(c context.Context) (*coordinator.Cli ent, error) {
41 if p.LogdogClientFunc != nil {
42 return p.LogdogClientFunc(c)
43 }
44 return raw_presentation.NewClient(c, "")
45 }
46
47 // GetBuildInfo resolves a Milo protobuf Step for a given BuildBot build.
48 //
49 // On failure, it returns a (potentially-wrapped) gRPC error.
50 //
51 // This:
52 //
53 // 1) Fetches the BuildBot build JSON from datastore.
54 // 2) Resolves the LogDog annotation stream path from the BuildBot state.
55 // 3) Fetches the LogDog annotation stream and resolves it into a Step.
56 // 4) Merges some operational BuildBot build information into the Step.
57 func (p *BuildInfoProvider) GetBuildInfo(c context.Context, req *milo.BuildInfoR equest_BuildBot,
58 projectHint cfgtypes.ProjectName) (*milo.BuildInfoResponse, error) {
59
60 logging.Infof(c, "Loading build info for master %q, builder %q, build #% d",
61 req.MasterName, req.BuilderName, req.BuildNumber)
62
63 // Load the BuildBot build from datastore.
64 build, err := getBuild(c, req.MasterName, req.BuilderName, int(req.Build Number))
65 switch err {
66 case errBuildNotFound:
67 return nil, grpcutil.Errf(codes.NotFound, "Build #%d for master %q, builder %q was not found",
68 req.BuildNumber, req.MasterName, req.BuilderName)
69 case errNotAuth:
70 return nil, grpcutil.Unauthenticated
71 case nil:
72 // continue
73 default:
74 logging.WithError(err).Errorf(c, "Failed to load build info.")
75 return nil, grpcutil.Internal
76 }
77
78 // Create a new LogDog client.
79 client, err := p.newLogdogClient(c)
80 if err != nil {
81 logging.WithError(err).Errorf(c, "Failed to create LogDog client .")
82 return nil, grpcutil.Internal
83 }
84
85 // Identify the LogDog annotation stream from the build.
86 //
87 // This will return a gRPC error on failure.
88 addr, err := getLogDogAnnotationAddr(c, client, build, projectHint)
89 if err != nil {
90 return nil, err
91 }
92 logging.Infof(c, "Resolved annotation stream: %s / %s", addr.Project, ad dr.Path)
93
94 // Load the annotation protobuf.
95 as := raw_presentation.AnnotationStream{
96 Client: client,
97 Path: addr.Path,
98 Project: addr.Project,
99 }
100 if err := as.Normalize(); err != nil {
101 logging.WithError(err).Errorf(c, "Failed to normalize annotation stream.")
102 return nil, grpcutil.Internal
103 }
104
105 step, err := as.Fetch(c)
106 if err != nil {
107 logging.WithError(err).Errorf(c, "Failed to load annotation stre am.")
108 return nil, grpcutil.Errf(codes.Internal, "failed to load LogDog annotation stream from: %s", as.Path)
109 }
110
111 // Merge the information together.
112 if err := mergeBuildIntoAnnotation(c, step, build); err != nil {
113 logging.WithError(err).Errorf(c, "Failed to merge annotation wit h build.")
114 return nil, grpcutil.Errf(codes.Internal, "failed to merge annot ation and build data")
115 }
116
117 prefix, name := as.Path.Split()
118 return &milo.BuildInfoResponse{
119 Project: string(as.Project),
120 Step: step,
121 AnnotationStream: &miloProto.LogdogStream{
122 Server: client.Host,
123 Prefix: string(prefix),
124 Name: string(name),
125 },
126 }, nil
127 }
128
129 // Resolve BuildBot and LogDog build information. We do this
130 //
131 // This returns an AnnotationStream instance with its project and path
132 // populated.
133 //
134 // This function is messy and implementation-specific. That's the point of this
135 // endpoint, though. All of the nastiness here should be replaced with something
136 // more elegant once that becomes available. In the meantime...
137 func getLogDogAnnotationAddr(c context.Context, client *coordinator.Client, buil d *buildbotBuild,
138 projectHint cfgtypes.ProjectName) (*types.StreamAddr, error) {
139
140 if v, ok := build.getPropertyValue("log_location").(string); ok && v != "" {
141 addr, err := types.ParseURL(v)
142 if err == nil {
143 return addr, nil
144 }
145
146 logging.Fields{
147 logging.ErrorKey: err,
148 "log_location": v,
149 }.Debugf(c, "'log_location' property did not parse as LogDog URL .")
150 }
151
152 // logdog_annotation_url (if present, must be valid)
153 if v, ok := build.getPropertyValue("logdog_annotation_url").(string); ok && v != "" {
154 addr, err := types.ParseURL(v)
155 if err != nil {
156 logging.Fields{
157 logging.ErrorKey: err,
158 "url": v,
159 }.Errorf(c, "Failed to parse 'logdog_annotation_url' pro perty.")
160 return nil, grpcutil.Errf(codes.FailedPrecondition, "bui ld has invalid annotation URL")
161 }
162
163 return addr, nil
164 }
165
166 // Modern builds will have this information in their build properties.
167 var addr types.StreamAddr
168 prefix, _ := build.getPropertyValue("logdog_prefix").(string)
169 project, _ := build.getPropertyValue("logdog_project").(string)
170 if prefix != "" && project != "" {
171 // Construct the full annotation path.
172 addr.Project = cfgtypes.ProjectName(project)
173 addr.Path = types.StreamName(prefix).Join("annotations")
174
175 logging.Debugf(c, "Resolved path/project from build properties." )
176 return &addr, nil
177 }
178
179 // From here on out, we will need a project hint.
180 if projectHint == "" {
181 return nil, grpcutil.Errf(codes.NotFound, "annotation stream not found")
182 }
183 addr.Project = projectHint
184
185 // Execute a LogDog service query to see if we can identify the stream.
186 err := func() error {
187 var annotationStream *coordinator.LogStream
188 err := client.Query(c, addr.Project, "", coordinator.QueryOption s{
189 Tags: map[string]string{
190 "buildbot.master": build.Master,
191 "buildbot.builder": build.Buildername,
192 "buildbot.buildnumber": strconv.Itoa(build.Numbe r),
193 },
194 ContentType: miloProto.ContentTypeAnnotations,
195 }, func(ls *coordinator.LogStream) bool {
196 // Only need the first (hopefully only?) result.
197 annotationStream = ls
198 return false
199 })
200 if err != nil {
201 logging.WithError(err).Errorf(c, "Failed to issue log st ream query.")
202 return grpcutil.Internal
203 }
204
205 if annotationStream != nil {
206 addr.Path = annotationStream.Path
207 }
208 return nil
209 }()
210 if err != nil {
211 return nil, err
212 }
213 if addr.Path != "" {
214 logging.Debugf(c, "Resolved path/project via tag query.")
215 return &addr, nil
216 }
217
218 // Last-ditch effort: generate a prefix based on the build properties. T his
219 // re-implements the "_build_prefix" function in:
220 // https://chromium.googlesource.com/chromium/tools/build/+/2d23e5284cc3 1f31c6bc07aa1d3fc5b1c454c3b4/scripts/slave/logdog_bootstrap.py#363
221 isAlnum := func(r rune) bool {
222 return ((r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r > = '0' && r <= '9'))
223 }
224 normalize := func(v string) string {
225 v = strings.Map(func(r rune) rune {
226 if isAlnum(r) {
227 return r
228 }
229 switch r {
230 case ':', '_', '-', '.':
231 return r
232 default:
233 return '_'
234 }
235 }, v)
236 if r, _ := utf8.DecodeRuneInString(v); r == utf8.RuneError || !i sAlnum(r) {
237 v = "s_" + v
238 }
239 return v
240 }
241 addr.Path = types.StreamPath(fmt.Sprintf("bb/%s/%s/%s/+/annotations",
242 normalize(build.Master), normalize(build.Buildername), normalize (strconv.Itoa(build.Number))))
243
244 logging.Debugf(c, "Generated path/project algorithmically.")
245 return &addr, nil
246 }
247
248 // mergeBuildInfoIntoAnnotation merges BuildBot-specific build informtion into
249 // a LogDog annotation protobuf.
250 //
251 // This consists of augmenting the Step's properties with BuildBot's properties,
252 // favoring the Step's version of the properties if there are two with the same
253 // name.
254 func mergeBuildIntoAnnotation(c context.Context, step *miloProto.Step, build *bu ildbotBuild) error {
255 allProps := stringset.New(len(step.Property) + len(build.Properties))
256 for _, prop := range step.Property {
257 allProps.Add(prop.Name)
258 }
259 for _, prop := range build.Properties {
260 // Annotation protobuf overrides BuildBot properties.
261 if allProps.Has(prop.Name) {
262 continue
263 }
264 allProps.Add(prop.Name)
265
266 step.Property = append(step.Property, &miloProto.Step_Property{
267 Name: prop.Name,
268 Value: fmt.Sprintf("%v", prop.Value),
269 })
270 }
271
272 return nil
273 }
OLDNEW
« no previous file with comments | « milo/build_source/buildbot/builder_test.go ('k') | milo/build_source/buildbot/buildinfo_test.go » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698