Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(382)

Side by Side Diff: milo/appengine/buildbot/build.go

Issue 2944983003: [milo] {buildbucket,buildbot,swarming,logdog} -> backends/*. (Closed)
Patch Set: fix the tests Created 3 years, 6 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
« no previous file with comments | « no previous file | milo/appengine/buildbot/build_test.go » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
(Empty)
1 // Copyright 2016 The LUCI Authors. All rights reserved.
2 // Use of this source code is governed under the Apache License, Version 2.0
3 // that can be found in the LICENSE file.
4
5 package buildbot
6
7 import (
8 "encoding/json"
9 "errors"
10 "fmt"
11 "io/ioutil"
12 "math"
13 "path/filepath"
14 "regexp"
15 "sort"
16 "strings"
17 "time"
18
19 "golang.org/x/net/context"
20
21 "github.com/luci/gae/service/datastore"
22 "github.com/luci/luci-go/common/data/stringset"
23 "github.com/luci/luci-go/common/logging"
24 "github.com/luci/luci-go/milo/api/resp"
25 "github.com/luci/luci-go/milo/appengine/common/model"
26 )
27
28 var errBuildNotFound = errors.New("Build not found")
29
30 // getBuild fetches a buildbot build from the datastore and checks ACLs.
31 // The return code matches the master responses.
32 func getBuild(c context.Context, master, builder string, buildNum int) (*buildbo tBuild, error) {
33 result := &buildbotBuild{
34 Master: master,
35 Buildername: builder,
36 Number: buildNum,
37 }
38
39 err := datastore.Get(c, result)
40 err = checkAccess(c, err, result.Internal)
41 if err == errMasterNotFound {
42 err = errBuildNotFound
43 }
44
45 return result, err
46 }
47
48 // result2Status translates a buildbot result integer into a model.Status.
49 func result2Status(s *int) (status model.Status) {
50 if s == nil {
51 return model.Running
52 }
53 switch *s {
54 case 0:
55 status = model.Success
56 case 1:
57 status = model.Warning
58 case 2:
59 status = model.Failure
60 case 3:
61 status = model.NotRun // Skipped
62 case 4:
63 status = model.Exception
64 case 5:
65 status = model.WaitingDependency // Retry
66 default:
67 panic(fmt.Errorf("Unknown status %d", s))
68 }
69 return
70 }
71
72 // buildbotTimeToTime converts a buildbot time representation (pointer to float
73 // of seconds since epoch) to a native time.Time object.
74 func buildbotTimeToTime(t *float64) (result time.Time) {
75 if t != nil {
76 result = time.Unix(int64(*t), int64(*t*1e9)%1e9).UTC()
77 }
78 return
79 }
80
81 // parseTimes translates a buildbot time tuple (start, end) into a triplet
82 // of (Started time, Ending time, duration).
83 // If times[1] is nil and buildFinished is not, ended will be set to buildFinish ed
84 // time.
85 func parseTimes(buildFinished *float64, times []*float64) (started, ended time.T ime, duration time.Duration) {
86 if len(times) != 2 {
87 panic(fmt.Errorf("Expected 2 floats for times, got %v", times))
88 }
89 if times[0] == nil {
90 // Some steps don't have timing info. In that case, just return nils.
91 return
92 }
93 started = buildbotTimeToTime(times[0])
94 switch {
95 case times[1] != nil:
96 ended = buildbotTimeToTime(times[1])
97 duration = ended.Sub(started)
98 case buildFinished != nil:
99 ended = buildbotTimeToTime(buildFinished)
100 duration = ended.Sub(started)
101 default:
102 duration = time.Since(started)
103 }
104 return
105 }
106
107 // getBanner parses the OS information from the build and maybe returns a banner .
108 func getBanner(c context.Context, b *buildbotBuild) *resp.LogoBanner {
109 logging.Infof(c, "OS: %s/%s", b.OSFamily, b.OSVersion)
110 osLogo := func() *resp.Logo {
111 result := &resp.Logo{}
112 switch b.OSFamily {
113 case "windows":
114 result.LogoBase = resp.Windows
115 case "Darwin":
116 result.LogoBase = resp.OSX
117 case "Debian":
118 result.LogoBase = resp.Ubuntu
119 default:
120 return nil
121 }
122 result.Subtitle = b.OSVersion
123 return result
124 }()
125 if osLogo != nil {
126 return &resp.LogoBanner{
127 OS: []resp.Logo{*osLogo},
128 }
129 }
130 logging.Warningf(c, "No OS info found.")
131 return nil
132 }
133
134 // summary extracts the top level summary from a buildbot build as a
135 // BuildComponent
136 func summary(c context.Context, b *buildbotBuild) resp.BuildComponent {
137 // TODO(hinoka): use b.toStatus()
138 // Status
139 var status model.Status
140 if b.Currentstep != nil {
141 status = model.Running
142 } else {
143 status = result2Status(b.Results)
144 }
145
146 // Timing info
147 started, ended, duration := parseTimes(nil, b.Times)
148
149 // Link to bot and original build.
150 host := "build.chromium.org/p"
151 if b.Internal {
152 host = "uberchromegw.corp.google.com/i"
153 }
154 bot := resp.NewLink(
155 b.Slave,
156 fmt.Sprintf("https://%s/%s/buildslaves/%s", host, b.Master, b.Sl ave),
157 )
158 source := resp.NewLink(
159 fmt.Sprintf("%s/%s/%d", b.Master, b.Buildername, b.Number),
160 fmt.Sprintf("https://%s/%s/builders/%s/builds/%d",
161 host, b.Master, b.Buildername, b.Number),
162 )
163
164 // The link to the builder page.
165 parent := resp.NewLink(b.Buildername, ".")
166
167 // Do a best effort lookup for the bot information to fill in OS/Platfor m info.
168 banner := getBanner(c, b)
169
170 sum := resp.BuildComponent{
171 ParentLabel: parent,
172 Label: fmt.Sprintf("#%d", b.Number),
173 Banner: banner,
174 Status: status,
175 Started: started,
176 Finished: ended,
177 Bot: bot,
178 Source: source,
179 Duration: duration,
180 Type: resp.Summary, // This is more or less ignored.
181 LevelsDeep: 1,
182 Text: []string{}, // Status messages. Eg "This build fai led on..xyz"
183 }
184
185 return sum
186 }
187
188 var rLineBreak = regexp.MustCompile("<br */?>")
189
190 // components takes a full buildbot build struct and extract step info from all
191 // of the steps and returns it as a list of milo Build Components.
192 func components(b *buildbotBuild) (result []*resp.BuildComponent) {
193 endingTime := b.Times[1]
194 for _, step := range b.Steps {
195 if step.Hidden == true {
196 continue
197 }
198 bc := &resp.BuildComponent{
199 Label: step.Name,
200 }
201 // Step text sometimes contains <br>, which we want to parse int o new lines.
202 for _, t := range step.Text {
203 for _, line := range rLineBreak.Split(t, -1) {
204 bc.Text = append(bc.Text, line)
205 }
206 }
207
208 // Figure out the status.
209 if !step.IsStarted {
210 bc.Status = model.NotRun
211 } else if !step.IsFinished {
212 bc.Status = model.Running
213 } else {
214 if len(step.Results) > 0 {
215 status := int(step.Results[0].(float64))
216 bc.Status = result2Status(&status)
217 } else {
218 bc.Status = model.Success
219 }
220 }
221
222 // Raise the interesting-ness if the step is not "Success".
223 if bc.Status != model.Success {
224 bc.Verbosity = resp.Interesting
225 }
226
227 remainingAliases := stringset.New(len(step.Aliases))
228 for linkAnchor := range step.Aliases {
229 remainingAliases.Add(linkAnchor)
230 }
231
232 getLinksWithAliases := func(logLink *resp.Link, isLog bool) resp .LinkSet {
233 // Generate alias links.
234 var aliases resp.LinkSet
235 if remainingAliases.Del(logLink.Label) {
236 stepAliases := step.Aliases[logLink.Label]
237 aliases = make(resp.LinkSet, len(stepAliases))
238 for i, alias := range stepAliases {
239 aliases[i] = alias.toLink()
240 }
241 }
242
243 // Step log link takes primary, with aliases as secondar y.
244 links := make(resp.LinkSet, 1, 1+len(aliases))
245 links[0] = logLink
246
247 for _, a := range aliases {
248 a.Alias = true
249 }
250 return append(links, aliases...)
251 }
252
253 for _, l := range step.Logs {
254 logLink := resp.NewLink(l[0], l[1])
255
256 links := getLinksWithAliases(logLink, true)
257 if logLink.Label == "stdio" {
258 bc.MainLink = links
259 } else {
260 bc.SubLink = append(bc.SubLink, links)
261 }
262 }
263
264 // Step links are stored as maps of name: url
265 // Because Go doesn't believe in nice things, we now create anot her array
266 // just so that we can iterate through this map in order.
267 names := make([]string, 0, len(step.Urls))
268 for name := range step.Urls {
269 names = append(names, name)
270 }
271 sort.Strings(names)
272 for _, name := range names {
273 logLink := resp.NewLink(name, step.Urls[name])
274
275 bc.SubLink = append(bc.SubLink, getLinksWithAliases(logL ink, false))
276 }
277
278 // Add any unused aliases directly.
279 if remainingAliases.Len() > 0 {
280 unusedAliases := remainingAliases.ToSlice()
281 sort.Strings(unusedAliases)
282
283 for _, label := range unusedAliases {
284 var baseLink resp.LinkSet
285 for _, alias := range step.Aliases[label] {
286 aliasLink := alias.toLink()
287 if len(baseLink) == 0 {
288 aliasLink.Label = label
289 } else {
290 aliasLink.Alias = true
291 }
292 baseLink = append(baseLink, aliasLink)
293 }
294
295 if len(baseLink) > 0 {
296 bc.SubLink = append(bc.SubLink, baseLink )
297 }
298 }
299 }
300
301 // Figure out the times.
302 bc.Started, bc.Finished, bc.Duration = parseTimes(endingTime, st ep.Times)
303
304 result = append(result, bc)
305 }
306 return
307 }
308
309 // parseProp returns a string representation of v.
310 func parseProp(v interface{}) string {
311 // if v is a whole number, force it into an int. json.Marshal() would t urn
312 // it into what looks like a float instead. We want this to remain and
313 // int instead of a number.
314 if vf, ok := v.(float64); ok {
315 if math.Floor(vf) == vf {
316 return fmt.Sprintf("%d", int64(vf))
317 }
318 }
319 // return the json representation of the value.
320 b, err := json.Marshal(v)
321 if err == nil {
322 return string(b)
323 }
324 return fmt.Sprintf("%v", v)
325 }
326
327 // Prop is a struct used to store a value and group so that we can make a map
328 // of key:Prop to pass into parseProp() for the purpose of cross referencing
329 // one prop while working on another.
330 type Prop struct {
331 Value interface{}
332 Group string
333 }
334
335 // properties extracts all properties from buildbot builds and groups them into
336 // property groups.
337 func properties(b *buildbotBuild) (result []*resp.PropertyGroup) {
338 groups := map[string]*resp.PropertyGroup{}
339 allProps := map[string]Prop{}
340 for _, prop := range b.Properties {
341 allProps[prop.Name] = Prop{
342 Value: prop.Value,
343 Group: prop.Source,
344 }
345 }
346 for key, prop := range allProps {
347 value := prop.Value
348 groupName := prop.Group
349 if _, ok := groups[groupName]; !ok {
350 groups[groupName] = &resp.PropertyGroup{GroupName: group Name}
351 }
352 vs := parseProp(value)
353 groups[groupName].Property = append(groups[groupName].Property, &resp.Property{
354 Key: key,
355 Value: vs,
356 })
357 }
358 // Insert the groups into a list in alphabetical order.
359 // You have to make a separate sorting data structure because Go doesn't like
360 // sorting things for you.
361 groupNames := []string{}
362 for n := range groups {
363 groupNames = append(groupNames, n)
364 }
365 sort.Strings(groupNames)
366 for _, k := range groupNames {
367 group := groups[k]
368 // Also take this oppertunity to sort the properties within the groups.
369 sort.Sort(group)
370 result = append(result, group)
371 }
372 return
373 }
374
375 // blame extracts the commit and blame information from a buildbot build and
376 // returns it as a list of Commits.
377 func blame(b *buildbotBuild) (result []*resp.Commit) {
378 for _, c := range b.Sourcestamp.Changes {
379 files := c.GetFiles()
380 result = append(result, &resp.Commit{
381 AuthorEmail: c.Who,
382 Repo: c.Repository,
383 CommitTime: time.Unix(int64(c.When), 0).UTC(),
384 Revision: resp.NewLink(c.Revision, c.Revlink),
385 Description: c.Comments,
386 Title: strings.Split(c.Comments, "\n")[0],
387 File: files,
388 })
389 }
390 return
391 }
392
393 // sourcestamp extracts the source stamp from various parts of a buildbot build,
394 // including the properties.
395 func sourcestamp(c context.Context, b *buildbotBuild) *resp.SourceStamp {
396 ss := &resp.SourceStamp{}
397 rietveld := ""
398 got_revision := ""
399 repository := ""
400 issue := int64(-1)
401 // TODO(hinoka): Gerrit URLs.
402 for _, prop := range b.Properties {
403 switch prop.Name {
404 case "rietveld":
405 if v, ok := prop.Value.(string); ok {
406 rietveld = v
407 } else {
408 logging.Warningf(c, "Field rietveld is not a str ing: %#v", prop.Value)
409 }
410 case "issue":
411 if v, ok := prop.Value.(float64); ok {
412 issue = int64(v)
413 } else {
414 logging.Warningf(c, "Field issue is not a float: %#v", prop.Value)
415 }
416
417 case "got_revision":
418 if v, ok := prop.Value.(string); ok {
419 got_revision = v
420 } else {
421 logging.Warningf(c, "Field got_revision is not a string: %#v", prop.Value)
422 }
423
424 case "repository":
425 if v, ok := prop.Value.(string); ok {
426 repository = v
427 }
428 }
429 }
430 if issue != -1 {
431 if rietveld != "" {
432 rietveld = strings.TrimRight(rietveld, "/")
433 ss.Changelist = resp.NewLink(
434 fmt.Sprintf("Issue %d", issue),
435 fmt.Sprintf("%s/%d", rietveld, issue),
436 )
437 } else {
438 logging.Warningf(c, "Found issue but not rietveld proper ty.")
439 }
440 }
441 if got_revision != "" {
442 ss.Revision = resp.NewLink(got_revision, "")
443 if repository != "" {
444 ss.Revision.URL = repository + "/+/" + got_revision
445 }
446 }
447 return ss
448 }
449
450 func getDebugBuild(c context.Context, builder string, buildNum int) (*buildbotBu ild, error) {
451 fname := fmt.Sprintf("%s.%d.json", builder, buildNum)
452 // ../buildbot below assumes that
453 // - this code is not executed by tests outside of this dir
454 // - this dir is a sibling of frontend dir
455 path := filepath.Join("..", "buildbot", "testdata", fname)
456 raw, err := ioutil.ReadFile(path)
457 if err != nil {
458 return nil, err
459 }
460 b := &buildbotBuild{}
461 return b, json.Unmarshal(raw, b)
462 }
463
464 // build fetches a buildbot build and translates it into a miloBuild.
465 func Build(c context.Context, master, builder string, buildNum int) (*resp.MiloB uild, error) {
466 var b *buildbotBuild
467 var err error
468 if master == "debug" {
469 b, err = getDebugBuild(c, builder, buildNum)
470 } else {
471 b, err = getBuild(c, master, builder, buildNum)
472 }
473 if err != nil {
474 return nil, err
475 }
476
477 // Modify the build for rendering.
478 updatePostProcessBuild(b)
479
480 // TODO(hinoka): Do all fields concurrently.
481 return &resp.MiloBuild{
482 SourceStamp: sourcestamp(c, b),
483 Summary: summary(c, b),
484 Components: components(b),
485 PropertyGroup: properties(b),
486 Blame: blame(b),
487 }, nil
488 }
489
490 // updatePostProcessBuild transforms a build from its raw JSON format into the
491 // format that should be presented to users.
492 //
493 // Post-processing includes:
494 // - If the build is LogDog-only, promotes aliases (LogDog links) to
495 // first-class links in the build.
496 func updatePostProcessBuild(b *buildbotBuild) {
497 // If this is a LogDog-only build, we want to promote the LogDog links.
498 if loc, ok := b.getPropertyValue("log_location").(string); ok && strings .HasPrefix(loc, "logdog://") {
499 linkMap := map[string]string{}
500 for sidx := range b.Steps {
501 promoteLogDogLinks(&b.Steps[sidx], sidx == 0, linkMap)
502 }
503
504 // Update "Logs". This field is part of BuildBot, and is the ama lgamation
505 // of all logs in the build's steps. Since each log is out of co ntext of its
506 // original step, we can't apply the promotion logic; instead, w e will use
507 // the link map to map any old URLs that were matched in "promot eLogDogLnks"
508 // to their new URLs.
509 for _, link := range b.Logs {
510 // "link" is in the form: [NAME, URL]
511 if len(link) != 2 {
512 continue
513 }
514
515 if newURL, ok := linkMap[link[1]]; ok {
516 link[1] = newURL
517 }
518 }
519 }
520 }
521
522 // promoteLogDogLinks updates the links in a BuildBot step to
523 // promote LogDog links.
524 //
525 // A build's links come in one of three forms:
526 // - Log Links, which link directly to BuildBot build logs.
527 // - URL Links, which are named links to arbitrary URLs.
528 // - Aliases, which attach to the label in one of the other types of links and
529 // augment it with additional named links.
530 //
531 // LogDog uses aliases exclusively to attach LogDog logs to other links. When
532 // the build is LogDog-only, though, the original links are actually junk. What
533 // we want to do is remove the original junk links and replace them with their
534 // alias counterparts, so that the "natural" BuildBot links are actually LogDog
535 // links.
536 //
537 // As URLs are re-mapped, the supplied "linkMap" will be updated to map the old
538 // URLs to the new ones.
539 func promoteLogDogLinks(s *buildbotStep, isInitialStep bool, linkMap map[string] string) {
540 type stepLog struct {
541 label string
542 url string
543 }
544
545 remainingAliases := stringset.New(len(s.Aliases))
546 for linkAnchor := range s.Aliases {
547 remainingAliases.Add(linkAnchor)
548 }
549
550 maybePromoteAliases := func(sl *stepLog, isLog bool) []*stepLog {
551 // As a special case, if this is the first step ("steps" in Buil dBot), we
552 // will refrain from promoting aliases for "stdio", since "stdio " represents
553 // the raw BuildBot logs.
554 if isLog && isInitialStep && sl.label == "stdio" {
555 // No aliases, don't modify this log.
556 return []*stepLog{sl}
557 }
558
559 // If there are no aliases, we should obviously not promote them . This will
560 // be the case for pre-LogDog steps such as build setup.
561 aliases := s.Aliases[sl.label]
562 if len(aliases) == 0 {
563 return []*stepLog{sl}
564 }
565
566 // We have chosen to promote the aliases. Therefore, we will not include
567 // them as aliases in the modified step.
568 remainingAliases.Del(sl.label)
569
570 result := make([]*stepLog, len(aliases))
571 for i, alias := range aliases {
572 aliasStepLog := stepLog{alias.Text, alias.URL}
573
574 // Any link named "logdog" (Annotee cosmetic implementat ion detail) will
575 // inherit the name of the original log.
576 if isLog {
577 if aliasStepLog.label == "logdog" {
578 aliasStepLog.label = sl.label
579 }
580 }
581
582 result[i] = &aliasStepLog
583 }
584
585 // If we performed mapping, add the OLD -> NEW URL mapping to li nkMap.
586 //
587 // Since multpiple aliases can apply to a single log, and we hav e to pick
588 // one, here, we'll arbitrarily pick the last one. This is maybe more
589 // consistent than the first one because linkMap, itself, will e nd up
590 // holding the last mapping for any given URL.
591 if len(result) > 0 {
592 linkMap[sl.url] = result[len(result)-1].url
593 }
594
595 return result
596 }
597
598 // Update step logs.
599 newLogs := make([][]string, 0, len(s.Logs))
600 for _, l := range s.Logs {
601 for _, res := range maybePromoteAliases(&stepLog{l[0], l[1]}, tr ue) {
602 newLogs = append(newLogs, []string{res.label, res.url})
603 }
604 }
605 s.Logs = newLogs
606
607 // Update step URLs.
608 newURLs := make(map[string]string, len(s.Urls))
609 for label, link := range s.Urls {
610 urlLinks := maybePromoteAliases(&stepLog{label, link}, false)
611 if len(urlLinks) > 0 {
612 // Use the last URL link, since our URL map can only tol erate one link.
613 // The expected case here is that len(urlLinks) == 1, th ough, but it's
614 // possible that multiple aliases can be included for a single URL, so
615 // we need to handle that.
616 newValue := urlLinks[len(urlLinks)-1]
617 newURLs[newValue.label] = newValue.url
618 } else {
619 newURLs[label] = link
620 }
621 }
622 s.Urls = newURLs
623
624 // Preserve any aliases that haven't been promoted.
625 var newAliases map[string][]*buildbotLinkAlias
626 if l := remainingAliases.Len(); l > 0 {
627 newAliases = make(map[string][]*buildbotLinkAlias, l)
628 remainingAliases.Iter(func(v string) bool {
629 newAliases[v] = s.Aliases[v]
630 return true
631 })
632 }
633 s.Aliases = newAliases
634 }
OLDNEW
« no previous file with comments | « no previous file | milo/appengine/buildbot/build_test.go » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698