Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(9)

Side by Side Diff: milo/appengine/buildbot/build.go

Issue 2944633003: [milo] Add BuildSummary and common models. (Closed)
Patch Set: add comments Created 3 years, 6 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
« no previous file with comments | « milo/appengine/backend/git/gitiles.go ('k') | milo/appengine/buildbot/builder.go » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 // Copyright 2016 The LUCI Authors. All rights reserved. 1 // Copyright 2016 The LUCI Authors. All rights reserved.
2 // Use of this source code is governed under the Apache License, Version 2.0 2 // Use of this source code is governed under the Apache License, Version 2.0
3 // that can be found in the LICENSE file. 3 // that can be found in the LICENSE file.
4 4
5 package buildbot 5 package buildbot
6 6
7 import ( 7 import (
8 "encoding/json" 8 "encoding/json"
9 "errors" 9 "errors"
10 "fmt" 10 "fmt"
11 "io/ioutil" 11 "io/ioutil"
12 "math" 12 "math"
13 "path/filepath" 13 "path/filepath"
14 "regexp" 14 "regexp"
15 "sort" 15 "sort"
16 "strings" 16 "strings"
17 "time" 17 "time"
18 18
19 "golang.org/x/net/context" 19 "golang.org/x/net/context"
20 20
21 "github.com/luci/gae/service/datastore" 21 "github.com/luci/gae/service/datastore"
22 "github.com/luci/luci-go/common/data/stringset" 22 "github.com/luci/luci-go/common/data/stringset"
23 "github.com/luci/luci-go/common/logging" 23 "github.com/luci/luci-go/common/logging"
24 "github.com/luci/luci-go/milo/api/resp" 24 "github.com/luci/luci-go/milo/api/resp"
25 "github.com/luci/luci-go/milo/appengine/common/model"
25 ) 26 )
26 27
27 var errBuildNotFound = errors.New("Build not found") 28 var errBuildNotFound = errors.New("Build not found")
28 29
29 // getBuild fetches a buildbot build from the datastore and checks ACLs. 30 // getBuild fetches a buildbot build from the datastore and checks ACLs.
30 // The return code matches the master responses. 31 // The return code matches the master responses.
31 func getBuild(c context.Context, master, builder string, buildNum int) (*buildbo tBuild, error) { 32 func getBuild(c context.Context, master, builder string, buildNum int) (*buildbo tBuild, error) {
32 result := &buildbotBuild{ 33 result := &buildbotBuild{
33 Master: master, 34 Master: master,
34 Buildername: builder, 35 Buildername: builder,
35 Number: buildNum, 36 Number: buildNum,
36 } 37 }
37 38
38 err := datastore.Get(c, result) 39 err := datastore.Get(c, result)
39 err = checkAccess(c, err, result.Internal) 40 err = checkAccess(c, err, result.Internal)
40 if err == errMasterNotFound { 41 if err == errMasterNotFound {
41 err = errBuildNotFound 42 err = errBuildNotFound
42 } 43 }
43 44
44 return result, err 45 return result, err
45 } 46 }
46 47
47 // result2Status translates a buildbot result integer into a resp.Status. 48 // result2Status translates a buildbot result integer into a model.Status.
48 func result2Status(s *int) (status resp.Status) { 49 func result2Status(s *int) (status model.Status) {
49 if s == nil { 50 if s == nil {
50 » » return resp.Running 51 » » return model.Running
51 } 52 }
52 switch *s { 53 switch *s {
53 case 0: 54 case 0:
54 » » status = resp.Success 55 » » status = model.Success
55 case 1: 56 case 1:
56 » » status = resp.Warning 57 » » status = model.Warning
57 case 2: 58 case 2:
58 » » status = resp.Failure 59 » » status = model.Failure
59 case 3: 60 case 3:
60 » » status = resp.NotRun // Skipped 61 » » status = model.NotRun // Skipped
61 case 4: 62 case 4:
62 » » status = resp.Exception 63 » » status = model.Exception
63 case 5: 64 case 5:
64 » » status = resp.WaitingDependency // Retry 65 » » status = model.WaitingDependency // Retry
65 default: 66 default:
66 panic(fmt.Errorf("Unknown status %d", s)) 67 panic(fmt.Errorf("Unknown status %d", s))
67 } 68 }
68 return 69 return
69 } 70 }
70 71
71 // buildbotTimeToTime converts a buildbot time representation (pointer to float 72 // buildbotTimeToTime converts a buildbot time representation (pointer to float
72 // of seconds since epoch) to a native time.Time object. 73 // of seconds since epoch) to a native time.Time object.
73 func buildbotTimeToTime(t *float64) (result time.Time) { 74 func buildbotTimeToTime(t *float64) (result time.Time) {
74 if t != nil { 75 if t != nil {
(...skipping 53 matching lines...) Expand 10 before | Expand all | Expand 10 after
128 } 129 }
129 logging.Warningf(c, "No OS info found.") 130 logging.Warningf(c, "No OS info found.")
130 return nil 131 return nil
131 } 132 }
132 133
133 // summary extracts the top level summary from a buildbot build as a 134 // summary extracts the top level summary from a buildbot build as a
134 // BuildComponent 135 // BuildComponent
135 func summary(c context.Context, b *buildbotBuild) resp.BuildComponent { 136 func summary(c context.Context, b *buildbotBuild) resp.BuildComponent {
136 // TODO(hinoka): use b.toStatus() 137 // TODO(hinoka): use b.toStatus()
137 // Status 138 // Status
138 » var status resp.Status 139 » var status model.Status
139 if b.Currentstep != nil { 140 if b.Currentstep != nil {
140 » » status = resp.Running 141 » » status = model.Running
141 } else { 142 } else {
142 status = result2Status(b.Results) 143 status = result2Status(b.Results)
143 } 144 }
144 145
145 // Timing info 146 // Timing info
146 started, ended, duration := parseTimes(nil, b.Times) 147 started, ended, duration := parseTimes(nil, b.Times)
147 148
148 // Link to bot and original build. 149 // Link to bot and original build.
149 host := "build.chromium.org/p" 150 host := "build.chromium.org/p"
150 if b.Internal { 151 if b.Internal {
151 host = "uberchromegw.corp.google.com/i" 152 host = "uberchromegw.corp.google.com/i"
152 } 153 }
153 » bot := &resp.Link{ 154 » bot := resp.NewLink(
154 » » Label: b.Slave, 155 » » b.Slave,
155 » » URL: fmt.Sprintf("https://%s/%s/buildslaves/%s", host, b.Maste r, b.Slave), 156 » » fmt.Sprintf("https://%s/%s/buildslaves/%s", host, b.Master, b.Sl ave),
156 » } 157 » )
157 » source := &resp.Link{ 158 » source := resp.NewLink(
158 » » Label: fmt.Sprintf("%s/%s/%d", b.Master, b.Buildername, b.Number ), 159 » » fmt.Sprintf("%s/%s/%d", b.Master, b.Buildername, b.Number),
159 » » URL: fmt.Sprintf( 160 » » fmt.Sprintf("https://%s/%s/builders/%s/builds/%d",
160 » » » "https://%s/%s/builders/%s/builds/%d",
161 host, b.Master, b.Buildername, b.Number), 161 host, b.Master, b.Buildername, b.Number),
162 » } 162 » )
163 163
164 // The link to the builder page. 164 // The link to the builder page.
165 » parent := &resp.Link{ 165 » parent := resp.NewLink(b.Buildername, ".")
166 » » Label: b.Buildername,
167 » » URL: ".",
168 » }
169 166
170 // Do a best effort lookup for the bot information to fill in OS/Platfor m info. 167 // Do a best effort lookup for the bot information to fill in OS/Platfor m info.
171 banner := getBanner(c, b) 168 banner := getBanner(c, b)
172 169
173 sum := resp.BuildComponent{ 170 sum := resp.BuildComponent{
174 ParentLabel: parent, 171 ParentLabel: parent,
175 Label: fmt.Sprintf("#%d", b.Number), 172 Label: fmt.Sprintf("#%d", b.Number),
176 Banner: banner, 173 Banner: banner,
177 Status: status, 174 Status: status,
178 Started: started, 175 Started: started,
(...skipping 24 matching lines...) Expand all
203 } 200 }
204 // Step text sometimes contains <br>, which we want to parse int o new lines. 201 // Step text sometimes contains <br>, which we want to parse int o new lines.
205 for _, t := range step.Text { 202 for _, t := range step.Text {
206 for _, line := range rLineBreak.Split(t, -1) { 203 for _, line := range rLineBreak.Split(t, -1) {
207 bc.Text = append(bc.Text, line) 204 bc.Text = append(bc.Text, line)
208 } 205 }
209 } 206 }
210 207
211 // Figure out the status. 208 // Figure out the status.
212 if !step.IsStarted { 209 if !step.IsStarted {
213 » » » bc.Status = resp.NotRun 210 » » » bc.Status = model.NotRun
214 } else if !step.IsFinished { 211 } else if !step.IsFinished {
215 » » » bc.Status = resp.Running 212 » » » bc.Status = model.Running
216 } else { 213 } else {
217 if len(step.Results) > 0 { 214 if len(step.Results) > 0 {
218 status := int(step.Results[0].(float64)) 215 status := int(step.Results[0].(float64))
219 bc.Status = result2Status(&status) 216 bc.Status = result2Status(&status)
220 } else { 217 } else {
221 » » » » bc.Status = resp.Success 218 » » » » bc.Status = model.Success
222 } 219 }
223 } 220 }
224 221
225 // Raise the interesting-ness if the step is not "Success". 222 // Raise the interesting-ness if the step is not "Success".
226 » » if bc.Status != resp.Success { 223 » » if bc.Status != model.Success {
227 bc.Verbosity = resp.Interesting 224 bc.Verbosity = resp.Interesting
228 } 225 }
229 226
230 remainingAliases := stringset.New(len(step.Aliases)) 227 remainingAliases := stringset.New(len(step.Aliases))
231 for linkAnchor := range step.Aliases { 228 for linkAnchor := range step.Aliases {
232 remainingAliases.Add(linkAnchor) 229 remainingAliases.Add(linkAnchor)
233 } 230 }
234 231
235 getLinksWithAliases := func(logLink *resp.Link, isLog bool) resp .LinkSet { 232 getLinksWithAliases := func(logLink *resp.Link, isLog bool) resp .LinkSet {
236 // Generate alias links. 233 // Generate alias links.
(...skipping 10 matching lines...) Expand all
247 links := make(resp.LinkSet, 1, 1+len(aliases)) 244 links := make(resp.LinkSet, 1, 1+len(aliases))
248 links[0] = logLink 245 links[0] = logLink
249 246
250 for _, a := range aliases { 247 for _, a := range aliases {
251 a.Alias = true 248 a.Alias = true
252 } 249 }
253 return append(links, aliases...) 250 return append(links, aliases...)
254 } 251 }
255 252
256 for _, l := range step.Logs { 253 for _, l := range step.Logs {
257 » » » logLink := resp.Link{ 254 » » » logLink := resp.NewLink(l[0], l[1])
258 » » » » Label: l[0],
259 » » » » URL: l[1],
260 » » » }
261 255
262 » » » links := getLinksWithAliases(&logLink, true) 256 » » » links := getLinksWithAliases(logLink, true)
263 if logLink.Label == "stdio" { 257 if logLink.Label == "stdio" {
264 bc.MainLink = links 258 bc.MainLink = links
265 } else { 259 } else {
266 bc.SubLink = append(bc.SubLink, links) 260 bc.SubLink = append(bc.SubLink, links)
267 } 261 }
268 } 262 }
269 263
270 // Step links are stored as maps of name: url 264 // Step links are stored as maps of name: url
271 // Because Go doesn't believe in nice things, we now create anot her array 265 // Because Go doesn't believe in nice things, we now create anot her array
272 // just so that we can iterate through this map in order. 266 // just so that we can iterate through this map in order.
273 names := make([]string, 0, len(step.Urls)) 267 names := make([]string, 0, len(step.Urls))
274 for name := range step.Urls { 268 for name := range step.Urls {
275 names = append(names, name) 269 names = append(names, name)
276 } 270 }
277 sort.Strings(names) 271 sort.Strings(names)
278 for _, name := range names { 272 for _, name := range names {
279 » » » logLink := resp.Link{ 273 » » » logLink := resp.NewLink(name, step.Urls[name])
280 » » » » Label: name,
281 » » » » URL: step.Urls[name],
282 » » » }
283 274
284 » » » bc.SubLink = append(bc.SubLink, getLinksWithAliases(&log Link, false)) 275 » » » bc.SubLink = append(bc.SubLink, getLinksWithAliases(logL ink, false))
285 } 276 }
286 277
287 // Add any unused aliases directly. 278 // Add any unused aliases directly.
288 if remainingAliases.Len() > 0 { 279 if remainingAliases.Len() > 0 {
289 unusedAliases := remainingAliases.ToSlice() 280 unusedAliases := remainingAliases.ToSlice()
290 sort.Strings(unusedAliases) 281 sort.Strings(unusedAliases)
291 282
292 for _, label := range unusedAliases { 283 for _, label := range unusedAliases {
293 var baseLink resp.LinkSet 284 var baseLink resp.LinkSet
294 for _, alias := range step.Aliases[label] { 285 for _, alias := range step.Aliases[label] {
(...skipping 88 matching lines...) Expand 10 before | Expand all | Expand 10 after
383 374
384 // blame extracts the commit and blame information from a buildbot build and 375 // blame extracts the commit and blame information from a buildbot build and
385 // returns it as a list of Commits. 376 // returns it as a list of Commits.
386 func blame(b *buildbotBuild) (result []*resp.Commit) { 377 func blame(b *buildbotBuild) (result []*resp.Commit) {
387 for _, c := range b.Sourcestamp.Changes { 378 for _, c := range b.Sourcestamp.Changes {
388 files := c.GetFiles() 379 files := c.GetFiles()
389 result = append(result, &resp.Commit{ 380 result = append(result, &resp.Commit{
390 AuthorEmail: c.Who, 381 AuthorEmail: c.Who,
391 Repo: c.Repository, 382 Repo: c.Repository,
392 CommitTime: time.Unix(int64(c.When), 0).UTC(), 383 CommitTime: time.Unix(int64(c.When), 0).UTC(),
393 » » » Revision: &resp.Link{ 384 » » » Revision: resp.NewLink(c.Revision, c.Revlink),
394 » » » » URL: c.Revlink,
395 » » » » Label: c.Revision,
396 » » » },
397 Description: c.Comments, 385 Description: c.Comments,
398 Title: strings.Split(c.Comments, "\n")[0], 386 Title: strings.Split(c.Comments, "\n")[0],
399 File: files, 387 File: files,
400 }) 388 })
401 } 389 }
402 return 390 return
403 } 391 }
404 392
405 // sourcestamp extracts the source stamp from various parts of a buildbot build, 393 // sourcestamp extracts the source stamp from various parts of a buildbot build,
406 // including the properties. 394 // including the properties.
(...skipping 28 matching lines...) Expand all
435 423
436 case "repository": 424 case "repository":
437 if v, ok := prop.Value.(string); ok { 425 if v, ok := prop.Value.(string); ok {
438 repository = v 426 repository = v
439 } 427 }
440 } 428 }
441 } 429 }
442 if issue != -1 { 430 if issue != -1 {
443 if rietveld != "" { 431 if rietveld != "" {
444 rietveld = strings.TrimRight(rietveld, "/") 432 rietveld = strings.TrimRight(rietveld, "/")
445 » » » ss.Changelist = &resp.Link{ 433 » » » ss.Changelist = resp.NewLink(
446 » » » » Label: fmt.Sprintf("Issue %d", issue), 434 » » » » fmt.Sprintf("Issue %d", issue),
447 » » » » URL: fmt.Sprintf("%s/%d", rietveld, issue), 435 » » » » fmt.Sprintf("%s/%d", rietveld, issue),
448 » » » } 436 » » » )
449 } else { 437 } else {
450 logging.Warningf(c, "Found issue but not rietveld proper ty.") 438 logging.Warningf(c, "Found issue but not rietveld proper ty.")
451 } 439 }
452 } 440 }
453 if got_revision != "" { 441 if got_revision != "" {
454 » » ss.Revision = &resp.Link{ 442 » » ss.Revision = resp.NewLink(got_revision, "")
455 » » » Label: got_revision,
456 » » }
457 if repository != "" { 443 if repository != "" {
458 ss.Revision.URL = repository + "/+/" + got_revision 444 ss.Revision.URL = repository + "/+/" + got_revision
459 } 445 }
460 } 446 }
461 return ss 447 return ss
462 } 448 }
463 449
464 func getDebugBuild(c context.Context, builder string, buildNum int) (*buildbotBu ild, error) { 450 func getDebugBuild(c context.Context, builder string, buildNum int) (*buildbotBu ild, error) {
465 fname := fmt.Sprintf("%s.%d.json", builder, buildNum) 451 fname := fmt.Sprintf("%s.%d.json", builder, buildNum)
466 // ../buildbot below assumes that 452 // ../buildbot below assumes that
(...skipping 172 matching lines...) Expand 10 before | Expand all | Expand 10 after
639 var newAliases map[string][]*buildbotLinkAlias 625 var newAliases map[string][]*buildbotLinkAlias
640 if l := remainingAliases.Len(); l > 0 { 626 if l := remainingAliases.Len(); l > 0 {
641 newAliases = make(map[string][]*buildbotLinkAlias, l) 627 newAliases = make(map[string][]*buildbotLinkAlias, l)
642 remainingAliases.Iter(func(v string) bool { 628 remainingAliases.Iter(func(v string) bool {
643 newAliases[v] = s.Aliases[v] 629 newAliases[v] = s.Aliases[v]
644 return true 630 return true
645 }) 631 })
646 } 632 }
647 s.Aliases = newAliases 633 s.Aliases = newAliases
648 } 634 }
OLDNEW
« no previous file with comments | « milo/appengine/backend/git/gitiles.go ('k') | milo/appengine/buildbot/builder.go » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698