| OLD | NEW |
|---|
| (Empty) |
| 1 // Copyright 2016 The LUCI Authors. All rights reserved. | |
| 2 // Use of this source code is governed under the Apache License, Version 2.0 | |
| 3 // that can be found in the LICENSE file. | |
| 4 | |
| 5 package buildbucket | |
| 6 | |
| 7 import ( | |
| 8 "encoding/json" | |
| 9 "fmt" | |
| 10 "net/url" | |
| 11 "os" | |
| 12 "path" | |
| 13 "path/filepath" | |
| 14 "strconv" | |
| 15 "strings" | |
| 16 "time" | |
| 17 | |
| 18 "golang.org/x/net/context" | |
| 19 "google.golang.org/api/googleapi" | |
| 20 | |
| 21 "github.com/luci/gae/service/info" | |
| 22 "github.com/luci/luci-go/common/api/buildbucket/buildbucket/v1" | |
| 23 "github.com/luci/luci-go/common/clock" | |
| 24 "github.com/luci/luci-go/common/errors" | |
| 25 "github.com/luci/luci-go/common/logging" | |
| 26 "github.com/luci/luci-go/common/retry" | |
| 27 "github.com/luci/luci-go/common/retry/transient" | |
| 28 "github.com/luci/luci-go/common/sync/parallel" | |
| 29 | |
| 30 "github.com/luci/luci-go/milo/api/resp" | |
| 31 "github.com/luci/luci-go/milo/common" | |
| 32 "github.com/luci/luci-go/milo/common/model" | |
| 33 ) | |
| 34 | |
| 35 // search executes the search request with retries and exponential back-off. | |
| 36 func search(c context.Context, client *buildbucket.Service, req *buildbucket.Sea
rchCall) ( | |
| 37 *buildbucket.ApiSearchResponseMessage, error) { | |
| 38 | |
| 39 var res *buildbucket.ApiSearchResponseMessage | |
| 40 err := retry.Retry( | |
| 41 c, | |
| 42 transient.Only(retry.Default), | |
| 43 func() error { | |
| 44 var err error | |
| 45 res, err = req.Do() | |
| 46 if apiErr, ok := err.(*googleapi.Error); ok && apiErr.Co
de >= 500 { | |
| 47 err = transient.Tag.Apply(apiErr) | |
| 48 } | |
| 49 return err | |
| 50 }, | |
| 51 func(err error, wait time.Duration) { | |
| 52 logging.WithError(err).Warningf(c, "buildbucket search r
equest failed transiently, will retry in %s", wait) | |
| 53 }) | |
| 54 return res, err | |
| 55 } | |
| 56 | |
| 57 // fetchBuilds fetches builds given a criteria. | |
| 58 // The returned builds are sorted by build creation descending. | |
| 59 // count defines maximum number of builds to fetch; if <0, defaults to 100. | |
| 60 func fetchBuilds(c context.Context, client *buildbucket.Service, bucket, builder
, | |
| 61 status string, count int) ([]*buildbucket.ApiCommonBuildMessage, error)
{ | |
| 62 | |
| 63 req := client.Search() | |
| 64 req.Bucket(bucket) | |
| 65 req.Status(status) | |
| 66 req.Tag("builder:" + builder) | |
| 67 | |
| 68 if count < 0 { | |
| 69 count = 100 | |
| 70 } | |
| 71 | |
| 72 fetched := make([]*buildbucket.ApiCommonBuildMessage, 0, count) | |
| 73 start := clock.Now(c) | |
| 74 for len(fetched) < count { | |
| 75 req.MaxBuilds(int64(count - len(fetched))) | |
| 76 | |
| 77 res, err := search(c, client, req) | |
| 78 switch { | |
| 79 case err != nil: | |
| 80 return fetched, err | |
| 81 case res.Error != nil: | |
| 82 return fetched, fmt.Errorf(res.Error.Message) | |
| 83 } | |
| 84 | |
| 85 fetched = append(fetched, res.Builds...) | |
| 86 | |
| 87 if len(res.Builds) == 0 || res.NextCursor == "" { | |
| 88 break | |
| 89 } | |
| 90 req.StartCursor(res.NextCursor) | |
| 91 } | |
| 92 logging.Debugf(c, "Fetched %d %s builds in %s", len(fetched), status, cl
ock.Since(c, start)) | |
| 93 return fetched, nil | |
| 94 } | |
| 95 | |
| 96 // toMiloBuild converts a buildbucket build to a milo build. | |
| 97 // In case of an error, returns a build with a description of the error | |
| 98 // and logs the error. | |
| 99 func toMiloBuild(c context.Context, build *buildbucket.ApiCommonBuildMessage) *r
esp.BuildSummary { | |
| 100 // Parsing of parameters and result details is best effort. | |
| 101 var params buildParameters | |
| 102 if err := json.NewDecoder(strings.NewReader(build.ParametersJson)).Decod
e(¶ms); err != nil { | |
| 103 logging.Errorf(c, "Could not parse parameters of build %d: %s",
build.Id, err) | |
| 104 } | |
| 105 var resultDetails resultDetails | |
| 106 if err := json.NewDecoder(strings.NewReader(build.ResultDetailsJson)).De
code(&resultDetails); err != nil { | |
| 107 logging.Errorf(c, "Could not parse result details of build %d: %
s", build.Id, err) | |
| 108 } | |
| 109 | |
| 110 result := &resp.BuildSummary{ | |
| 111 Text: []string{fmt.Sprintf("buildbucket id %d", build.Id)}, | |
| 112 Revision: resultDetails.Properties.GotRevision, | |
| 113 } | |
| 114 if result.Revision == "" { | |
| 115 result.Revision = params.Properties.Revision | |
| 116 } | |
| 117 | |
| 118 var err error | |
| 119 result.Status, err = parseStatus(build) | |
| 120 if err != nil { | |
| 121 // almost never happens | |
| 122 logging.WithError(err).Errorf(c, "could not convert status of bu
ild %d", build.Id) | |
| 123 result.Status = model.InfraFailure | |
| 124 result.Text = append(result.Text, fmt.Sprintf("invalid build: %s
", err)) | |
| 125 } | |
| 126 | |
| 127 result.PendingTime.Started = parseTimestamp(build.CreatedTs) | |
| 128 switch build.Status { | |
| 129 case "SCHEDULED": | |
| 130 result.PendingTime.Duration = clock.Since(c, result.PendingTime.
Started) | |
| 131 | |
| 132 case "STARTED": | |
| 133 result.ExecutionTime.Started = parseTimestamp(build.StatusChange
dTs) | |
| 134 result.ExecutionTime.Duration = clock.Since(c, result.PendingTim
e.Started) | |
| 135 result.PendingTime.Finished = result.ExecutionTime.Started | |
| 136 result.PendingTime.Duration = result.PendingTime.Finished.Sub(re
sult.PendingTime.Started) | |
| 137 | |
| 138 case "COMPLETED": | |
| 139 // buildbucket does not provide build start time or execution du
ration. | |
| 140 result.ExecutionTime.Finished = parseTimestamp(build.CompletedTs
) | |
| 141 } | |
| 142 | |
| 143 cl := getChangeList(build, ¶ms, &resultDetails) | |
| 144 if cl != nil { | |
| 145 result.Blame = []*resp.Commit{cl} | |
| 146 } | |
| 147 | |
| 148 tags := ParseTags(build.Tags) | |
| 149 | |
| 150 if build.Url != "" { | |
| 151 u := build.Url | |
| 152 parsed, err := url.Parse(u) | |
| 153 | |
| 154 // map milo links to itself | |
| 155 switch { | |
| 156 case err != nil: | |
| 157 logging.Errorf(c, "invalid URL in build %d: %s", build.I
d, err) | |
| 158 case parsed.Host == info.DefaultVersionHostname(c): | |
| 159 parsed.Host = "" | |
| 160 parsed.Scheme = "" | |
| 161 u = parsed.String() | |
| 162 } | |
| 163 | |
| 164 result.Link = resp.NewLink(strconv.FormatInt(build.Id, 10), u) | |
| 165 | |
| 166 // compute the best link label | |
| 167 if taskID := tags["swarming_task_id"]; taskID != "" { | |
| 168 result.Link.Label = taskID | |
| 169 } else if resultDetails.Properties.BuildNumber != 0 { | |
| 170 result.Link.Label = strconv.Itoa(resultDetails.Propertie
s.BuildNumber) | |
| 171 } else if parsed != nil { | |
| 172 // does the URL look like a buildbot build URL? | |
| 173 pattern := fmt.Sprintf( | |
| 174 `/%s/builders/%s/builds/`, | |
| 175 strings.TrimPrefix(build.Bucket, "master."), par
ams.BuilderName) | |
| 176 beforeBuildNumber, buildNumberStr := path.Split(parsed.P
ath) | |
| 177 _, err := strconv.Atoi(buildNumberStr) | |
| 178 if strings.HasSuffix(beforeBuildNumber, pattern) && err
== nil { | |
| 179 result.Link.Label = buildNumberStr | |
| 180 } | |
| 181 } | |
| 182 } | |
| 183 | |
| 184 return result | |
| 185 } | |
| 186 | |
| 187 func getDebugBuilds(c context.Context, bucket, builder string, maxCompletedBuild
s int, target *resp.Builder) error { | |
| 188 // ../buildbucket below assumes that | |
| 189 // - this code is not executed by tests outside of this dir | |
| 190 // - this dir is a sibling of frontend dir | |
| 191 resFile, err := os.Open(filepath.Join( | |
| 192 "..", "buildbucket", "testdata", bucket, builder+".json")) | |
| 193 if err != nil { | |
| 194 return err | |
| 195 } | |
| 196 defer resFile.Close() | |
| 197 | |
| 198 res := &buildbucket.ApiSearchResponseMessage{} | |
| 199 if err := json.NewDecoder(resFile).Decode(res); err != nil { | |
| 200 return err | |
| 201 } | |
| 202 | |
| 203 for _, bb := range res.Builds { | |
| 204 mb := toMiloBuild(c, bb) | |
| 205 switch mb.Status { | |
| 206 case model.NotRun: | |
| 207 target.PendingBuilds = append(target.PendingBuilds, mb) | |
| 208 | |
| 209 case model.Running: | |
| 210 target.CurrentBuilds = append(target.CurrentBuilds, mb) | |
| 211 | |
| 212 case model.Success, model.Failure, model.InfraFailure, model.War
ning: | |
| 213 if len(target.FinishedBuilds) < maxCompletedBuilds { | |
| 214 target.FinishedBuilds = append(target.FinishedBu
ilds, mb) | |
| 215 } | |
| 216 | |
| 217 default: | |
| 218 panic("impossible") | |
| 219 } | |
| 220 } | |
| 221 return nil | |
| 222 } | |
| 223 | |
| 224 type builderQuery struct { | |
| 225 Bucket string | |
| 226 Builder string | |
| 227 Limit int | |
| 228 } | |
| 229 | |
| 230 // builderImpl is the implementation for getting a milo builder page from buildb
ucket. | |
| 231 // if maxCompletedBuilds < 0, 25 is used. | |
| 232 func builderImpl(c context.Context, q builderQuery) (*resp.Builder, error) { | |
| 233 settings := common.GetSettings(c) | |
| 234 if settings.Buildbucket == nil || settings.Buildbucket.Host == "" { | |
| 235 logging.Errorf(c, "missing buildbucket settings") | |
| 236 return nil, errors.New("missing buildbucket settings") | |
| 237 } | |
| 238 host := settings.Buildbucket.Host | |
| 239 | |
| 240 if q.Limit < 0 { | |
| 241 q.Limit = 20 | |
| 242 } | |
| 243 | |
| 244 result := &resp.Builder{ | |
| 245 Name: q.Builder, | |
| 246 } | |
| 247 if host == "debug" { | |
| 248 return result, getDebugBuilds(c, q.Bucket, q.Builder, q.Limit, r
esult) | |
| 249 } | |
| 250 client, err := newBuildbucketClient(c, host) | |
| 251 if err != nil { | |
| 252 return nil, err | |
| 253 } | |
| 254 | |
| 255 fetch := func(target *[]*resp.BuildSummary, status string, count int) er
ror { | |
| 256 builds, err := fetchBuilds(c, client, q.Bucket, q.Builder, statu
s, count) | |
| 257 if err != nil { | |
| 258 logging.Errorf(c, "Could not fetch builds with status %s
: %s", status, err) | |
| 259 return err | |
| 260 } | |
| 261 *target = make([]*resp.BuildSummary, len(builds)) | |
| 262 for i, bb := range builds { | |
| 263 (*target)[i] = toMiloBuild(c, bb) | |
| 264 } | |
| 265 return nil | |
| 266 } | |
| 267 // fetch pending, current and finished builds concurrently. | |
| 268 // Why not a single request? Because we need different build number | |
| 269 // limits for different statuses. | |
| 270 return result, parallel.FanOutIn(func(work chan<- func() error) { | |
| 271 work <- func() error { | |
| 272 return fetch(&result.PendingBuilds, StatusScheduled, -1) | |
| 273 } | |
| 274 work <- func() error { | |
| 275 return fetch(&result.CurrentBuilds, StatusStarted, -1) | |
| 276 } | |
| 277 work <- func() error { | |
| 278 return fetch(&result.FinishedBuilds, StatusCompleted, q.
Limit) | |
| 279 } | |
| 280 }) | |
| 281 } | |
| 282 | |
| 283 // parseTimestamp converts buildbucket timestamp in microseconds to time.Time | |
| 284 func parseTimestamp(microseconds int64) time.Time { | |
| 285 if microseconds == 0 { | |
| 286 return time.Time{} | |
| 287 } | |
| 288 return time.Unix(microseconds/1e6, microseconds%1e6*1000).UTC() | |
| 289 } | |
| 290 | |
| 291 type newBuildsFirst []*resp.BuildSummary | |
| 292 | |
| 293 func (a newBuildsFirst) Len() int { return len(a) } | |
| 294 func (a newBuildsFirst) Swap(i, j int) { a[i], a[j] = a[j], a[i] } | |
| 295 func (a newBuildsFirst) Less(i, j int) bool { | |
| 296 return a[i].PendingTime.Started.After(a[j].PendingTime.Started) | |
| 297 } | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 2974793002:
[milo] build_source -> buildSource. (Closed)
