| OLD | NEW |
|---|
| (Empty) |
| 1 // Copyright 2016 The LUCI Authors. All rights reserved. | |
| 2 // Use of this source code is governed under the Apache License, Version 2.0 | |
| 3 // that can be found in the LICENSE file. | |
| 4 | |
| 5 package buildbucket | |
| 6 | |
| 7 import ( | |
| 8 "encoding/json" | |
| 9 "fmt" | |
| 10 "net/url" | |
| 11 "os" | |
| 12 "path" | |
| 13 "path/filepath" | |
| 14 "strconv" | |
| 15 "strings" | |
| 16 "time" | |
| 17 | |
| 18 "golang.org/x/net/context" | |
| 19 "google.golang.org/api/googleapi" | |
| 20 | |
| 21 "github.com/luci/gae/service/info" | |
| 22 "github.com/luci/luci-go/common/api/buildbucket/buildbucket/v1" | |
| 23 "github.com/luci/luci-go/common/clock" | |
| 24 "github.com/luci/luci-go/common/errors" | |
| 25 "github.com/luci/luci-go/common/logging" | |
| 26 "github.com/luci/luci-go/common/retry" | |
| 27 "github.com/luci/luci-go/common/sync/parallel" | |
| 28 | |
| 29 "github.com/luci/luci-go/milo/api/resp" | |
| 30 "github.com/luci/luci-go/milo/appengine/common" | |
| 31 "github.com/luci/luci-go/milo/appengine/common/model" | |
| 32 ) | |
| 33 | |
| 34 // search executes the search request with retries and exponential back-off. | |
| 35 func search(c context.Context, client *buildbucket.Service, req *buildbucket.Sea
rchCall) ( | |
| 36 *buildbucket.ApiSearchResponseMessage, error) { | |
| 37 | |
| 38 var res *buildbucket.ApiSearchResponseMessage | |
| 39 err := retry.Retry( | |
| 40 c, | |
| 41 retry.TransientOnly(retry.Default), | |
| 42 func() error { | |
| 43 var err error | |
| 44 res, err = req.Do() | |
| 45 if apiErr, ok := err.(*googleapi.Error); ok && apiErr.Co
de >= 500 { | |
| 46 err = errors.WrapTransient(apiErr) | |
| 47 } | |
| 48 return err | |
| 49 }, | |
| 50 func(err error, wait time.Duration) { | |
| 51 logging.WithError(err).Warningf(c, "buildbucket search r
equest failed transiently, will retry in %s", wait) | |
| 52 }) | |
| 53 return res, err | |
| 54 } | |
| 55 | |
| 56 // fetchBuilds fetches builds given a criteria. | |
| 57 // The returned builds are sorted by build creation descending. | |
| 58 // count defines maximum number of builds to fetch; if <0, defaults to 100. | |
| 59 func fetchBuilds(c context.Context, client *buildbucket.Service, bucket, builder
, | |
| 60 status string, count int) ([]*buildbucket.ApiCommonBuildMessage, error)
{ | |
| 61 | |
| 62 req := client.Search() | |
| 63 req.Bucket(bucket) | |
| 64 req.Status(status) | |
| 65 req.Tag("builder:" + builder) | |
| 66 | |
| 67 if count < 0 { | |
| 68 count = 100 | |
| 69 } | |
| 70 | |
| 71 fetched := make([]*buildbucket.ApiCommonBuildMessage, 0, count) | |
| 72 start := clock.Now(c) | |
| 73 for len(fetched) < count { | |
| 74 req.MaxBuilds(int64(count - len(fetched))) | |
| 75 | |
| 76 res, err := search(c, client, req) | |
| 77 switch { | |
| 78 case err != nil: | |
| 79 return fetched, err | |
| 80 case res.Error != nil: | |
| 81 return fetched, fmt.Errorf(res.Error.Message) | |
| 82 } | |
| 83 | |
| 84 fetched = append(fetched, res.Builds...) | |
| 85 | |
| 86 if len(res.Builds) == 0 || res.NextCursor == "" { | |
| 87 break | |
| 88 } | |
| 89 req.StartCursor(res.NextCursor) | |
| 90 } | |
| 91 logging.Debugf(c, "Fetched %d %s builds in %s", len(fetched), status, cl
ock.Since(c, start)) | |
| 92 return fetched, nil | |
| 93 } | |
| 94 | |
| 95 // toMiloBuild converts a buildbucket build to a milo build. | |
| 96 // In case of an error, returns a build with a description of the error | |
| 97 // and logs the error. | |
| 98 func toMiloBuild(c context.Context, build *buildbucket.ApiCommonBuildMessage) *r
esp.BuildSummary { | |
| 99 // Parsing of parameters and result details is best effort. | |
| 100 var params buildParameters | |
| 101 if err := json.NewDecoder(strings.NewReader(build.ParametersJson)).Decod
e(¶ms); err != nil { | |
| 102 logging.Errorf(c, "Could not parse parameters of build %d: %s",
build.Id, err) | |
| 103 } | |
| 104 var resultDetails resultDetails | |
| 105 if err := json.NewDecoder(strings.NewReader(build.ResultDetailsJson)).De
code(&resultDetails); err != nil { | |
| 106 logging.Errorf(c, "Could not parse result details of build %d: %
s", build.Id, err) | |
| 107 } | |
| 108 | |
| 109 result := &resp.BuildSummary{ | |
| 110 Text: []string{fmt.Sprintf("buildbucket id %d", build.Id)}, | |
| 111 Revision: resultDetails.Properties.GotRevision, | |
| 112 } | |
| 113 if result.Revision == "" { | |
| 114 result.Revision = params.Properties.Revision | |
| 115 } | |
| 116 | |
| 117 var err error | |
| 118 result.Status, err = parseStatus(build) | |
| 119 if err != nil { | |
| 120 // almost never happens | |
| 121 logging.WithError(err).Errorf(c, "could not convert status of bu
ild %d", build.Id) | |
| 122 result.Status = model.InfraFailure | |
| 123 result.Text = append(result.Text, fmt.Sprintf("invalid build: %s
", err)) | |
| 124 } | |
| 125 | |
| 126 result.PendingTime.Started = parseTimestamp(build.CreatedTs) | |
| 127 switch build.Status { | |
| 128 case "SCHEDULED": | |
| 129 result.PendingTime.Duration = clock.Since(c, result.PendingTime.
Started) | |
| 130 | |
| 131 case "STARTED": | |
| 132 result.ExecutionTime.Started = parseTimestamp(build.StatusChange
dTs) | |
| 133 result.ExecutionTime.Duration = clock.Since(c, result.PendingTim
e.Started) | |
| 134 result.PendingTime.Finished = result.ExecutionTime.Started | |
| 135 result.PendingTime.Duration = result.PendingTime.Finished.Sub(re
sult.PendingTime.Started) | |
| 136 | |
| 137 case "COMPLETED": | |
| 138 // buildbucket does not provide build start time or execution du
ration. | |
| 139 result.ExecutionTime.Finished = parseTimestamp(build.CompletedTs
) | |
| 140 } | |
| 141 | |
| 142 cl := getChangeList(build, ¶ms, &resultDetails) | |
| 143 if cl != nil { | |
| 144 result.Blame = []*resp.Commit{cl} | |
| 145 } | |
| 146 | |
| 147 tags := ParseTags(build.Tags) | |
| 148 | |
| 149 if build.Url != "" { | |
| 150 u := build.Url | |
| 151 parsed, err := url.Parse(u) | |
| 152 | |
| 153 // map milo links to itself | |
| 154 switch { | |
| 155 case err != nil: | |
| 156 logging.Errorf(c, "invalid URL in build %d: %s", build.I
d, err) | |
| 157 case parsed.Host == info.DefaultVersionHostname(c): | |
| 158 parsed.Host = "" | |
| 159 parsed.Scheme = "" | |
| 160 u = parsed.String() | |
| 161 } | |
| 162 | |
| 163 result.Link = resp.NewLink(strconv.FormatInt(build.Id, 10), u) | |
| 164 | |
| 165 // compute the best link label | |
| 166 if taskID := tags["swarming_task_id"]; taskID != "" { | |
| 167 result.Link.Label = taskID | |
| 168 } else if resultDetails.Properties.BuildNumber != 0 { | |
| 169 result.Link.Label = strconv.Itoa(resultDetails.Propertie
s.BuildNumber) | |
| 170 } else if parsed != nil { | |
| 171 // does the URL look like a buildbot build URL? | |
| 172 pattern := fmt.Sprintf( | |
| 173 `/%s/builders/%s/builds/`, | |
| 174 strings.TrimPrefix(build.Bucket, "master."), par
ams.BuilderName) | |
| 175 beforeBuildNumber, buildNumberStr := path.Split(parsed.P
ath) | |
| 176 _, err := strconv.Atoi(buildNumberStr) | |
| 177 if strings.HasSuffix(beforeBuildNumber, pattern) && err
== nil { | |
| 178 result.Link.Label = buildNumberStr | |
| 179 } | |
| 180 } | |
| 181 } | |
| 182 | |
| 183 return result | |
| 184 } | |
| 185 | |
| 186 func getDebugBuilds(c context.Context, bucket, builder string, maxCompletedBuild
s int, target *resp.Builder) error { | |
| 187 // ../buildbucket below assumes that | |
| 188 // - this code is not executed by tests outside of this dir | |
| 189 // - this dir is a sibling of frontend dir | |
| 190 resFile, err := os.Open(filepath.Join( | |
| 191 "..", "buildbucket", "testdata", bucket, builder+".json")) | |
| 192 if err != nil { | |
| 193 return err | |
| 194 } | |
| 195 defer resFile.Close() | |
| 196 | |
| 197 res := &buildbucket.ApiSearchResponseMessage{} | |
| 198 if err := json.NewDecoder(resFile).Decode(res); err != nil { | |
| 199 return err | |
| 200 } | |
| 201 | |
| 202 for _, bb := range res.Builds { | |
| 203 mb := toMiloBuild(c, bb) | |
| 204 switch mb.Status { | |
| 205 case model.NotRun: | |
| 206 target.PendingBuilds = append(target.PendingBuilds, mb) | |
| 207 | |
| 208 case model.Running: | |
| 209 target.CurrentBuilds = append(target.CurrentBuilds, mb) | |
| 210 | |
| 211 case model.Success, model.Failure, model.InfraFailure, model.War
ning: | |
| 212 if len(target.FinishedBuilds) < maxCompletedBuilds { | |
| 213 target.FinishedBuilds = append(target.FinishedBu
ilds, mb) | |
| 214 } | |
| 215 | |
| 216 default: | |
| 217 panic("impossible") | |
| 218 } | |
| 219 } | |
| 220 return nil | |
| 221 } | |
| 222 | |
| 223 type builderQuery struct { | |
| 224 Bucket string | |
| 225 Builder string | |
| 226 Limit int | |
| 227 } | |
| 228 | |
| 229 // builderImpl is the implementation for getting a milo builder page from buildb
ucket. | |
| 230 // if maxCompletedBuilds < 0, 25 is used. | |
| 231 func builderImpl(c context.Context, q builderQuery) (*resp.Builder, error) { | |
| 232 settings := common.GetSettings(c) | |
| 233 if settings.Buildbucket == nil || settings.Buildbucket.Host == "" { | |
| 234 logging.Errorf(c, "missing buildbucket settings") | |
| 235 return nil, errors.New("missing buildbucket settings") | |
| 236 } | |
| 237 host := settings.Buildbucket.Host | |
| 238 | |
| 239 if q.Limit < 0 { | |
| 240 q.Limit = 20 | |
| 241 } | |
| 242 | |
| 243 result := &resp.Builder{ | |
| 244 Name: q.Builder, | |
| 245 } | |
| 246 if host == "debug" { | |
| 247 return result, getDebugBuilds(c, q.Bucket, q.Builder, q.Limit, r
esult) | |
| 248 } | |
| 249 client, err := newBuildbucketClient(c, host) | |
| 250 if err != nil { | |
| 251 return nil, err | |
| 252 } | |
| 253 | |
| 254 fetch := func(target *[]*resp.BuildSummary, status string, count int) er
ror { | |
| 255 builds, err := fetchBuilds(c, client, q.Bucket, q.Builder, statu
s, count) | |
| 256 if err != nil { | |
| 257 logging.Errorf(c, "Could not fetch builds with status %s
: %s", status, err) | |
| 258 return err | |
| 259 } | |
| 260 *target = make([]*resp.BuildSummary, len(builds)) | |
| 261 for i, bb := range builds { | |
| 262 (*target)[i] = toMiloBuild(c, bb) | |
| 263 } | |
| 264 return nil | |
| 265 } | |
| 266 // fetch pending, current and finished builds concurrently. | |
| 267 // Why not a single request? Because we need different build number | |
| 268 // limits for different statuses. | |
| 269 return result, parallel.FanOutIn(func(work chan<- func() error) { | |
| 270 work <- func() error { | |
| 271 return fetch(&result.PendingBuilds, StatusScheduled, -1) | |
| 272 } | |
| 273 work <- func() error { | |
| 274 return fetch(&result.CurrentBuilds, StatusStarted, -1) | |
| 275 } | |
| 276 work <- func() error { | |
| 277 return fetch(&result.FinishedBuilds, StatusCompleted, q.
Limit) | |
| 278 } | |
| 279 }) | |
| 280 } | |
| 281 | |
| 282 // parseTimestamp converts buildbucket timestamp in microseconds to time.Time | |
| 283 func parseTimestamp(microseconds int64) time.Time { | |
| 284 if microseconds == 0 { | |
| 285 return time.Time{} | |
| 286 } | |
| 287 return time.Unix(microseconds/1e6, microseconds%1e6*1000).UTC() | |
| 288 } | |
| 289 | |
| 290 type newBuildsFirst []*resp.BuildSummary | |
| 291 | |
| 292 func (a newBuildsFirst) Len() int { return len(a) } | |
| 293 func (a newBuildsFirst) Swap(i, j int) { a[i], a[j] = a[j], a[i] } | |
| 294 func (a newBuildsFirst) Less(i, j int) bool { | |
| 295 return a[i].PendingTime.Started.After(a[j].PendingTime.Started) | |
| 296 } | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 2944983003:
[milo] {buildbucket,buildbot,swarming,logdog} -> backends/*. (Closed)
