| OLD | NEW |
|---|
| (Empty) |
| 1 // Copyright 2016 The LUCI Authors. All rights reserved. | |
| 2 // Use of this source code is governed under the Apache License, Version 2.0 | |
| 3 // that can be found in the LICENSE file. | |
| 4 | |
| 5 package buildbot | |
| 6 | |
| 7 import ( | |
| 8 "crypto/sha1" | |
| 9 "encoding/base64" | |
| 10 "errors" | |
| 11 "fmt" | |
| 12 "sort" | |
| 13 "strings" | |
| 14 "time" | |
| 15 | |
| 16 "github.com/luci/gae/service/datastore" | |
| 17 "github.com/luci/gae/service/memcache" | |
| 18 | |
| 19 "github.com/luci/luci-go/common/clock" | |
| 20 "github.com/luci/luci-go/common/logging" | |
| 21 "github.com/luci/luci-go/milo/api/resp" | |
| 22 "golang.org/x/net/context" | |
| 23 ) | |
| 24 | |
| 25 // builderRef is used for keying specific builds in a master json. | |
| 26 type builderRef struct { | |
| 27 builder string | |
| 28 buildNum int | |
| 29 } | |
| 30 | |
| 31 // buildMap contains all of the current build within a master json. We use this | |
| 32 // because buildbot returns all current builds as within the slaves portion, whe
reas | |
| 33 // it's eaiser to map thenm by builders instead. | |
| 34 type buildMap map[builderRef]*buildbotBuild | |
| 35 | |
| 36 // mergeText merges buildbot summary texts, which sometimes separates | |
| 37 // words that should be merged together, this combines them into a single | |
| 38 // line. | |
| 39 func mergeText(text []string) []string { | |
| 40 result := make([]string, 0, len(text)) | |
| 41 merge := false | |
| 42 for _, line := range text { | |
| 43 if merge { | |
| 44 merge = false | |
| 45 result[len(result)-1] += " " + line | |
| 46 continue | |
| 47 } | |
| 48 result = append(result, line) | |
| 49 switch line { | |
| 50 case "build", "failed", "exception": | |
| 51 merge = true | |
| 52 default: | |
| 53 merge = false | |
| 54 } | |
| 55 } | |
| 56 | |
| 57 // We can remove error messages about the step "steps" if it's part of a
longer | |
| 58 // message because this step is an artifact of running on recipes and it
's | |
| 59 // not important to users. | |
| 60 if len(result) > 1 { | |
| 61 switch result[0] { | |
| 62 case "failed steps", "exception steps": | |
| 63 result = result[1:] | |
| 64 } | |
| 65 } | |
| 66 return result | |
| 67 } | |
| 68 | |
| 69 func getBuildSummary(b *buildbotBuild) *resp.BuildSummary { | |
| 70 started, finished, duration := parseTimes(nil, b.Times) | |
| 71 return &resp.BuildSummary{ | |
| 72 Link: resp.NewLink(fmt.Sprintf("#%d", b.Number), fmt.Sprintf("
%d", b.Number)), | |
| 73 Status: b.toStatus(), | |
| 74 ExecutionTime: resp.Interval{ | |
| 75 Started: started, | |
| 76 Finished: finished, | |
| 77 Duration: duration, | |
| 78 }, | |
| 79 Text: mergeText(b.Text), | |
| 80 Blame: blame(b), | |
| 81 Revision: b.Sourcestamp.Revision, | |
| 82 } | |
| 83 } | |
| 84 | |
| 85 // getBuilds fetches all of the recent builds from the . Note that | |
| 86 // getBuilds() does not perform ACL checks. | |
| 87 func getBuilds( | |
| 88 c context.Context, masterName, builderName string, finished bool, limit
int, cursor *datastore.Cursor) ( | |
| 89 []*resp.BuildSummary, *datastore.Cursor, error) { | |
| 90 | |
| 91 // TODO(hinoka): Builder specific structs. | |
| 92 result := []*resp.BuildSummary{} | |
| 93 q := datastore.NewQuery("buildbotBuild") | |
| 94 q = q.Eq("finished", finished) | |
| 95 q = q.Eq("master", masterName) | |
| 96 q = q.Eq("builder", builderName) | |
| 97 q = q.Order("-number") | |
| 98 if cursor != nil { | |
| 99 q = q.Start(*cursor) | |
| 100 } | |
| 101 buildbots, nextCursor, err := runBuildsQuery(c, q, int32(limit)) | |
| 102 if err != nil { | |
| 103 return nil, nil, err | |
| 104 } | |
| 105 for _, b := range buildbots { | |
| 106 result = append(result, getBuildSummary(b)) | |
| 107 } | |
| 108 return result, nextCursor, nil | |
| 109 } | |
| 110 | |
| 111 // maybeSetGetCursor is a cheesy way to implement bidirectional paging with forw
ard-only | |
| 112 // datastore cursor by creating a mapping of nextCursor -> thisCursor | |
| 113 // in memcache. maybeSetGetCursor stores the future mapping, then returns prevC
ursor | |
| 114 // in the mapping for thisCursor -> prevCursor, if available. | |
| 115 func maybeSetGetCursor(c context.Context, thisCursor, nextCursor *datastore.Curs
or, limit int) (*datastore.Cursor, bool) { | |
| 116 key := func(c datastore.Cursor) string { | |
| 117 // Memcache key limit is 250 bytes, hash our cursor to get under
this limit. | |
| 118 blob := sha1.Sum([]byte(c.String())) | |
| 119 return fmt.Sprintf("v2:cursors:buildbot_builders:%d:%s", limit,
base64.StdEncoding.EncodeToString(blob[:])) | |
| 120 } | |
| 121 // Set the next cursor to this cursor mapping, if available. | |
| 122 if nextCursor != nil { | |
| 123 item := memcache.NewItem(c, key(*nextCursor)) | |
| 124 if thisCursor == nil { | |
| 125 // Make sure we know it exists, just empty | |
| 126 item.SetValue([]byte{}) | |
| 127 } else { | |
| 128 item.SetValue([]byte((*thisCursor).String())) | |
| 129 } | |
| 130 item.SetExpiration(24 * time.Hour) | |
| 131 memcache.Set(c, item) | |
| 132 } | |
| 133 // Try to get the last cursor, if valid and available. | |
| 134 if thisCursor == nil { | |
| 135 return nil, false | |
| 136 } | |
| 137 if item, err := memcache.GetKey(c, key(*thisCursor)); err == nil { | |
| 138 if len(item.Value()) == 0 { | |
| 139 return nil, true | |
| 140 } | |
| 141 if prevCursor, err := datastore.DecodeCursor(c, string(item.Valu
e())); err == nil { | |
| 142 return &prevCursor, true | |
| 143 } | |
| 144 } | |
| 145 return nil, false | |
| 146 } | |
| 147 | |
| 148 var errMasterNotFound = errors.New( | |
| 149 "Either the request resource was not found or you have insufficient perm
issions") | |
| 150 var errNotAuth = errors.New("You are not authenticated, try logging in") | |
| 151 | |
| 152 type errBuilderNotFound struct { | |
| 153 master string | |
| 154 builder string | |
| 155 available []string | |
| 156 } | |
| 157 | |
| 158 func (e errBuilderNotFound) Error() string { | |
| 159 avail := strings.Join(e.available, "\n") | |
| 160 return fmt.Sprintf("Cannot find builder %q in master %q.\nAvailable buil
ders: \n%s", | |
| 161 e.builder, e.master, avail) | |
| 162 } | |
| 163 | |
| 164 func summarizeSlavePool( | |
| 165 baseURL string, slaves []string, slaveMap map[string]*buildbotSlave) *re
sp.MachinePool { | |
| 166 | |
| 167 mp := &resp.MachinePool{ | |
| 168 Total: len(slaves), | |
| 169 Bots: make([]resp.Bot, 0, len(slaves)), | |
| 170 } | |
| 171 for _, slaveName := range slaves { | |
| 172 slave, ok := slaveMap[slaveName] | |
| 173 bot := resp.Bot{ | |
| 174 Name: *resp.NewLink( | |
| 175 slaveName, | |
| 176 fmt.Sprintf("%s/buildslaves/%s", baseURL, slaveN
ame), | |
| 177 ), | |
| 178 } | |
| 179 switch { | |
| 180 case !ok: | |
| 181 // This shouldn't happen | |
| 182 case !slave.Connected: | |
| 183 bot.Status = resp.Disconnected | |
| 184 mp.Disconnected++ | |
| 185 case len(slave.RunningbuildsMap) > 0: | |
| 186 bot.Status = resp.Busy | |
| 187 mp.Busy++ | |
| 188 default: | |
| 189 bot.Status = resp.Idle | |
| 190 mp.Idle++ | |
| 191 } | |
| 192 mp.Bots = append(mp.Bots, bot) | |
| 193 } | |
| 194 return mp | |
| 195 } | |
| 196 | |
| 197 // builderImpl is the implementation for getting a milo builder page from buildb
ot. | |
| 198 // This gets: | |
| 199 // * Current Builds from querying the master json from the datastore. | |
| 200 // * Recent Builds from a cron job that backfills the recent builds. | |
| 201 func builderImpl( | |
| 202 c context.Context, masterName, builderName string, limit int, cursor str
ing) ( | |
| 203 *resp.Builder, error) { | |
| 204 | |
| 205 var thisCursor *datastore.Cursor | |
| 206 if cursor != "" { | |
| 207 tmpCur, err := datastore.DecodeCursor(c, cursor) | |
| 208 if err != nil { | |
| 209 return nil, fmt.Errorf("bad cursor: %s", err) | |
| 210 } | |
| 211 thisCursor = &tmpCur | |
| 212 } | |
| 213 | |
| 214 result := &resp.Builder{ | |
| 215 Name: builderName, | |
| 216 } | |
| 217 master, internal, t, err := getMasterJSON(c, masterName) | |
| 218 if err != nil { | |
| 219 return nil, err | |
| 220 } | |
| 221 if clock.Now(c).Sub(t) > 2*time.Minute { | |
| 222 warning := fmt.Sprintf( | |
| 223 "WARNING: Master data is stale (last updated %s)", t) | |
| 224 logging.Warningf(c, warning) | |
| 225 result.Warning = warning | |
| 226 } | |
| 227 | |
| 228 p, ok := master.Builders[builderName] | |
| 229 if !ok { | |
| 230 // This long block is just to return a good error message when a
n invalid | |
| 231 // buildbot builder is specified. | |
| 232 keys := make([]string, 0, len(master.Builders)) | |
| 233 for k := range master.Builders { | |
| 234 keys = append(keys, k) | |
| 235 } | |
| 236 sort.Strings(keys) | |
| 237 return nil, errBuilderNotFound{masterName, builderName, keys} | |
| 238 } | |
| 239 // Extract pending builds out of the master json. | |
| 240 result.PendingBuilds = make([]*resp.BuildSummary, len(p.PendingBuildStat
es)) | |
| 241 logging.Debugf(c, "Number of pending builds: %d", len(p.PendingBuildStat
es)) | |
| 242 for i, pb := range p.PendingBuildStates { | |
| 243 start := time.Unix(int64(pb.SubmittedAt), 0).UTC() | |
| 244 result.PendingBuilds[i] = &resp.BuildSummary{ | |
| 245 PendingTime: resp.Interval{ | |
| 246 Started: start, | |
| 247 Duration: clock.Now(c).UTC().Sub(start), | |
| 248 }, | |
| 249 } | |
| 250 result.PendingBuilds[i].Blame = make([]*resp.Commit, len(pb.Sour
ce.Changes)) | |
| 251 for j, cm := range pb.Source.Changes { | |
| 252 result.PendingBuilds[i].Blame[j] = &resp.Commit{ | |
| 253 AuthorEmail: cm.Who, | |
| 254 CommitURL: cm.Revlink, | |
| 255 } | |
| 256 } | |
| 257 } | |
| 258 | |
| 259 baseURL := "https://build.chromium.org/p/" | |
| 260 if internal { | |
| 261 baseURL = "https://uberchromegw.corp.google.com/i/" | |
| 262 } | |
| 263 result.MachinePool = summarizeSlavePool(baseURL+master.Name, p.Slaves, m
aster.Slaves) | |
| 264 | |
| 265 // This is CPU bound anyways, so there's no need to do this in parallel. | |
| 266 finishedBuilds, nextCursor, err := getBuilds(c, masterName, builderName,
true, limit, thisCursor) | |
| 267 if err != nil { | |
| 268 return nil, err | |
| 269 } | |
| 270 if prevCursor, ok := maybeSetGetCursor(c, thisCursor, nextCursor, limit)
; ok { | |
| 271 if prevCursor == nil { | |
| 272 // Magic string to signal display prev without cursor | |
| 273 result.PrevCursor = "EMPTY" | |
| 274 } else { | |
| 275 result.PrevCursor = (*prevCursor).String() | |
| 276 } | |
| 277 } | |
| 278 if nextCursor != nil { | |
| 279 result.NextCursor = (*nextCursor).String() | |
| 280 } | |
| 281 // Cursor is not needed for current builds. | |
| 282 currentBuilds, _, err := getBuilds(c, masterName, builderName, false, 0,
nil) | |
| 283 if err != nil { | |
| 284 return nil, err | |
| 285 } | |
| 286 // currentBuilds is presented in reversed order, so flip it | |
| 287 for i, j := 0, len(currentBuilds)-1; i < j; i, j = i+1, j-1 { | |
| 288 currentBuilds[i], currentBuilds[j] = currentBuilds[j], currentBu
ilds[i] | |
| 289 } | |
| 290 result.CurrentBuilds = currentBuilds | |
| 291 | |
| 292 for _, fb := range finishedBuilds { | |
| 293 if fb != nil { | |
| 294 result.FinishedBuilds = append(result.FinishedBuilds, fb
) | |
| 295 } | |
| 296 } | |
| 297 return result, nil | |
| 298 } | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 2944983003:
[milo] {buildbucket,buildbot,swarming,logdog} -> backends/*. (Closed)
