| OLD | NEW |
|---|
| (Empty) |
| 1 // Copyright 2016 The LUCI Authors. All rights reserved. | |
| 2 // Use of this source code is governed under the Apache License, Version 2.0 | |
| 3 // that can be found in the LICENSE file. | |
| 4 | |
| 5 package buildbot | |
| 6 | |
| 7 import ( | |
| 8 "bytes" | |
| 9 "compress/gzip" | |
| 10 "encoding/json" | |
| 11 "fmt" | |
| 12 "io/ioutil" | |
| 13 "strconv" | |
| 14 | |
| 15 "github.com/luci/gae/service/datastore" | |
| 16 "github.com/luci/luci-go/milo/api/resp" | |
| 17 "github.com/luci/luci-go/milo/appengine/common/model" | |
| 18 ) | |
| 19 | |
| 20 // This file contains all of the structs that define buildbot json endpoints. | |
| 21 // This is primarily used for unmarshalling buildbot master and build json. | |
| 22 // json.UnmarshalJSON can directly unmarshal buildbot jsons into these structs. | |
| 23 // Many of the structs were initially built using https://mholt.github.io/json-t
o-go/ | |
| 24 | |
| 25 // buildbotStep represents a single step in a buildbot build. | |
| 26 type buildbotStep struct { | |
| 27 // We actually don't care about ETA. This is represented as a string if | |
| 28 // it's fetched from a build json, but a float if it's dug out of the | |
| 29 // slave portion of a master json. We'll just set it to interface and | |
| 30 // ignore it. | |
| 31 Eta interface{} `json:"eta"` | |
| 32 Expectations [][]interface{} `json:"expectations"` | |
| 33 Hidden bool `json:"hidden"` | |
| 34 IsFinished bool `json:"isFinished"` | |
| 35 IsStarted bool `json:"isStarted"` | |
| 36 Logs [][]string `json:"logs"` | |
| 37 Name string `json:"name"` | |
| 38 Results []interface{} `json:"results"` | |
| 39 Statistics struct { | |
| 40 } `json:"statistics"` | |
| 41 StepNumber int `json:"step_number"` | |
| 42 Text []string `json:"text"` | |
| 43 Times []*float64 `json:"times"` | |
| 44 Urls map[string]string `json:"urls"` | |
| 45 | |
| 46 // Log link aliases. The key is a log name that is being aliases. It sh
ould, | |
| 47 // generally, exist within the Logs. The value is the set of aliases att
ached | |
| 48 // to that key. | |
| 49 Aliases map[string][]*buildbotLinkAlias `json:"aliases"` | |
| 50 } | |
| 51 | |
| 52 // buildbotSourceStamp is a list of changes (commits) tagged with where the chan
ges | |
| 53 // came from, ie. the project/repository. Also includes a "main" revision." | |
| 54 type buildbotSourceStamp struct { | |
| 55 Branch *string `json:"branch"` | |
| 56 Changes []buildbotChange `json:"changes"` | |
| 57 Haspatch bool `json:"hasPatch"` | |
| 58 Project string `json:"project"` | |
| 59 Repository string `json:"repository"` | |
| 60 Revision string `json:"revision"` | |
| 61 } | |
| 62 | |
| 63 type buildbotLinkAlias struct { | |
| 64 URL string `json:"url"` | |
| 65 Text string `json:"text"` | |
| 66 } | |
| 67 | |
| 68 func (a *buildbotLinkAlias) toLink() *resp.Link { | |
| 69 return resp.NewLink(a.Text, a.URL) | |
| 70 } | |
| 71 | |
| 72 type buildbotProperty struct { | |
| 73 Name string | |
| 74 Value interface{} | |
| 75 Source string | |
| 76 } | |
| 77 | |
| 78 func (p *buildbotProperty) MarshalJSON() ([]byte, error) { | |
| 79 return json.Marshal([]interface{}{p.Name, p.Value, p.Source}) | |
| 80 } | |
| 81 | |
| 82 func (p *buildbotProperty) UnmarshalJSON(d []byte) error { | |
| 83 // The raw BuildBot representation is a slice of interfaces. | |
| 84 var raw []interface{} | |
| 85 if err := json.Unmarshal(d, &raw); err != nil { | |
| 86 return err | |
| 87 } | |
| 88 | |
| 89 switch len(raw) { | |
| 90 case 3: | |
| 91 if s, ok := raw[2].(string); ok { | |
| 92 p.Source = s | |
| 93 } | |
| 94 fallthrough | |
| 95 | |
| 96 case 2: | |
| 97 p.Value = raw[1] | |
| 98 fallthrough | |
| 99 | |
| 100 case 1: | |
| 101 if s, ok := raw[0].(string); ok { | |
| 102 p.Name = s | |
| 103 } | |
| 104 } | |
| 105 return nil | |
| 106 } | |
| 107 | |
| 108 // buildbotBuild is a single build json on buildbot. | |
| 109 type buildbotBuild struct { | |
| 110 Master string `gae:"$master"` | |
| 111 Blame []string `json:"blame" gae:"-"` | |
| 112 Buildername string `json:"builderName"` | |
| 113 // This needs to be reflected. This can be either a String or a buildbo
tStep. | |
| 114 Currentstep interface{} `json:"currentStep" gae:"-"` | |
| 115 // We don't care about this one. | |
| 116 Eta interface{} `json:"eta" gae:"-"` | |
| 117 Logs [][]string `json:"logs" gae:"-"` | |
| 118 Number int `json:"number"` | |
| 119 // This is a slice of tri-tuples of [property name, value, source]. | |
| 120 // property name is always a string | |
| 121 // value can be a string or float | |
| 122 // source is optional, but is always a string if present | |
| 123 Properties []*buildbotProperty `json:"properties" gae:"-"` | |
| 124 Reason string `json:"reason"` | |
| 125 Results *int `json:"results" gae:"-"` | |
| 126 Slave string `json:"slave"` | |
| 127 Sourcestamp *buildbotSourceStamp `json:"sourceStamp" gae:"-"` | |
| 128 Steps []buildbotStep `json:"steps" gae:"-"` | |
| 129 Text []string `json:"text" gae:"-"` | |
| 130 Times []*float64 `json:"times" gae:"-"` | |
| 131 // This one is injected by Milo. Does not exist in a normal json query. | |
| 132 TimeStamp *int `json:"timeStamp" gae:"-"` | |
| 133 // This one is marked by Milo, denotes whether or not the build is inter
nal. | |
| 134 Internal bool `json:"internal" gae:"-"` | |
| 135 // This one is computed by Milo for indexing purposes. It does so by | |
| 136 // checking to see if times[1] is null or not. | |
| 137 Finished bool `json:"finished"` | |
| 138 // OS is a string representation of the OS the build ran on. This is | |
| 139 // derived best-effort from the slave information in the master JSON. | |
| 140 // This information is injected into the buildbot builds via puppet, and | |
| 141 // comes as Family + Version. Family is (windows, Darwin, Debian), whil
e | |
| 142 // Version is the version of the OS, such as (XP, 7, 10) for windows. | |
| 143 OSFamily string `json:"osFamily"` | |
| 144 OSVersion string `json:"osVersion"` | |
| 145 } | |
| 146 | |
| 147 func (b *buildbotBuild) toStatus() model.Status { | |
| 148 var result model.Status | |
| 149 if b.Currentstep != nil { | |
| 150 result = model.Running | |
| 151 } else { | |
| 152 result = result2Status(b.Results) | |
| 153 } | |
| 154 return result | |
| 155 } | |
| 156 | |
| 157 var _ datastore.PropertyLoadSaver = (*buildbotBuild)(nil) | |
| 158 var _ datastore.MetaGetterSetter = (*buildbotBuild)(nil) | |
| 159 | |
| 160 // getID is a helper function that returns the datastore key for a given | |
| 161 // build. | |
| 162 func (b *buildbotBuild) getID() string { | |
| 163 s := []string{b.Master, b.Buildername, strconv.Itoa(b.Number)} | |
| 164 id, err := json.Marshal(s) | |
| 165 if err != nil { | |
| 166 panic(err) // This really shouldn't fail. | |
| 167 } | |
| 168 return string(id) | |
| 169 } | |
| 170 | |
| 171 // setKeys is the inverse of getID(). | |
| 172 func (b *buildbotBuild) setKeys(id string) error { | |
| 173 s := []string{} | |
| 174 err := json.Unmarshal([]byte(id), &s) | |
| 175 if err != nil { | |
| 176 return err | |
| 177 } | |
| 178 if len(s) != 3 { | |
| 179 return fmt.Errorf("%s does not have 3 items", id) | |
| 180 } | |
| 181 b.Master = s[0] | |
| 182 b.Buildername = s[1] | |
| 183 b.Number, err = strconv.Atoi(s[2]) | |
| 184 return err // or nil. | |
| 185 } | |
| 186 | |
| 187 // GetMeta is overridden so that a query for "id" calls getID() instead of | |
| 188 // the superclass method. | |
| 189 func (b *buildbotBuild) GetMeta(key string) (interface{}, bool) { | |
| 190 if key == "id" { | |
| 191 if b.Master == "" || b.Buildername == "" { | |
| 192 panic(fmt.Errorf("No Master or Builder found")) | |
| 193 } | |
| 194 return b.getID(), true | |
| 195 } | |
| 196 return datastore.GetPLS(b).GetMeta(key) | |
| 197 } | |
| 198 | |
| 199 // GetAllMeta is overridden for the same reason GetMeta() is. | |
| 200 func (b *buildbotBuild) GetAllMeta() datastore.PropertyMap { | |
| 201 p := datastore.GetPLS(b).GetAllMeta() | |
| 202 p.SetMeta("id", b.getID()) | |
| 203 return p | |
| 204 } | |
| 205 | |
| 206 // SetMeta is the inverse of GetMeta(). | |
| 207 func (b *buildbotBuild) SetMeta(key string, val interface{}) bool { | |
| 208 if key == "id" { | |
| 209 err := b.setKeys(val.(string)) | |
| 210 if err != nil { | |
| 211 panic(err) | |
| 212 } | |
| 213 } | |
| 214 return datastore.GetPLS(b).SetMeta(key, val) | |
| 215 } | |
| 216 | |
| 217 // Load translates a propertymap into the struct and loads the data into | |
| 218 // the struct. | |
| 219 func (b *buildbotBuild) Load(p datastore.PropertyMap) error { | |
| 220 if _, ok := p["data"]; !ok { | |
| 221 // This is probably from a keys-only query. No need to load the
rest. | |
| 222 return datastore.GetPLS(b).Load(p) | |
| 223 } | |
| 224 gz, err := p.Slice("data")[0].Project(datastore.PTBytes) | |
| 225 if err != nil { | |
| 226 return err | |
| 227 } | |
| 228 reader, err := gzip.NewReader(bytes.NewReader(gz.([]byte))) | |
| 229 if err != nil { | |
| 230 return err | |
| 231 } | |
| 232 bs, err := ioutil.ReadAll(reader) | |
| 233 if err != nil { | |
| 234 return err | |
| 235 } | |
| 236 return json.Unmarshal(bs, b) | |
| 237 } | |
| 238 | |
| 239 func (b *buildbotBuild) getPropertyValue(name string) interface{} { | |
| 240 for _, prop := range b.Properties { | |
| 241 if prop.Name == name { | |
| 242 return prop.Value | |
| 243 } | |
| 244 } | |
| 245 return "" | |
| 246 } | |
| 247 | |
| 248 type errTooBig struct { | |
| 249 error | |
| 250 } | |
| 251 | |
| 252 // Save returns a property map of the struct to save in the datastore. It | |
| 253 // contains two fields, the ID which is the key, and a data field which is a | |
| 254 // serialized and gzipped representation of the entire struct. | |
| 255 func (b *buildbotBuild) Save(withMeta bool) (datastore.PropertyMap, error) { | |
| 256 bs, err := json.Marshal(b) | |
| 257 if err != nil { | |
| 258 return nil, err | |
| 259 } | |
| 260 gzbs := bytes.Buffer{} | |
| 261 gsw := gzip.NewWriter(&gzbs) | |
| 262 _, err = gsw.Write(bs) | |
| 263 if err != nil { | |
| 264 return nil, err | |
| 265 } | |
| 266 err = gsw.Close() | |
| 267 if err != nil { | |
| 268 return nil, err | |
| 269 } | |
| 270 blob := gzbs.Bytes() | |
| 271 // Datastore has a max size of 1MB. If the blob is over 9.5MB, it proba
bly | |
| 272 // won't fit after accounting for overhead. | |
| 273 if len(blob) > 950000 { | |
| 274 return nil, errTooBig{ | |
| 275 fmt.Errorf("buildbotBuild: Build too big to store (%d by
tes)", len(blob))} | |
| 276 } | |
| 277 p := datastore.PropertyMap{ | |
| 278 "data": datastore.MkPropertyNI(blob), | |
| 279 } | |
| 280 if withMeta { | |
| 281 p["id"] = datastore.MkPropertyNI(b.getID()) | |
| 282 p["master"] = datastore.MkProperty(b.Master) | |
| 283 p["builder"] = datastore.MkProperty(b.Buildername) | |
| 284 p["number"] = datastore.MkProperty(b.Number) | |
| 285 p["finished"] = datastore.MkProperty(b.Finished) | |
| 286 } | |
| 287 return p, nil | |
| 288 } | |
| 289 | |
| 290 type buildbotPending struct { | |
| 291 Source buildbotSourceStamp `json:"source"` | |
| 292 Reason string `json:"reason"` | |
| 293 SubmittedAt int `json:"submittedAt"` | |
| 294 BuilderName string `json:"builderName"` | |
| 295 } | |
| 296 | |
| 297 // buildbotBuilder is a builder struct from the master json, _not_ the builder j
son. | |
| 298 type buildbotBuilder struct { | |
| 299 Basedir string `json:"basedir"` | |
| 300 CachedBuilds []int `json:"cachedBuilds"` | |
| 301 PendingBuilds int `json:"pendingBuilds"` | |
| 302 // This one is specific to the pubsub interface. This is limited to 75, | |
| 303 // so it could differ from PendingBuilds | |
| 304 PendingBuildStates []*buildbotPending `json:"pendingBuildStates"` | |
| 305 Category string `json:"category"` | |
| 306 CurrentBuilds []int `json:"currentBuilds"` | |
| 307 Slaves []string `json:"slaves"` | |
| 308 State string `json:"state"` | |
| 309 } | |
| 310 | |
| 311 // buildbotChangeSource is a changesource (ie polling source) usually tied to a
master's scheduler. | |
| 312 type buildbotChangeSource struct { | |
| 313 Description string `json:"description"` | |
| 314 } | |
| 315 | |
| 316 // buildbotChange describes a commit in a repository as part of a changesource o
f blamelist. | |
| 317 type buildbotChange struct { | |
| 318 At string `json:"at"` | |
| 319 Branch *string `json:"branch"` | |
| 320 Category string `json:"category"` | |
| 321 Comments string `json:"comments"` | |
| 322 // This could be a list of strings or list of struct { Name string } . | |
| 323 Files []interface{} `json:"files"` | |
| 324 Number int `json:"number"` | |
| 325 Project string `json:"project"` | |
| 326 Properties [][]interface{} `json:"properties"` | |
| 327 Repository string `json:"repository"` | |
| 328 Rev string `json:"rev"` | |
| 329 Revision string `json:"revision"` | |
| 330 Revlink string `json:"revlink"` | |
| 331 When int `json:"when"` | |
| 332 Who string `json:"who"` | |
| 333 } | |
| 334 | |
| 335 func (bc *buildbotChange) GetFiles() []string { | |
| 336 files := make([]string, 0, len(bc.Files)) | |
| 337 for _, f := range bc.Files { | |
| 338 // Buildbot stores files both as a string, or as a dict with a s
ingle entry | |
| 339 // named "name". It doesn't matter to us what the type is, but
we need | |
| 340 // to reflect on the type anyways. | |
| 341 switch fn := f.(type) { | |
| 342 case string: | |
| 343 files = append(files, fn) | |
| 344 case map[string]interface{}: | |
| 345 if name, ok := fn["name"]; ok { | |
| 346 files = append(files, fmt.Sprintf("%s", name)) | |
| 347 } | |
| 348 } | |
| 349 } | |
| 350 return files | |
| 351 } | |
| 352 | |
| 353 // buildbotSlave describes a slave on a master from a master json, and also incl
udes the | |
| 354 // full builds of any currently running builds. | |
| 355 type buildbotSlave struct { | |
| 356 // RecentBuilds is a map of builder name to a list of recent finished bu
ild | |
| 357 // numbers on that builder. | |
| 358 RecentBuilds map[string][]int `json:"builders"` | |
| 359 Connected bool `json:"connected"` | |
| 360 Host string `json:"host"` | |
| 361 Name string `json:"name"` | |
| 362 Runningbuilds []*buildbotBuild `json:"runningBuilds"` | |
| 363 Version string `json:"version"` | |
| 364 // This is like runningbuilds, but instead of storing the full build, | |
| 365 // just reference the build by builder: build num. | |
| 366 RunningbuildsMap map[string][]int `json:"runningBuildsMap"` | |
| 367 } | |
| 368 | |
| 369 type buildbotProject struct { | |
| 370 BuildbotURL string `json:"buildbotURL"` | |
| 371 Title string `json:"title"` | |
| 372 Titleurl string `json:"titleURL"` | |
| 373 } | |
| 374 | |
| 375 // buildbotMaster This is json definition for https://build.chromium.org/p/<mast
er>/json | |
| 376 // endpoints. | |
| 377 type buildbotMaster struct { | |
| 378 AcceptingBuilds struct { | |
| 379 AcceptingBuilds bool `json:"accepting_builds"` | |
| 380 } `json:"accepting_builds"` | |
| 381 | |
| 382 Builders map[string]*buildbotBuilder `json:"builders"` | |
| 383 | |
| 384 Buildstate struct { | |
| 385 AcceptingBuilds bool `json:"accepting_builds"` | |
| 386 Builders []struct { | |
| 387 Basedir string `json:"basedir"` | |
| 388 Buildername string `json:"builderName"` | |
| 389 Cachedbuilds []int `json:"cachedBuilds"` | |
| 390 Category string `json:"category"` | |
| 391 Currentbuilds []int `json:"currentBuilds"` | |
| 392 Slaves []string `json:"slaves"` | |
| 393 State string `json:"state"` | |
| 394 } `json:"builders"` | |
| 395 Project struct { | |
| 396 BuildbotURL string `json:"buildbotURL"` | |
| 397 Title string `json:"title"` | |
| 398 Titleurl string `json:"titleURL"` | |
| 399 } `json:"project"` | |
| 400 Timestamp float64 `json:"timestamp"` | |
| 401 } `json:"buildstate"` | |
| 402 | |
| 403 ChangeSources map[string]buildbotChangeSource `json:"change_sources"` | |
| 404 | |
| 405 Changes map[string]buildbotChange `json:"changes"` | |
| 406 | |
| 407 Clock struct { | |
| 408 Current struct { | |
| 409 Local string `json:"local"` | |
| 410 Utc string `json:"utc"` | |
| 411 UtcTs float64 `json:"utc_ts"` | |
| 412 } `json:"current"` | |
| 413 ServerStarted struct { | |
| 414 Local string `json:"local"` | |
| 415 Utc string `json:"utc"` | |
| 416 UtcTs float64 `json:"utc_ts"` | |
| 417 } `json:"server_started"` | |
| 418 ServerUptime float64 `json:"server_uptime"` | |
| 419 } `json:"clock"` | |
| 420 | |
| 421 Project buildbotProject `json:"project"` | |
| 422 | |
| 423 Slaves map[string]*buildbotSlave `json:"slaves"` | |
| 424 | |
| 425 Varz struct { | |
| 426 AcceptingBuilds bool `json:"accepting_builds"` | |
| 427 Builders map[string]struct { | |
| 428 ConnectedSlaves int `json:"connected_slaves"` | |
| 429 CurrentBuilds int `json:"current_builds"` | |
| 430 PendingBuilds int `json:"pending_builds"` | |
| 431 State string `json:"state"` | |
| 432 TotalSlaves int `json:"total_slaves"` | |
| 433 } `json:"builders"` | |
| 434 ServerUptime float64 `json:"server_uptime"` | |
| 435 } `json:"varz"` | |
| 436 | |
| 437 // This is injected by the pubsub publisher on the buildbot side. | |
| 438 Name string `json:"name"` | |
| 439 } | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 2944983003:
[milo] {buildbucket,buildbot,swarming,logdog} -> backends/*. (Closed)
