| OLD | NEW |
|---|
| (Empty) |
| 1 // Copyright 2017 The LUCI Authors. All rights reserved. | |
| 2 // Use of this source code is governed under the Apache License, Version 2.0 | |
| 3 // that can be found in the LICENSE file. | |
| 4 | |
| 5 package swarming | |
| 6 | |
| 7 import ( | |
| 8 "strconv" | |
| 9 "strings" | |
| 10 "unicode/utf8" | |
| 11 | |
| 12 swarming "github.com/luci/luci-go/common/api/swarming/swarming/v1" | |
| 13 "github.com/luci/luci-go/common/errors" | |
| 14 "github.com/luci/luci-go/common/logging" | |
| 15 miloProto "github.com/luci/luci-go/common/proto/milo" | |
| 16 "github.com/luci/luci-go/grpc/grpcutil" | |
| 17 "github.com/luci/luci-go/logdog/common/types" | |
| 18 "github.com/luci/luci-go/luci_config/common/cfgtypes" | |
| 19 milo "github.com/luci/luci-go/milo/api/proto" | |
| 20 "github.com/luci/luci-go/swarming/tasktemplate" | |
| 21 | |
| 22 "golang.org/x/net/context" | |
| 23 ) | |
| 24 | |
| 25 // BuildInfoProvider provides build information. | |
| 26 // | |
| 27 // In a production system, this will be completely defaults. For testing, the | |
| 28 // various services and data sources may be substituted for testing stubs. | |
| 29 type BuildInfoProvider struct { | |
| 30 bl buildLoader | |
| 31 | |
| 32 // swarmingServiceFunc returns a swarmingService instance for the suppli
ed | |
| 33 // parameters. | |
| 34 // | |
| 35 // If nil, a production fetcher will be generated. | |
| 36 swarmingServiceFunc func(c context.Context, host string) (swarmingServic
e, error) | |
| 37 } | |
| 38 | |
| 39 func (p *BuildInfoProvider) newSwarmingService(c context.Context, host string) (
swarmingService, error) { | |
| 40 if p.swarmingServiceFunc == nil { | |
| 41 return newProdService(c, host) | |
| 42 } | |
| 43 return p.swarmingServiceFunc(c, host) | |
| 44 } | |
| 45 | |
| 46 // GetBuildInfo resolves a Milo protobuf Step for a given Swarming task. | |
| 47 func (p *BuildInfoProvider) GetBuildInfo(c context.Context, req *milo.BuildInfoR
equest_Swarming, | |
| 48 projectHint cfgtypes.ProjectName) (*milo.BuildInfoResponse, error) { | |
| 49 | |
| 50 // Load the Swarming task (no log content). | |
| 51 sf, err := p.newSwarmingService(c, req.Host) | |
| 52 if err != nil { | |
| 53 logging.WithError(err).Errorf(c, "Failed to create Swarming fetc
her.") | |
| 54 return nil, grpcutil.Internal | |
| 55 } | |
| 56 | |
| 57 // Use default Swarming host. | |
| 58 host := sf.getHost() | |
| 59 logging.Infof(c, "Loading build info for Swarming host %s, task %s.", ho
st, req.Task) | |
| 60 | |
| 61 fetchParams := swarmingFetchParams{ | |
| 62 fetchReq: true, | |
| 63 fetchRes: true, | |
| 64 } | |
| 65 fr, err := swarmingFetch(c, sf, req.Task, fetchParams) | |
| 66 if err != nil { | |
| 67 if err == errNotMiloJob { | |
| 68 logging.Warningf(c, "User requested nonexistent task or
does not have permissions.") | |
| 69 return nil, grpcutil.NotFound | |
| 70 } | |
| 71 | |
| 72 logging.WithError(err).Errorf(c, "Failed to load Swarming task."
) | |
| 73 return nil, grpcutil.Internal | |
| 74 } | |
| 75 | |
| 76 // Determine the LogDog annotation stream path for this Swarming task. | |
| 77 // | |
| 78 // On failure, will return a gRPC error. | |
| 79 stream, err := resolveLogDogAnnotations(c, fr.req, projectHint, host, re
q.Task, fr.res.TryNumber) | |
| 80 if err != nil { | |
| 81 logging.WithError(err).Warningf(c, "Failed to get annotation str
eam parameters.") | |
| 82 return nil, err | |
| 83 } | |
| 84 | |
| 85 logging.Fields{ | |
| 86 "host": stream.Host, | |
| 87 "project": stream.Project, | |
| 88 "path": stream.Path, | |
| 89 }.Infof(c, "Resolved LogDog annotation stream.") | |
| 90 | |
| 91 as, err := p.bl.newEmptyAnnotationStream(c, stream) | |
| 92 if err != nil { | |
| 93 logging.WithError(err).Errorf(c, "Failed to create LogDog annota
tion stream.") | |
| 94 return nil, grpcutil.Internal | |
| 95 } | |
| 96 | |
| 97 // Fetch LogDog annotation stream data. | |
| 98 step, err := as.Fetch(c) | |
| 99 if err != nil { | |
| 100 logging.WithError(err).Warningf(c, "Failed to load annotation st
ream.") | |
| 101 return nil, grpcutil.Internal | |
| 102 } | |
| 103 | |
| 104 // Add Swarming task parameters to the Milo step. | |
| 105 if err := addTaskToMiloStep(c, sf.getHost(), fr.res, step); err != nil { | |
| 106 return nil, err | |
| 107 } | |
| 108 | |
| 109 prefix, name := as.Path.Split() | |
| 110 return &milo.BuildInfoResponse{ | |
| 111 Project: string(as.Project), | |
| 112 Step: step, | |
| 113 AnnotationStream: &miloProto.LogdogStream{ | |
| 114 Server: as.Client.Host, | |
| 115 Prefix: string(prefix), | |
| 116 Name: string(name), | |
| 117 }, | |
| 118 }, nil | |
| 119 } | |
| 120 | |
| 121 // resolveLogDogAnnotations returns a configured AnnotationStream given the inpu
t | |
| 122 // parameters. | |
| 123 // | |
| 124 // This will return a gRPC error on failure. | |
| 125 // | |
| 126 // This function is messy and implementation-specific. That's the point of this | |
| 127 // endpoint, though. All of the nastiness here should be replaced with something | |
| 128 // more elegant once that becomes available. In the meantime... | |
| 129 func resolveLogDogAnnotations(c context.Context, sr *swarming.SwarmingRpcsTaskRe
quest, projectHint cfgtypes.ProjectName, | |
| 130 host, taskID string, tryNumber int64) (*types.StreamAddr, error) { | |
| 131 | |
| 132 // Try and resolve from explicit tags (preferred). | |
| 133 tags := swarmingTags(sr.Tags) | |
| 134 addr, err := resolveLogDogStreamAddrFromTags(tags, taskID, tryNumber) | |
| 135 if err == nil { | |
| 136 return addr, nil | |
| 137 } | |
| 138 logging.WithError(err).Debugf(c, "Could not resolve stream address from
tags.") | |
| 139 | |
| 140 // If this is a Kitchen command, maybe we can infer our LogDog project f
rom | |
| 141 // the command-line. | |
| 142 if sr.Properties == nil { | |
| 143 logging.Warningf(c, "No request properties, can't infer annotati
on stream path.") | |
| 144 return nil, grpcutil.NotFound | |
| 145 } | |
| 146 | |
| 147 addr = &types.StreamAddr{} | |
| 148 var isKitchen bool | |
| 149 if addr.Project, isKitchen = getLogDogProjectFromKitchen(sr.Properties.C
ommand); !isKitchen { | |
| 150 logging.Warningf(c, "Not a Kitchen CLI, can't infer annotation s
tream path.") | |
| 151 return nil, grpcutil.NotFound | |
| 152 } | |
| 153 | |
| 154 if addr.Project == "" { | |
| 155 addr.Project = projectHint | |
| 156 } | |
| 157 if addr.Project == "" { | |
| 158 logging.Warningf(c, "Don't know how to get annotation stream pat
h.") | |
| 159 return nil, grpcutil.NotFound | |
| 160 } | |
| 161 | |
| 162 // This is a Kitchen run, and it has a project! Construct the annotation | |
| 163 // stream path. | |
| 164 // | |
| 165 // This is an implementation of: | |
| 166 // https://chromium.googlesource.com/infra/infra/+/a7032e3e240d4b81a1912
bfaf29a20d02f665cc1/go/src/infra/tools/kitchen/cook_logdog.go#129 | |
| 167 runID, err := getRunID(taskID, tryNumber) | |
| 168 if err != nil { | |
| 169 logging.Fields{ | |
| 170 logging.ErrorKey: err, | |
| 171 "taskID": taskID, | |
| 172 "tryNumber": tryNumber, | |
| 173 }.Errorf(c, "Failed to get Run ID for task/try.") | |
| 174 return nil, grpcutil.Internal | |
| 175 } | |
| 176 | |
| 177 prefix, err := types.MakeStreamName("", "swarm", host, runID) | |
| 178 if err != nil { | |
| 179 logging.WithError(err).Errorf(c, "Failed to generate Swarming pr
efix.") | |
| 180 return nil, grpcutil.Internal | |
| 181 } | |
| 182 addr.Path = prefix.Join("annotations") | |
| 183 return addr, nil | |
| 184 } | |
| 185 | |
| 186 func getLogDogProjectFromKitchen(cmd []string) (proj cfgtypes.ProjectName, isKit
chen bool) { | |
| 187 // Is this a Kitchen command? | |
| 188 switch { | |
| 189 case len(cmd) == 0: | |
| 190 return | |
| 191 case !strings.HasPrefix(cmd[0], "kitchen"): | |
| 192 return | |
| 193 } | |
| 194 isKitchen = true | |
| 195 cmd = cmd[1:] | |
| 196 | |
| 197 // Scan through for the "-logdog-project" argument. | |
| 198 for i, arg := range cmd { | |
| 199 if arg == "-logdog-project" { | |
| 200 if i < len(cmd)-2 { | |
| 201 proj = cfgtypes.ProjectName(cmd[i+1]) | |
| 202 return | |
| 203 } | |
| 204 break | |
| 205 } | |
| 206 } | |
| 207 return | |
| 208 } | |
| 209 | |
| 210 func resolveLogDogStreamAddrFromTags(tags map[string]string, taskID string, tryN
umber int64) (*types.StreamAddr, error) { | |
| 211 // If we don't have a LUCI project, abort. | |
| 212 luciProject, logLocation := tags["luci_project"], tags["log_location"] | |
| 213 switch { | |
| 214 case luciProject == "": | |
| 215 return nil, errors.New("no 'luci_project' tag") | |
| 216 case logLocation == "": | |
| 217 return nil, errors.New("no 'log_location' tag") | |
| 218 } | |
| 219 | |
| 220 // Gather our Swarming task template parameters and perform a substituti
on. | |
| 221 runID, err := getRunID(taskID, tryNumber) | |
| 222 if err != nil { | |
| 223 return nil, errors.Annotate(err).Err() | |
| 224 } | |
| 225 p := tasktemplate.Params{ | |
| 226 SwarmingRunID: runID, | |
| 227 } | |
| 228 if logLocation, err = p.Resolve(logLocation); err != nil { | |
| 229 return nil, errors.Annotate(err).Reason("failed to resolve swarm
ing task templating in 'log_location'").Err() | |
| 230 } | |
| 231 | |
| 232 addr, err := types.ParseURL(logLocation) | |
| 233 if err != nil { | |
| 234 return nil, errors.Annotate(err).Reason("could not parse LogDog
stream from location").Err() | |
| 235 } | |
| 236 | |
| 237 // The LogDog stream's project should match the LUCI project. | |
| 238 if string(addr.Project) != luciProject { | |
| 239 return nil, errors.Reason("stream project %(streamProject)q does
n't match LUCI project %(luciProject)q"). | |
| 240 D("luciProject", luciProject). | |
| 241 D("streamProject", addr.Project). | |
| 242 Err() | |
| 243 } | |
| 244 | |
| 245 return addr, nil | |
| 246 } | |
| 247 | |
| 248 // getRunID converts a Swarming task ID and try number into a Swarming Run ID. | |
| 249 // | |
| 250 // The run ID is a permutation of the last four bits of the Swarming Task ID. | |
| 251 // Therefore, we chop it off of the string, mutate it, and then add it back. | |
| 252 // | |
| 253 // TODO(dnj): Replace this with Swarming API call once finished. | |
| 254 func getRunID(taskID string, tryNumber int64) (string, error) { | |
| 255 // Slice off the last character form the task ID. | |
| 256 if len(taskID) == 0 { | |
| 257 return "", errors.New("swarming task ID is empty") | |
| 258 } | |
| 259 | |
| 260 // Parse "tryNumber" as a hex string. | |
| 261 if tryNumber < 0 || tryNumber > 0x0F { | |
| 262 return "", errors.Reason("try number %(try)d exceeds 4 bits"). | |
| 263 D("try", tryNumber). | |
| 264 Err() | |
| 265 } | |
| 266 | |
| 267 lastChar, lastCharSize := utf8.DecodeLastRuneInString(taskID) | |
| 268 v, err := strconv.ParseUint(string(lastChar), 16, 8) | |
| 269 if err != nil { | |
| 270 return "", errors.Annotate(err).Reason("failed to parse hex from
rune: %(rune)r"). | |
| 271 D("rune", lastChar). | |
| 272 Err() | |
| 273 } | |
| 274 | |
| 275 return taskID[:len(taskID)-lastCharSize] + strconv.FormatUint((v|uint64(
tryNumber)), 16), nil | |
| 276 } | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 2944983003:
[milo] {buildbucket,buildbot,swarming,logdog} -> backends/*. (Closed)
