Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(126)

Unified Diff: milo/appengine/buildbot/buildinfo.go

Issue 2944983003: [milo] {buildbucket,buildbot,swarming,logdog} -> backends/*. (Closed)
Patch Set: fix the tests Created 3 years, 6 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View side-by-side diff with in-line comments
Download patch
« no previous file with comments | « milo/appengine/buildbot/builder_test.go ('k') | milo/appengine/buildbot/buildinfo_test.go » ('j') | no next file with comments »
Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
Index: milo/appengine/buildbot/buildinfo.go
diff --git a/milo/appengine/buildbot/buildinfo.go b/milo/appengine/buildbot/buildinfo.go
deleted file mode 100644
index 84d404ae01b2e03523e734ed7321fd0bd9185374..0000000000000000000000000000000000000000
--- a/milo/appengine/buildbot/buildinfo.go
+++ /dev/null
@@ -1,273 +0,0 @@
-// Copyright 2017 The LUCI Authors. All rights reserved.
-// Use of this source code is governed under the Apache License, Version 2.0
-// that can be found in the LICENSE file.
-
-package buildbot
-
-import (
- "fmt"
- "strconv"
- "strings"
- "unicode/utf8"
-
- "github.com/luci/luci-go/common/data/stringset"
- "github.com/luci/luci-go/common/logging"
- miloProto "github.com/luci/luci-go/common/proto/milo"
- "github.com/luci/luci-go/grpc/grpcutil"
- "github.com/luci/luci-go/logdog/client/coordinator"
- "github.com/luci/luci-go/logdog/common/types"
- "github.com/luci/luci-go/luci_config/common/cfgtypes"
- milo "github.com/luci/luci-go/milo/api/proto"
- "github.com/luci/luci-go/milo/appengine/logdog"
-
- "google.golang.org/grpc/codes"
-
- "golang.org/x/net/context"
-)
-
-// BuildInfoProvider is a configuration that provides build information.
-//
-// In a production system, this will be completely defaults. For testing, the
-// various services and data sources may be substituted for testing stubs.
-type BuildInfoProvider struct {
- // LogdogClientFunc returns a coordinator Client instance for the supplied
- // parameters.
- //
- // If nil, a production client will be generated.
- LogdogClientFunc func(c context.Context) (*coordinator.Client, error)
-}
-
-func (p *BuildInfoProvider) newLogdogClient(c context.Context) (*coordinator.Client, error) {
- if p.LogdogClientFunc != nil {
- return p.LogdogClientFunc(c)
- }
- return logdog.NewClient(c, "")
-}
-
-// GetBuildInfo resolves a Milo protobuf Step for a given BuildBot build.
-//
-// On failure, it returns a (potentially-wrapped) gRPC error.
-//
-// This:
-//
-// 1) Fetches the BuildBot build JSON from datastore.
-// 2) Resolves the LogDog annotation stream path from the BuildBot state.
-// 3) Fetches the LogDog annotation stream and resolves it into a Step.
-// 4) Merges some operational BuildBot build information into the Step.
-func (p *BuildInfoProvider) GetBuildInfo(c context.Context, req *milo.BuildInfoRequest_BuildBot,
- projectHint cfgtypes.ProjectName) (*milo.BuildInfoResponse, error) {
-
- logging.Infof(c, "Loading build info for master %q, builder %q, build #%d",
- req.MasterName, req.BuilderName, req.BuildNumber)
-
- // Load the BuildBot build from datastore.
- build, err := getBuild(c, req.MasterName, req.BuilderName, int(req.BuildNumber))
- switch err {
- case errBuildNotFound:
- return nil, grpcutil.Errf(codes.NotFound, "Build #%d for master %q, builder %q was not found",
- req.BuildNumber, req.MasterName, req.BuilderName)
- case errNotAuth:
- return nil, grpcutil.Unauthenticated
- case nil:
- // continue
- default:
- logging.WithError(err).Errorf(c, "Failed to load build info.")
- return nil, grpcutil.Internal
- }
-
- // Create a new LogDog client.
- client, err := p.newLogdogClient(c)
- if err != nil {
- logging.WithError(err).Errorf(c, "Failed to create LogDog client.")
- return nil, grpcutil.Internal
- }
-
- // Identify the LogDog annotation stream from the build.
- //
- // This will return a gRPC error on failure.
- addr, err := getLogDogAnnotationAddr(c, client, build, projectHint)
- if err != nil {
- return nil, err
- }
- logging.Infof(c, "Resolved annotation stream: %s / %s", addr.Project, addr.Path)
-
- // Load the annotation protobuf.
- as := logdog.AnnotationStream{
- Client: client,
- Path: addr.Path,
- Project: addr.Project,
- }
- if err := as.Normalize(); err != nil {
- logging.WithError(err).Errorf(c, "Failed to normalize annotation stream.")
- return nil, grpcutil.Internal
- }
-
- step, err := as.Fetch(c)
- if err != nil {
- logging.WithError(err).Errorf(c, "Failed to load annotation stream.")
- return nil, grpcutil.Errf(codes.Internal, "failed to load LogDog annotation stream from: %s", as.Path)
- }
-
- // Merge the information together.
- if err := mergeBuildIntoAnnotation(c, step, build); err != nil {
- logging.WithError(err).Errorf(c, "Failed to merge annotation with build.")
- return nil, grpcutil.Errf(codes.Internal, "failed to merge annotation and build data")
- }
-
- prefix, name := as.Path.Split()
- return &milo.BuildInfoResponse{
- Project: string(as.Project),
- Step: step,
- AnnotationStream: &miloProto.LogdogStream{
- Server: client.Host,
- Prefix: string(prefix),
- Name: string(name),
- },
- }, nil
-}
-
-// Resolve BuildBot and LogDog build information. We do this
-//
-// This returns an AnnotationStream instance with its project and path
-// populated.
-//
-// This function is messy and implementation-specific. That's the point of this
-// endpoint, though. All of the nastiness here should be replaced with something
-// more elegant once that becomes available. In the meantime...
-func getLogDogAnnotationAddr(c context.Context, client *coordinator.Client, build *buildbotBuild,
- projectHint cfgtypes.ProjectName) (*types.StreamAddr, error) {
-
- if v, ok := build.getPropertyValue("log_location").(string); ok && v != "" {
- addr, err := types.ParseURL(v)
- if err == nil {
- return addr, nil
- }
-
- logging.Fields{
- logging.ErrorKey: err,
- "log_location": v,
- }.Debugf(c, "'log_location' property did not parse as LogDog URL.")
- }
-
- // logdog_annotation_url (if present, must be valid)
- if v, ok := build.getPropertyValue("logdog_annotation_url").(string); ok && v != "" {
- addr, err := types.ParseURL(v)
- if err != nil {
- logging.Fields{
- logging.ErrorKey: err,
- "url": v,
- }.Errorf(c, "Failed to parse 'logdog_annotation_url' property.")
- return nil, grpcutil.Errf(codes.FailedPrecondition, "build has invalid annotation URL")
- }
-
- return addr, nil
- }
-
- // Modern builds will have this information in their build properties.
- var addr types.StreamAddr
- prefix, _ := build.getPropertyValue("logdog_prefix").(string)
- project, _ := build.getPropertyValue("logdog_project").(string)
- if prefix != "" && project != "" {
- // Construct the full annotation path.
- addr.Project = cfgtypes.ProjectName(project)
- addr.Path = types.StreamName(prefix).Join("annotations")
-
- logging.Debugf(c, "Resolved path/project from build properties.")
- return &addr, nil
- }
-
- // From here on out, we will need a project hint.
- if projectHint == "" {
- return nil, grpcutil.Errf(codes.NotFound, "annotation stream not found")
- }
- addr.Project = projectHint
-
- // Execute a LogDog service query to see if we can identify the stream.
- err := func() error {
- var annotationStream *coordinator.LogStream
- err := client.Query(c, addr.Project, "", coordinator.QueryOptions{
- Tags: map[string]string{
- "buildbot.master": build.Master,
- "buildbot.builder": build.Buildername,
- "buildbot.buildnumber": strconv.Itoa(build.Number),
- },
- ContentType: miloProto.ContentTypeAnnotations,
- }, func(ls *coordinator.LogStream) bool {
- // Only need the first (hopefully only?) result.
- annotationStream = ls
- return false
- })
- if err != nil {
- logging.WithError(err).Errorf(c, "Failed to issue log stream query.")
- return grpcutil.Internal
- }
-
- if annotationStream != nil {
- addr.Path = annotationStream.Path
- }
- return nil
- }()
- if err != nil {
- return nil, err
- }
- if addr.Path != "" {
- logging.Debugf(c, "Resolved path/project via tag query.")
- return &addr, nil
- }
-
- // Last-ditch effort: generate a prefix based on the build properties. This
- // re-implements the "_build_prefix" function in:
- // https://chromium.googlesource.com/chromium/tools/build/+/2d23e5284cc31f31c6bc07aa1d3fc5b1c454c3b4/scripts/slave/logdog_bootstrap.py#363
- isAlnum := func(r rune) bool {
- return ((r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9'))
- }
- normalize := func(v string) string {
- v = strings.Map(func(r rune) rune {
- if isAlnum(r) {
- return r
- }
- switch r {
- case ':', '_', '-', '.':
- return r
- default:
- return '_'
- }
- }, v)
- if r, _ := utf8.DecodeRuneInString(v); r == utf8.RuneError || !isAlnum(r) {
- v = "s_" + v
- }
- return v
- }
- addr.Path = types.StreamPath(fmt.Sprintf("bb/%s/%s/%s/+/annotations",
- normalize(build.Master), normalize(build.Buildername), normalize(strconv.Itoa(build.Number))))
-
- logging.Debugf(c, "Generated path/project algorithmically.")
- return &addr, nil
-}
-
-// mergeBuildInfoIntoAnnotation merges BuildBot-specific build informtion into
-// a LogDog annotation protobuf.
-//
-// This consists of augmenting the Step's properties with BuildBot's properties,
-// favoring the Step's version of the properties if there are two with the same
-// name.
-func mergeBuildIntoAnnotation(c context.Context, step *miloProto.Step, build *buildbotBuild) error {
- allProps := stringset.New(len(step.Property) + len(build.Properties))
- for _, prop := range step.Property {
- allProps.Add(prop.Name)
- }
- for _, prop := range build.Properties {
- // Annotation protobuf overrides BuildBot properties.
- if allProps.Has(prop.Name) {
- continue
- }
- allProps.Add(prop.Name)
-
- step.Property = append(step.Property, &miloProto.Step_Property{
- Name: prop.Name,
- Value: fmt.Sprintf("%v", prop.Value),
- })
- }
-
- return nil
-}
« no previous file with comments | « milo/appengine/buildbot/builder_test.go ('k') | milo/appengine/buildbot/buildinfo_test.go » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698