| OLD | NEW |
| 1 // Copyright 2015 The LUCI Authors. All rights reserved. | 1 // Copyright 2015 The LUCI Authors. All rights reserved. |
| 2 // Use of this source code is governed under the Apache License, Version 2.0 | 2 // Use of this source code is governed under the Apache License, Version 2.0 |
| 3 // that can be found in the LICENSE file. | 3 // that can be found in the LICENSE file. |
| 4 | 4 |
| 5 package swarming | 5 package swarming |
| 6 | 6 |
| 7 import ( | 7 import ( |
| 8 "bytes" | 8 "bytes" |
| 9 "fmt" | 9 "fmt" |
| 10 "net/http" | 10 "net/http" |
| 11 "net/url" | 11 "net/url" |
| 12 "strings" | 12 "strings" |
| 13 "time" | 13 "time" |
| 14 | 14 |
| 15 "golang.org/x/net/context" | 15 "golang.org/x/net/context" |
| 16 | 16 |
| 17 swarming "github.com/luci/luci-go/common/api/swarming/swarming/v1" | 17 swarming "github.com/luci/luci-go/common/api/swarming/swarming/v1" |
| 18 "github.com/luci/luci-go/common/errors" | 18 "github.com/luci/luci-go/common/errors" |
| 19 "github.com/luci/luci-go/common/logging" | 19 "github.com/luci/luci-go/common/logging" |
| 20 "github.com/luci/luci-go/common/proto/google" | 20 "github.com/luci/luci-go/common/proto/google" |
| 21 miloProto "github.com/luci/luci-go/common/proto/milo" | 21 miloProto "github.com/luci/luci-go/common/proto/milo" |
| 22 "github.com/luci/luci-go/common/sync/parallel" | 22 "github.com/luci/luci-go/common/sync/parallel" |
| 23 "github.com/luci/luci-go/logdog/client/annotee" | 23 "github.com/luci/luci-go/logdog/client/annotee" |
| 24 "github.com/luci/luci-go/logdog/client/coordinator" | 24 "github.com/luci/luci-go/logdog/client/coordinator" |
| 25 "github.com/luci/luci-go/logdog/common/types" | 25 "github.com/luci/luci-go/logdog/common/types" |
| 26 "github.com/luci/luci-go/milo/api/resp" | 26 "github.com/luci/luci-go/milo/api/resp" |
| 27 » "github.com/luci/luci-go/milo/build_source/raw_presentation" | 27 » "github.com/luci/luci-go/milo/buildsource/rawpresentation" |
| 28 "github.com/luci/luci-go/milo/common" | 28 "github.com/luci/luci-go/milo/common" |
| 29 "github.com/luci/luci-go/milo/common/model" | 29 "github.com/luci/luci-go/milo/common/model" |
| 30 "github.com/luci/luci-go/server/auth" | 30 "github.com/luci/luci-go/server/auth" |
| 31 ) | 31 ) |
| 32 | 32 |
| 33 // errNotMiloJob is returned if a Swarming task is fetched that does not self- | 33 // errNotMiloJob is returned if a Swarming task is fetched that does not self- |
| 34 // identify as a Milo job. | 34 // identify as a Milo job. |
| 35 var errNotMiloJob = errors.New("Not a Milo Job or access denied") | 35 var errNotMiloJob = errors.New("Not a Milo Job or access denied") |
| 36 | 36 |
| 37 // SwarmingTimeLayout is time layout used by swarming. | 37 // SwarmingTimeLayout is time layout used by swarming. |
| (...skipping 417 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 455 // Add a link to the bot. | 455 // Add a link to the bot. |
| 456 if sr.BotId != "" { | 456 if sr.BotId != "" { |
| 457 build.Summary.Bot = resp.NewLink(sr.BotId, botPageURL(server, sr
.BotId)) | 457 build.Summary.Bot = resp.NewLink(sr.BotId, botPageURL(server, sr
.BotId)) |
| 458 } | 458 } |
| 459 | 459 |
| 460 return nil | 460 return nil |
| 461 } | 461 } |
| 462 | 462 |
| 463 // streamsFromAnnotatedLog takes in an annotated log and returns a fully | 463 // streamsFromAnnotatedLog takes in an annotated log and returns a fully |
| 464 // populated set of logdog streams | 464 // populated set of logdog streams |
| 465 func streamsFromAnnotatedLog(ctx context.Context, log string) (*raw_presentation
.Streams, error) { | 465 func streamsFromAnnotatedLog(ctx context.Context, log string) (*rawpresentation.
Streams, error) { |
| 466 c := &memoryClient{} | 466 c := &memoryClient{} |
| 467 p := annotee.New(ctx, annotee.Options{ | 467 p := annotee.New(ctx, annotee.Options{ |
| 468 Client: c, | 468 Client: c, |
| 469 MetadataUpdateInterval: -1, // Neverrrrrr send incr updates. | 469 MetadataUpdateInterval: -1, // Neverrrrrr send incr updates. |
| 470 Offline: true, | 470 Offline: true, |
| 471 }) | 471 }) |
| 472 | 472 |
| 473 is := annotee.Stream{ | 473 is := annotee.Stream{ |
| 474 Reader: bytes.NewBufferString(log), | 474 Reader: bytes.NewBufferString(log), |
| 475 Name: types.StreamName("stdout"), | 475 Name: types.StreamName("stdout"), |
| (...skipping 15 matching lines...) Expand all Loading... |
| 491 // for testing. | 491 // for testing. |
| 492 type buildLoader struct { | 492 type buildLoader struct { |
| 493 // logdogClientFunc returns a coordinator Client instance for the suppli
ed | 493 // logdogClientFunc returns a coordinator Client instance for the suppli
ed |
| 494 // parameters. | 494 // parameters. |
| 495 // | 495 // |
| 496 // If nil, a production client will be generated. | 496 // If nil, a production client will be generated. |
| 497 logDogClientFunc func(c context.Context, host string) (*coordinator.Clie
nt, error) | 497 logDogClientFunc func(c context.Context, host string) (*coordinator.Clie
nt, error) |
| 498 } | 498 } |
| 499 | 499 |
| 500 func (bl *buildLoader) newEmptyAnnotationStream(c context.Context, addr *types.S
treamAddr) ( | 500 func (bl *buildLoader) newEmptyAnnotationStream(c context.Context, addr *types.S
treamAddr) ( |
| 501 » *raw_presentation.AnnotationStream, error) { | 501 » *rawpresentation.AnnotationStream, error) { |
| 502 | 502 |
| 503 fn := bl.logDogClientFunc | 503 fn := bl.logDogClientFunc |
| 504 if fn == nil { | 504 if fn == nil { |
| 505 » » fn = raw_presentation.NewClient | 505 » » fn = rawpresentation.NewClient |
| 506 } | 506 } |
| 507 client, err := fn(c, addr.Host) | 507 client, err := fn(c, addr.Host) |
| 508 if err != nil { | 508 if err != nil { |
| 509 return nil, errors.Annotate(err, "failed to create LogDog client
").Err() | 509 return nil, errors.Annotate(err, "failed to create LogDog client
").Err() |
| 510 } | 510 } |
| 511 | 511 |
| 512 » as := raw_presentation.AnnotationStream{ | 512 » as := rawpresentation.AnnotationStream{ |
| 513 Client: client, | 513 Client: client, |
| 514 Project: addr.Project, | 514 Project: addr.Project, |
| 515 Path: addr.Path, | 515 Path: addr.Path, |
| 516 } | 516 } |
| 517 if err := as.Normalize(); err != nil { | 517 if err := as.Normalize(); err != nil { |
| 518 return nil, errors.Annotate(err, "failed to normalize annotation
stream parameters").Err() | 518 return nil, errors.Annotate(err, "failed to normalize annotation
stream parameters").Err() |
| 519 } | 519 } |
| 520 | 520 |
| 521 return &as, nil | 521 return &as, nil |
| 522 } | 522 } |
| (...skipping 57 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 580 return true | 580 return true |
| 581 }, | 581 }, |
| 582 } | 582 } |
| 583 fr, err := swarmingFetch(c, svc, taskID, fetchParams) | 583 fr, err := swarmingFetch(c, svc, taskID, fetchParams) |
| 584 if err != nil { | 584 if err != nil { |
| 585 return nil, err | 585 return nil, err |
| 586 } | 586 } |
| 587 | 587 |
| 588 var build resp.MiloBuild | 588 var build resp.MiloBuild |
| 589 var s *miloProto.Step | 589 var s *miloProto.Step |
| 590 » var lds *raw_presentation.Streams | 590 » var lds *rawpresentation.Streams |
| 591 » var ub raw_presentation.URLBuilder | 591 » var ub rawpresentation.URLBuilder |
| 592 | 592 |
| 593 // Load the build from the available data. | 593 // Load the build from the available data. |
| 594 // | 594 // |
| 595 // If the Swarming task explicitly specifies its log location, we prefer
that. | 595 // If the Swarming task explicitly specifies its log location, we prefer
that. |
| 596 // As a fallback, we will try and parse the Swarming task's output for | 596 // As a fallback, we will try and parse the Swarming task's output for |
| 597 // annotations. | 597 // annotations. |
| 598 switch { | 598 switch { |
| 599 case logDogStreamAddr != nil: | 599 case logDogStreamAddr != nil: |
| 600 logging.Infof(c, "Loading build from LogDog stream at: %s", logD
ogStreamAddr) | 600 logging.Infof(c, "Loading build from LogDog stream at: %s", logD
ogStreamAddr) |
| 601 | 601 |
| 602 // If the LogDog stream is available, load the step from that. | 602 // If the LogDog stream is available, load the step from that. |
| 603 as, err := bl.newEmptyAnnotationStream(c, logDogStreamAddr) | 603 as, err := bl.newEmptyAnnotationStream(c, logDogStreamAddr) |
| 604 if err != nil { | 604 if err != nil { |
| 605 return nil, errors.Annotate(err, "failed to create LogDo
g annotation stream").Err() | 605 return nil, errors.Annotate(err, "failed to create LogDo
g annotation stream").Err() |
| 606 } | 606 } |
| 607 | 607 |
| 608 prefix, _ := logDogStreamAddr.Path.Split() | 608 prefix, _ := logDogStreamAddr.Path.Split() |
| 609 » » ub = &raw_presentation.ViewerURLBuilder{ | 609 » » ub = &rawpresentation.ViewerURLBuilder{ |
| 610 Host: logDogStreamAddr.Host, | 610 Host: logDogStreamAddr.Host, |
| 611 Prefix: prefix, | 611 Prefix: prefix, |
| 612 Project: logDogStreamAddr.Project, | 612 Project: logDogStreamAddr.Project, |
| 613 } | 613 } |
| 614 | 614 |
| 615 if s, err = as.Fetch(c); err != nil { | 615 if s, err = as.Fetch(c); err != nil { |
| 616 switch errors.Unwrap(err) { | 616 switch errors.Unwrap(err) { |
| 617 case coordinator.ErrNoSuchStream: | 617 case coordinator.ErrNoSuchStream: |
| 618 // The stream was not found. This could be due
to one of two things: | 618 // The stream was not found. This could be due
to one of two things: |
| 619 // 1. The step just started and we're just waiti
ng for the logs | 619 // 1. The step just started and we're just waiti
ng for the logs |
| (...skipping 41 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 661 | 661 |
| 662 default: | 662 default: |
| 663 s = &miloProto.Step{} | 663 s = &miloProto.Step{} |
| 664 ub = swarmingURLBuilder(linkBase) | 664 ub = swarmingURLBuilder(linkBase) |
| 665 } | 665 } |
| 666 | 666 |
| 667 if s != nil { | 667 if s != nil { |
| 668 if err := addTaskToMiloStep(c, svc.getHost(), fr.res, s); err !=
nil { | 668 if err := addTaskToMiloStep(c, svc.getHost(), fr.res, s); err !=
nil { |
| 669 return nil, err | 669 return nil, err |
| 670 } | 670 } |
| 671 » » raw_presentation.AddLogDogToBuild(c, ub, s, &build) | 671 » » rawpresentation.AddLogDogToBuild(c, ub, s, &build) |
| 672 } | 672 } |
| 673 | 673 |
| 674 if err := addTaskToBuild(c, svc.getHost(), fr.res, &build); err != nil { | 674 if err := addTaskToBuild(c, svc.getHost(), fr.res, &build); err != nil { |
| 675 return nil, err | 675 return nil, err |
| 676 } | 676 } |
| 677 | 677 |
| 678 return &build, nil | 678 return &build, nil |
| 679 } | 679 } |
| 680 | 680 |
| 681 func infoComponent(st model.Status, label, text string) *resp.BuildComponent { | 681 func infoComponent(st model.Status, label, text string) *resp.BuildComponent { |
| (...skipping 86 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 768 for _, tag := range v { | 768 for _, tag := range v { |
| 769 var value string | 769 var value string |
| 770 parts := strings.SplitN(tag, ":", 2) | 770 parts := strings.SplitN(tag, ":", 2) |
| 771 if len(parts) == 2 { | 771 if len(parts) == 2 { |
| 772 value = parts[1] | 772 value = parts[1] |
| 773 } | 773 } |
| 774 res[parts[0]] = value | 774 res[parts[0]] = value |
| 775 } | 775 } |
| 776 return res | 776 return res |
| 777 } | 777 } |
| OLD | NEW |