| OLD | NEW |
|---|
| 1 package model | 1 package model |
| 2 | 2 |
| 3 import ( | 3 import ( |
| 4 "bytes" | 4 "bytes" |
| 5 "encoding/json" | 5 "encoding/json" |
| 6 "io/ioutil" | 6 "io/ioutil" |
| 7 "path/filepath" | 7 "path/filepath" |
| 8 "testing" | 8 "testing" |
| 9 | 9 |
| 10 . "github.com/smartystreets/goconvey/convey" | 10 . "github.com/smartystreets/goconvey/convey" |
| (...skipping 395 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 406 So(err, ShouldBeNil) | 406 So(err, ShouldBeNil) |
| 407 | 407 |
| 408 So(json.Unmarshal(b, &actual), S
houldBeNil) | 408 So(json.Unmarshal(b, &actual), S
houldBeNil) |
| 409 So(actual, ShouldResemble, t) | 409 So(actual, ShouldResemble, t) |
| 410 }) | 410 }) |
| 411 }) | 411 }) |
| 412 }) | 412 }) |
| 413 }) | 413 }) |
| 414 }) | 414 }) |
| 415 } | 415 } |
| 416 |
| 417 func TestMerge(t *testing.T) { |
| 418 t.Parallel() |
| 419 |
| 420 Convey("Merge", t, func() { |
| 421 Convey("AggregateTestLeaf", func() { |
| 422 leaf := &AggregateTestLeaf{ |
| 423 Bugs: []string{"crbug.com/baz"}, |
| 424 Expected: []string{"FAIL"}, |
| 425 Results: []ResultSummary{{2, "P"}, {30, "F"}}, |
| 426 Runtimes: []RuntimeSummary{{1, 500}, {4, 750}, {
1, 250}}, |
| 427 } |
| 428 x := &AggregateTestLeaf{ |
| 429 Bugs: []string{"crbug.com/foo", "crbug.com/b
ar"}, |
| 430 Results: []ResultSummary{{1, "P"}, {42, "X"}}, |
| 431 Runtimes: []RuntimeSummary{{2, 5000}, {4, 5000},
{8, 6000}}, |
| 432 } |
| 433 |
| 434 Convey("Use x.Bugs, x.Expected", func() { |
| 435 x.Expected = []string{"FAIL PASS"} |
| 436 So(leaf.Merge(x), ShouldBeNil) |
| 437 So(leaf.Bugs, ShouldResemble, x.Bugs) |
| 438 So(leaf.Expected, ShouldResemble, x.Expected) |
| 439 }) |
| 440 |
| 441 Convey(`Do not use x.Expected if == "PASS"`, func() { |
| 442 x.Expected = []string{"PASS"} |
| 443 So(leaf.Merge(x), ShouldBeNil) |
| 444 So(leaf.Bugs, ShouldResemble, x.Bugs) |
| 445 So(leaf.Expected, ShouldResemble, []string{"FAIL
"}) |
| 446 }) |
| 447 |
| 448 Convey("Merge matching Results", func() { |
| 449 So(leaf.Merge(x), ShouldBeNil) |
| 450 So(leaf.Results, ShouldResemble, []ResultSummary
{ |
| 451 {42, "X"}, |
| 452 {3, "P"}, |
| 453 {30, "F"}, |
| 454 }) |
| 455 }) |
| 456 |
| 457 Convey("Merge matching Runtimes", func() { |
| 458 So(leaf.Merge(x), ShouldBeNil) |
| 459 So(leaf.Runtimes, ShouldResemble, []RuntimeSumma
ry{ |
| 460 {8, 6000}, |
| 461 {6, 5000}, |
| 462 {1, 500}, |
| 463 {4, 750}, |
| 464 {1, 250}, |
| 465 }) |
| 466 }) |
| 467 }) |
| 468 |
| 469 Convey("AggregateTest", func() { |
| 470 Convey("Join all AggregateTest multi-level", func() { |
| 471 at := AggregateTest{ |
| 472 "foo": &AggregateTestLeaf{ |
| 473 Results: []ResultSummary{{10, "
P"}}, |
| 474 Runtimes: []RuntimeSummary{{1, 2
}}, |
| 475 }, |
| 476 "qux": AggregateTest{ |
| 477 "experiment": &AggregateTestLeaf
{ |
| 478 Results: []ResultSummar
y{{20, "A"}}, |
| 479 Runtimes: []RuntimeSumma
ry{{3, 5}}, |
| 480 }, |
| 481 "paper": &AggregateTestLeaf{ |
| 482 Results: []ResultSummar
y{{1, "B"}}, |
| 483 Runtimes: []RuntimeSumma
ry{{30, 10}}, |
| 484 }, |
| 485 }, |
| 486 } |
| 487 x := AggregateTest{ |
| 488 "bar": &AggregateTestLeaf{ |
| 489 Results: []ResultSummary{{20, "
F"}}, |
| 490 Runtimes: []RuntimeSummary{{3, 0
}}, |
| 491 }, |
| 492 "qux": AggregateTest{ |
| 493 "paper": &AggregateTestLeaf{ |
| 494 Results: []ResultSummar
y{{6, "F"}}, |
| 495 Runtimes: []RuntimeSumma
ry{{2, 3}}, |
| 496 }, |
| 497 "pencil": &AggregateTestLeaf{ |
| 498 Results: []ResultSummar
y{{75, "Z"}}, |
| 499 Runtimes: []RuntimeSumma
ry{{15, 60}}, |
| 500 }, |
| 501 }, |
| 502 } |
| 503 expected := AggregateTest{ |
| 504 "foo": &AggregateTestLeaf{ |
| 505 Results: []ResultSummary{{1, "N
"}, {10, "P"}}, |
| 506 Runtimes: []RuntimeSummary{{1, 0
}, {1, 2}}, |
| 507 }, |
| 508 "bar": &AggregateTestLeaf{ |
| 509 Results: []ResultSummary{{20, "
F"}}, |
| 510 Runtimes: []RuntimeSummary{{3, 0
}}, |
| 511 }, |
| 512 "qux": AggregateTest{ |
| 513 "experiment": &AggregateTestLeaf
{ |
| 514 Results: []ResultSummar
y{{1, "N"}, {20, "A"}}, |
| 515 Runtimes: []RuntimeSumma
ry{{1, 0}, {3, 5}}, |
| 516 }, |
| 517 "paper": &AggregateTestLeaf{ |
| 518 Results: []ResultSummar
y{{6, "F"}, {1, "B"}}, |
| 519 Runtimes: []RuntimeSumma
ry{{2, 3}, {30, 10}}, |
| 520 }, |
| 521 "pencil": &AggregateTestLeaf{ |
| 522 Results: []ResultSummar
y{{75, "Z"}}, |
| 523 Runtimes: []RuntimeSumma
ry{{15, 60}}, |
| 524 }, |
| 525 }, |
| 526 } |
| 527 So(at.Merge(x), ShouldBeNil) |
| 528 So(at, ShouldResemble, expected) |
| 529 }) |
| 530 |
| 531 Convey("Mismatched type", func() { |
| 532 Convey("Merge AggregateTestLeaf into AggregateTe
st", func() { |
| 533 at := AggregateTest{ |
| 534 "foo": AggregateTest{ |
| 535 "bar": &AggregateTestLea
f{ |
| 536 Results: []Resu
ltSummary{{10, "P"}}, |
| 537 Runtimes: []Runt
imeSummary{{1, 2}}, |
| 538 }, |
| 539 }, |
| 540 } |
| 541 x := AggregateTest{ |
| 542 "foo": AggregateTest{ |
| 543 "bar": AggregateTest{ |
| 544 "baz": &Aggregat
eTestLeaf{ |
| 545 Results:
[]ResultSummary{{20, "F"}}, |
| 546 Runtimes
: []RuntimeSummary{{3, 0}}, |
| 547 }, |
| 548 }, |
| 549 }, |
| 550 } |
| 551 err := at.Merge(x) |
| 552 So(err, ShouldNotBeNil) |
| 553 So(err.Error(), ShouldContainSubstring,
" *AggregateTestLeaf") |
| 554 }) |
| 555 |
| 556 Convey("Merge AggregateTest into AggregateTestLe
af", func() { |
| 557 at := AggregateTest{ |
| 558 "foo": AggregateTest{ |
| 559 "bar": &AggregateTestLea
f{ |
| 560 Results: []Resu
ltSummary{{10, "P"}}, |
| 561 Runtimes: []Runt
imeSummary{{1, 2}}, |
| 562 }, |
| 563 }, |
| 564 } |
| 565 x := AggregateTest{ |
| 566 "foo": &AggregateTestLeaf{ |
| 567 Results: []ResultSummar
y{{20, "F"}}, |
| 568 Runtimes: []RuntimeSumma
ry{{3, 0}}, |
| 569 }, |
| 570 } |
| 571 err := at.Merge(x) |
| 572 So(err, ShouldNotBeNil) |
| 573 So(err.Error(), ShouldContainSubstring,
" AggregateTest") |
| 574 }) |
| 575 }) |
| 576 |
| 577 Convey("Merge (all together)", func() { |
| 578 at := AggregateTest{ |
| 579 "presentation": AggregateTest{ |
| 580 "presentation": &AggregateTestLe
af{ |
| 581 Results: []ResultSummar
y{{10, "P"}}, |
| 582 Runtimes: []RuntimeSumma
ry{{1, 2}}, |
| 583 }, |
| 584 "readme": &AggregateTestLeaf{ |
| 585 Results: []ResultSummar
y{{20, "Q"}}, |
| 586 Runtimes: []RuntimeSumma
ry{{3, 4}}, |
| 587 }, |
| 588 }, |
| 589 "io": AggregateTest{ |
| 590 "serial": &AggregateTestLeaf{ |
| 591 Results: []ResultSummar
y{{40, "S"}}, |
| 592 Runtimes: []RuntimeSumma
ry{{8, 9}}, |
| 593 }, |
| 594 }, |
| 595 "hello": AggregateTest{ |
| 596 "world": AggregateTest{ |
| 597 "main": &AggregateTestLe
af{ |
| 598 Results: []Resu
ltSummary{{50, "T"}}, |
| 599 Runtimes: []Runt
imeSummary{{10, 11}}, |
| 600 }, |
| 601 }, |
| 602 }, |
| 603 } |
| 604 x := AggregateTest{ |
| 605 "presentation": AggregateTest{ |
| 606 "presentation": &AggregateTestLe
af{ |
| 607 Results: []ResultSummar
y{{10, "A"}}, |
| 608 Runtimes: []RuntimeSumma
ry{{1, 2}}, |
| 609 }, |
| 610 "readme": &AggregateTestLeaf{ |
| 611 Results: []ResultSummar
y{{10, "B"}}, |
| 612 Runtimes: []RuntimeSumma
ry{{1, 2}}, |
| 613 }, |
| 614 }, |
| 615 "io": AggregateTest{ |
| 616 "analog": &AggregateTestLeaf{ |
| 617 Results: []ResultSummar
y{{10, "D"}}, |
| 618 Runtimes: []RuntimeSumma
ry{{3, 0}}, |
| 619 }, |
| 620 "parallel": AggregateTest{ |
| 621 "duplex": &AggregateTest
Leaf{ |
| 622 Results: []Resu
ltSummary{{10, "E"}}, |
| 623 Runtimes: []Runt
imeSummary{{3, 5}}, |
| 624 }, |
| 625 "simplex": &AggregateTes
tLeaf{ |
| 626 Results: []Resu
ltSummary{{60, "H"}}, |
| 627 Runtimes: []Runt
imeSummary{{30, 50}}, |
| 628 }, |
| 629 }, |
| 630 }, |
| 631 "cool": AggregateTest{ |
| 632 "world": AggregateTest{ |
| 633 "main": &AggregateTestLe
af{ |
| 634 Results: []Resu
ltSummary{{10, "F"}}, |
| 635 Runtimes: []Runt
imeSummary{{3, 6}}, |
| 636 }, |
| 637 }, |
| 638 }, |
| 639 } |
| 640 expected := AggregateTest{ |
| 641 "presentation": AggregateTest{ |
| 642 "presentation": &AggregateTestLe
af{ |
| 643 Results: []ResultSummar
y{{10, "A"}, {10, "P"}}, |
| 644 Runtimes: []RuntimeSumma
ry{{2, 2}}, |
| 645 }, |
| 646 "readme": &AggregateTestLeaf{ |
| 647 Results: []ResultSummar
y{{10, "B"}, {20, "Q"}}, |
| 648 Runtimes: []RuntimeSumma
ry{{1, 2}, {3, 4}}, |
| 649 }, |
| 650 }, |
| 651 "io": AggregateTest{ |
| 652 "serial": &AggregateTestLeaf{ |
| 653 Results: []ResultSummar
y{{1, "N"}, {40, "S"}}, |
| 654 Runtimes: []RuntimeSumma
ry{{1, 0}, {8, 9}}, |
| 655 }, |
| 656 "analog": &AggregateTestLeaf{ |
| 657 Results: []ResultSummar
y{{10, "D"}}, |
| 658 Runtimes: []RuntimeSumma
ry{{3, 0}}, |
| 659 }, |
| 660 "parallel": AggregateTest{ |
| 661 "duplex": &AggregateTest
Leaf{ |
| 662 Results: []Resu
ltSummary{{10, "E"}}, |
| 663 Runtimes: []Runt
imeSummary{{3, 5}}, |
| 664 }, |
| 665 "simplex": &AggregateTes
tLeaf{ |
| 666 Results: []Resu
ltSummary{{60, "H"}}, |
| 667 Runtimes: []Runt
imeSummary{{30, 50}}, |
| 668 }, |
| 669 }, |
| 670 }, |
| 671 "hello": AggregateTest{ |
| 672 "world": AggregateTest{ |
| 673 "main": &AggregateTestLe
af{ |
| 674 Results: []Resu
ltSummary{{1, "N"}, {50, "T"}}, |
| 675 Runtimes: []Runt
imeSummary{{1, 0}, {10, 11}}, |
| 676 }, |
| 677 }, |
| 678 }, |
| 679 "cool": AggregateTest{ |
| 680 "world": AggregateTest{ |
| 681 "main": &AggregateTestLe
af{ |
| 682 Results: []Resu
ltSummary{{10, "F"}}, |
| 683 Runtimes: []Runt
imeSummary{{3, 6}}, |
| 684 }, |
| 685 }, |
| 686 }, |
| 687 } |
| 688 So(at.Merge(x), ShouldBeNil) |
| 689 So(at, ShouldResemble, expected) |
| 690 }) |
| 691 }) |
| 692 |
| 693 Convey("AggregateResult", func() { |
| 694 Convey("Basic merge with same test names", func() { |
| 695 ag := &AggregateResult{ |
| 696 Version: ResultsVersion, |
| 697 Builder: "foo_builder", |
| 698 BuilderInfo: &BuilderInfo{ |
| 699 SecondsEpoch: []int64{1, 2}, |
| 700 BlinkRevs: []Number{3, 4}, |
| 701 BuildNumbers: []Number{5, 6}, |
| 702 ChromeRevs: []string{"a", "1"}
, |
| 703 Tests: AggregateTest{ |
| 704 "foo": &AggregateTestLea
f{ |
| 705 Results: []Resu
ltSummary{{1, "A"}, {3, "B"}}, |
| 706 Runtimes: []Runt
imeSummary{{1, 2}, {3, 4}}, |
| 707 }, |
| 708 }, |
| 709 FailureMap: FailureLongNames, |
| 710 FailuresByType: map[string][]int
{ |
| 711 "AUDIO": {1, 2, 3, 4}, |
| 712 "CRASH": {100, 200}, |
| 713 "LEAKY": {0}, |
| 714 "PASS": {5}, |
| 715 }, |
| 716 }, |
| 717 } |
| 718 x := &AggregateResult{ |
| 719 Builder: "foo_builder", |
| 720 BuilderInfo: &BuilderInfo{ |
| 721 SecondsEpoch: []int64{10, 20}, |
| 722 BlinkRevs: []Number{30, 40}, |
| 723 BuildNumbers: []Number{50, 60}, |
| 724 ChromeRevs: []string{"b", "c"}
, |
| 725 Tests: AggregateTest{ |
| 726 "foo": &AggregateTestLea
f{ |
| 727 Results: []Resu
ltSummary{{1, "C"}, {3, "D"}}, |
| 728 Runtimes: []Runt
imeSummary{{10, 20}, {30, 40}}, |
| 729 }, |
| 730 }, |
| 731 FailureMap: map[string]string{}, |
| 732 FailuresByType: map[string][]int
{ |
| 733 "AUDIO": {80, 90}, |
| 734 "CRASH": {110, 220}, |
| 735 "FAILURE": {42}, |
| 736 "LEAKY": {2}, |
| 737 }, |
| 738 }, |
| 739 } |
| 740 expected := &AggregateResult{ |
| 741 Version: ResultsVersion, |
| 742 Builder: "foo_builder", |
| 743 BuilderInfo: &BuilderInfo{ |
| 744 SecondsEpoch: []int64{10, 20, 1,
2}, |
| 745 BlinkRevs: []Number{30, 40, 3
, 4}, |
| 746 BuildNumbers: []Number{50, 60, 5
, 6}, |
| 747 ChromeRevs: []string{"b", "c",
"a", "1"}, |
| 748 Tests: AggregateTest{ |
| 749 "foo": &AggregateTestLea
f{ |
| 750 Results: []Resu
ltSummary{{3, "D"}, {1, "C"}, {1, "A"}, {3, "B"}}, |
| 751 Runtimes: []Runt
imeSummary{{30, 40}, {10, 20}, {1, 2}, {3, 4}}, |
| 752 }, |
| 753 }, |
| 754 FailureMap: FailureLongNames, |
| 755 FailuresByType: map[string][]int
{ |
| 756 "AUDIO": {80, 90, 1, 2
, 3, 4}, |
| 757 "CRASH": {110, 220, 10
0, 200}, |
| 758 "FAILURE": {42}, |
| 759 "PASS": {5}, |
| 760 "LEAKY": {2, 0}, |
| 761 }, |
| 762 }, |
| 763 } |
| 764 So(ag.Merge(x), ShouldBeNil) |
| 765 So(ag, ShouldResemble, expected) |
| 766 }) |
| 767 }) |
| 768 }) |
| 769 } |
| 770 |
| 771 // TestMergeAndTrim tests are ported from "model/test/jsonresults_test.py" |
| 772 // in the Python implementation. |
| 773 func TestMergeAndTrim(t *testing.T) { |
| 774 t.Parallel() |
| 775 |
| 776 Convey("Test Merge/Trim", t, func() { |
| 777 Convey("empty aggregated results", func() { |
| 778 aggr := &AggregateResult{} |
| 779 x := &AggregateResult{ |
| 780 BuilderInfo: &BuilderInfo{ |
| 781 BuildNumbers: []Number{2, 1}, |
| 782 Tests: AggregateTest{ |
| 783 "001.html": &AggregateTestLeaf{ |
| 784 Results: []ResultSummar
y{{200, "F"}}, |
| 785 Runtimes: []RuntimeSumma
ry{{200, 0}}, |
| 786 }, |
| 787 }, |
| 788 }, |
| 789 } |
| 790 expected := &AggregateResult{ |
| 791 Version: ResultsVersion, |
| 792 BuilderInfo: &BuilderInfo{ |
| 793 FailureMap: FailureLongNames, |
| 794 BuildNumbers: []Number{2, 1}, |
| 795 Tests: AggregateTest{ |
| 796 "001.html": &AggregateTestLeaf{ |
| 797 Results: []ResultSummar
y{{200, "F"}}, |
| 798 Runtimes: []RuntimeSumma
ry{{200, 0}}, |
| 799 }, |
| 800 }, |
| 801 }, |
| 802 } |
| 803 So(aggr.Merge(x), ShouldBeNil) |
| 804 So(aggr, ShouldResemble, expected) |
| 805 }) |
| 806 |
| 807 Convey("duplicate build number", func() { |
| 808 aggr := &AggregateResult{ |
| 809 BuilderInfo: &BuilderInfo{ |
| 810 BuildNumbers: []Number{2, 1}, |
| 811 Tests: AggregateTest{ |
| 812 "001.html": &AggregateTestLeaf{ |
| 813 Results: []ResultSummar
y{{100, "F"}}, |
| 814 Runtimes: []RuntimeSumma
ry{{100, 0}}, |
| 815 }, |
| 816 }, |
| 817 }, |
| 818 } |
| 819 x := &AggregateResult{ |
| 820 BuilderInfo: &BuilderInfo{ |
| 821 BuildNumbers: []Number{2}, |
| 822 Tests: AggregateTest{ |
| 823 "001.html": &AggregateTestLeaf{ |
| 824 Results: []ResultSummar
y{{1, "F"}}, |
| 825 Runtimes: []RuntimeSumma
ry{{1, 0}}, |
| 826 }, |
| 827 }, |
| 828 }, |
| 829 } |
| 830 So(aggr.Merge(x), ShouldEqual, ErrBuildNumberConflict) |
| 831 }) |
| 832 |
| 833 Convey("incremental single test single run same result", func()
{ |
| 834 aggr := &AggregateResult{ |
| 835 BuilderInfo: &BuilderInfo{ |
| 836 BuildNumbers: []Number{2, 1}, |
| 837 Tests: AggregateTest{ |
| 838 "001.html": &AggregateTestLeaf{ |
| 839 Results: []ResultSummar
y{{200, "F"}}, |
| 840 Runtimes: []RuntimeSumma
ry{{200, 0}}, |
| 841 }, |
| 842 }, |
| 843 }, |
| 844 } |
| 845 x := &AggregateResult{ |
| 846 BuilderInfo: &BuilderInfo{ |
| 847 BuildNumbers: []Number{3}, |
| 848 Tests: AggregateTest{ |
| 849 "001.html": &AggregateTestLeaf{ |
| 850 Results: []ResultSummar
y{{1, "F"}}, |
| 851 Runtimes: []RuntimeSumma
ry{{1, 0}}, |
| 852 }, |
| 853 }, |
| 854 }, |
| 855 } |
| 856 expected := &AggregateResult{ |
| 857 Version: ResultsVersion, |
| 858 BuilderInfo: &BuilderInfo{ |
| 859 FailureMap: FailureLongNames, |
| 860 BuildNumbers: []Number{3, 2, 1}, |
| 861 Tests: AggregateTest{ |
| 862 "001.html": &AggregateTestLeaf{ |
| 863 Results: []ResultSummar
y{{201, "F"}}, |
| 864 Runtimes: []RuntimeSumma
ry{{201, 0}}, |
| 865 }, |
| 866 }, |
| 867 }, |
| 868 } |
| 869 So(aggr.Merge(x), ShouldBeNil) |
| 870 So(aggr, ShouldResemble, expected) |
| 871 }) |
| 872 |
| 873 Convey("single test single run different result", func() { |
| 874 aggr := &AggregateResult{ |
| 875 Version: ResultsVersion, |
| 876 BuilderInfo: &BuilderInfo{ |
| 877 BuildNumbers: []Number{2, 1}, |
| 878 Tests: AggregateTest{ |
| 879 "001.html": &AggregateTestLeaf{ |
| 880 Results: []ResultSummar
y{{200, "F"}}, |
| 881 Runtimes: []RuntimeSumma
ry{{200, 0}}, |
| 882 }, |
| 883 }, |
| 884 }, |
| 885 } |
| 886 x := &AggregateResult{ |
| 887 BuilderInfo: &BuilderInfo{ |
| 888 BuildNumbers: []Number{3}, |
| 889 Tests: AggregateTest{ |
| 890 "001.html": &AggregateTestLeaf{ |
| 891 Results: []ResultSummar
y{{1, "I"}}, |
| 892 Runtimes: []RuntimeSumma
ry{{1, 1}}, |
| 893 }, |
| 894 }, |
| 895 }, |
| 896 } |
| 897 expected := &AggregateResult{ |
| 898 Version: ResultsVersion, |
| 899 BuilderInfo: &BuilderInfo{ |
| 900 FailureMap: FailureLongNames, |
| 901 BuildNumbers: []Number{3, 2, 1}, |
| 902 Tests: AggregateTest{ |
| 903 "001.html": &AggregateTestLeaf{ |
| 904 Results: []ResultSummar
y{{1, "I"}, {200, "F"}}, |
| 905 Runtimes: []RuntimeSumma
ry{{1, 1}, {200, 0}}, |
| 906 }, |
| 907 }, |
| 908 }, |
| 909 } |
| 910 So(aggr.Merge(x), ShouldBeNil) |
| 911 So(aggr, ShouldResemble, expected) |
| 912 }) |
| 913 |
| 914 Convey("single test single run, result changed", func() { |
| 915 aggr := &AggregateResult{ |
| 916 BuilderInfo: &BuilderInfo{ |
| 917 BuildNumbers: []Number{2, 1}, |
| 918 Tests: AggregateTest{ |
| 919 "001.html": &AggregateTestLeaf{ |
| 920 Results: []ResultSummar
y{{200, "F"}, {10, "I"}}, |
| 921 Runtimes: []RuntimeSumma
ry{{200, 0}, {10, 1}}, |
| 922 }, |
| 923 }, |
| 924 }, |
| 925 } |
| 926 x := &AggregateResult{ |
| 927 BuilderInfo: &BuilderInfo{ |
| 928 BuildNumbers: []Number{3}, |
| 929 Tests: AggregateTest{ |
| 930 "001.html": &AggregateTestLeaf{ |
| 931 Results: []ResultSummar
y{{1, "I"}}, |
| 932 Runtimes: []RuntimeSumma
ry{{1, 1}}, |
| 933 }, |
| 934 }, |
| 935 }, |
| 936 } |
| 937 expected := &AggregateResult{ |
| 938 Version: ResultsVersion, |
| 939 BuilderInfo: &BuilderInfo{ |
| 940 FailureMap: FailureLongNames, |
| 941 BuildNumbers: []Number{3, 2, 1}, |
| 942 Tests: AggregateTest{ |
| 943 "001.html": &AggregateTestLeaf{ |
| 944 Results: []ResultSummar
y{{1, "I"}, {200, "F"}, {10, "I"}}, |
| 945 Runtimes: []RuntimeSumma
ry{{1, 1}, {200, 0}, {10, 1}}, |
| 946 }, |
| 947 }, |
| 948 }, |
| 949 } |
| 950 So(aggr.Merge(x), ShouldBeNil) |
| 951 So(aggr, ShouldResemble, expected) |
| 952 }) |
| 953 |
| 954 Convey("multiple tests single run", func() { |
| 955 aggr := &AggregateResult{ |
| 956 BuilderInfo: &BuilderInfo{ |
| 957 BuildNumbers: []Number{2, 1}, |
| 958 Tests: AggregateTest{ |
| 959 "001.html": &AggregateTestLeaf{ |
| 960 Results: []ResultSummar
y{{200, "F"}}, |
| 961 Runtimes: []RuntimeSumma
ry{{200, 0}}, |
| 962 }, |
| 963 "002.html": &AggregateTestLeaf{ |
| 964 Results: []ResultSummar
y{{100, "I"}}, |
| 965 Runtimes: []RuntimeSumma
ry{{100, 1}}, |
| 966 }, |
| 967 }, |
| 968 }, |
| 969 } |
| 970 x := &AggregateResult{ |
| 971 BuilderInfo: &BuilderInfo{ |
| 972 BuildNumbers: []Number{3}, |
| 973 Tests: AggregateTest{ |
| 974 "001.html": &AggregateTestLeaf{ |
| 975 Results: []ResultSummar
y{{1, "F"}}, |
| 976 Runtimes: []RuntimeSumma
ry{{1, 0}}, |
| 977 }, |
| 978 "002.html": &AggregateTestLeaf{ |
| 979 Results: []ResultSummar
y{{1, "I"}}, |
| 980 Runtimes: []RuntimeSumma
ry{{1, 1}}, |
| 981 }, |
| 982 }, |
| 983 }, |
| 984 } |
| 985 expected := &AggregateResult{ |
| 986 Version: ResultsVersion, |
| 987 BuilderInfo: &BuilderInfo{ |
| 988 FailureMap: FailureLongNames, |
| 989 BuildNumbers: []Number{3, 2, 1}, |
| 990 Tests: AggregateTest{ |
| 991 "001.html": &AggregateTestLeaf{ |
| 992 Results: []ResultSummar
y{{201, "F"}}, |
| 993 Runtimes: []RuntimeSumma
ry{{201, 0}}, |
| 994 }, |
| 995 "002.html": &AggregateTestLeaf{ |
| 996 Results: []ResultSummar
y{{101, "I"}}, |
| 997 Runtimes: []RuntimeSumma
ry{{101, 1}}, |
| 998 }, |
| 999 }, |
| 1000 }, |
| 1001 } |
| 1002 So(aggr.Merge(x), ShouldBeNil) |
| 1003 So(aggr, ShouldResemble, expected) |
| 1004 }) |
| 1005 |
| 1006 Convey("multiple tests single run no result", func() { |
| 1007 aggr := &AggregateResult{ |
| 1008 BuilderInfo: &BuilderInfo{ |
| 1009 BuildNumbers: []Number{2, 1}, |
| 1010 Tests: AggregateTest{ |
| 1011 "001.html": &AggregateTestLeaf{ |
| 1012 Results: []ResultSummar
y{{200, "F"}}, |
| 1013 Runtimes: []RuntimeSumma
ry{{200, 0}}, |
| 1014 }, |
| 1015 "002.html": &AggregateTestLeaf{ |
| 1016 Results: []ResultSummar
y{{100, "I"}}, |
| 1017 Runtimes: []RuntimeSumma
ry{{100, 1}}, |
| 1018 }, |
| 1019 }, |
| 1020 }, |
| 1021 } |
| 1022 x := &AggregateResult{ |
| 1023 BuilderInfo: &BuilderInfo{ |
| 1024 BuildNumbers: []Number{3}, |
| 1025 Tests: AggregateTest{ |
| 1026 "002.html": &AggregateTestLeaf{ |
| 1027 Results: []ResultSummar
y{{1, "I"}}, |
| 1028 Runtimes: []RuntimeSumma
ry{{1, 1}}, |
| 1029 }, |
| 1030 }, |
| 1031 }, |
| 1032 } |
| 1033 expected := &AggregateResult{ |
| 1034 Version: ResultsVersion, |
| 1035 BuilderInfo: &BuilderInfo{ |
| 1036 FailureMap: FailureLongNames, |
| 1037 BuildNumbers: []Number{3, 2, 1}, |
| 1038 Tests: AggregateTest{ |
| 1039 "001.html": &AggregateTestLeaf{ |
| 1040 Results: []ResultSummar
y{{1, "N"}, {200, "F"}}, |
| 1041 Runtimes: []RuntimeSumma
ry{{201, 0}}, |
| 1042 }, |
| 1043 "002.html": &AggregateTestLeaf{ |
| 1044 Results: []ResultSummar
y{{101, "I"}}, |
| 1045 Runtimes: []RuntimeSumma
ry{{101, 1}}, |
| 1046 }, |
| 1047 }, |
| 1048 }, |
| 1049 } |
| 1050 So(aggr.Merge(x), ShouldBeNil) |
| 1051 So(aggr, ShouldResemble, expected) |
| 1052 }) |
| 1053 |
| 1054 Convey("single test multiple runs", func() { |
| 1055 aggr := &AggregateResult{ |
| 1056 BuilderInfo: &BuilderInfo{ |
| 1057 BuildNumbers: []Number{2, 1}, |
| 1058 Tests: AggregateTest{ |
| 1059 "001.html": &AggregateTestLeaf{ |
| 1060 Results: []ResultSummar
y{{200, "F"}}, |
| 1061 Runtimes: []RuntimeSumma
ry{{200, 0}}, |
| 1062 }, |
| 1063 }, |
| 1064 }, |
| 1065 } |
| 1066 x := &AggregateResult{ |
| 1067 BuilderInfo: &BuilderInfo{ |
| 1068 BuildNumbers: []Number{4, 3}, |
| 1069 Tests: AggregateTest{ |
| 1070 "001.html": &AggregateTestLeaf{ |
| 1071 Results: []ResultSummar
y{{2, "I"}, {1, "Q"}}, |
| 1072 Runtimes: []RuntimeSumma
ry{{3, 2}}, |
| 1073 }, |
| 1074 }, |
| 1075 }, |
| 1076 } |
| 1077 expected := &AggregateResult{ |
| 1078 Version: ResultsVersion, |
| 1079 BuilderInfo: &BuilderInfo{ |
| 1080 FailureMap: FailureLongNames, |
| 1081 BuildNumbers: []Number{4, 3, 2, 1}, |
| 1082 Tests: AggregateTest{ |
| 1083 "001.html": &AggregateTestLeaf{ |
| 1084 Results: []ResultSummar
y{{1, "Q"}, {2, "I"}, {200, "F"}}, |
| 1085 Runtimes: []RuntimeSumma
ry{{3, 2}, {200, 0}}, |
| 1086 }, |
| 1087 }, |
| 1088 }, |
| 1089 } |
| 1090 So(aggr.Merge(x), ShouldBeNil) |
| 1091 So(aggr, ShouldResemble, expected) |
| 1092 }) |
| 1093 |
| 1094 Convey("multiple tests multiple runs", func() { |
| 1095 aggr := &AggregateResult{ |
| 1096 BuilderInfo: &BuilderInfo{ |
| 1097 BuildNumbers: []Number{2, 1}, |
| 1098 Tests: AggregateTest{ |
| 1099 "001.html": &AggregateTestLeaf{ |
| 1100 Results: []ResultSummar
y{{200, "F"}}, |
| 1101 Runtimes: []RuntimeSumma
ry{{200, 0}}, |
| 1102 }, |
| 1103 "002.html": &AggregateTestLeaf{ |
| 1104 Results: []ResultSummar
y{{10, "Z"}}, |
| 1105 Runtimes: []RuntimeSumma
ry{{10, 0}}, |
| 1106 }, |
| 1107 }, |
| 1108 }, |
| 1109 } |
| 1110 x := &AggregateResult{ |
| 1111 BuilderInfo: &BuilderInfo{ |
| 1112 BuildNumbers: []Number{4, 3}, |
| 1113 Tests: AggregateTest{ |
| 1114 "001.html": &AggregateTestLeaf{ |
| 1115 Results: []ResultSummar
y{{2, "I"}}, |
| 1116 Runtimes: []RuntimeSumma
ry{{2, 2}}, |
| 1117 }, |
| 1118 "002.html": &AggregateTestLeaf{ |
| 1119 Results: []ResultSummar
y{{1, "C"}}, |
| 1120 Runtimes: []RuntimeSumma
ry{{1, 1}}, |
| 1121 }, |
| 1122 }, |
| 1123 }, |
| 1124 } |
| 1125 expected := &AggregateResult{ |
| 1126 Version: ResultsVersion, |
| 1127 BuilderInfo: &BuilderInfo{ |
| 1128 FailureMap: FailureLongNames, |
| 1129 BuildNumbers: []Number{4, 3, 2, 1}, |
| 1130 Tests: AggregateTest{ |
| 1131 "001.html": &AggregateTestLeaf{ |
| 1132 Results: []ResultSummar
y{{2, "I"}, {200, "F"}}, |
| 1133 Runtimes: []RuntimeSumma
ry{{2, 2}, {200, 0}}, |
| 1134 }, |
| 1135 "002.html": &AggregateTestLeaf{ |
| 1136 Results: []ResultSummar
y{{1, "C"}, {10, "Z"}}, |
| 1137 Runtimes: []RuntimeSumma
ry{{1, 1}, {10, 0}}, |
| 1138 }, |
| 1139 }, |
| 1140 }, |
| 1141 } |
| 1142 So(aggr.Merge(x), ShouldBeNil) |
| 1143 So(aggr, ShouldResemble, expected) |
| 1144 }) |
| 1145 |
| 1146 Convey("incremental result older build", func() { |
| 1147 aggr := &AggregateResult{ |
| 1148 BuilderInfo: &BuilderInfo{ |
| 1149 BuildNumbers: []Number{3, 1}, |
| 1150 Tests: AggregateTest{ |
| 1151 "001.html": &AggregateTestLeaf{ |
| 1152 Results: []ResultSummar
y{{5, "F"}}, |
| 1153 Runtimes: []RuntimeSumma
ry{{5, 0}}, |
| 1154 }, |
| 1155 }, |
| 1156 }, |
| 1157 } |
| 1158 x := &AggregateResult{ |
| 1159 BuilderInfo: &BuilderInfo{ |
| 1160 BuildNumbers: []Number{2}, |
| 1161 Tests: AggregateTest{ |
| 1162 "001.html": &AggregateTestLeaf{ |
| 1163 Results: []ResultSummar
y{{1, "F"}}, |
| 1164 Runtimes: []RuntimeSumma
ry{{1, 0}}, |
| 1165 }, |
| 1166 }, |
| 1167 }, |
| 1168 } |
| 1169 expected := &AggregateResult{ |
| 1170 Version: ResultsVersion, |
| 1171 BuilderInfo: &BuilderInfo{ |
| 1172 FailureMap: FailureLongNames, |
| 1173 BuildNumbers: []Number{2, 3, 1}, |
| 1174 Tests: AggregateTest{ |
| 1175 "001.html": &AggregateTestLeaf{ |
| 1176 Results: []ResultSummar
y{{6, "F"}}, |
| 1177 Runtimes: []RuntimeSumma
ry{{6, 0}}, |
| 1178 }, |
| 1179 }, |
| 1180 }, |
| 1181 } |
| 1182 So(aggr.Merge(x), ShouldBeNil) |
| 1183 So(aggr, ShouldResemble, expected) |
| 1184 }) |
| 1185 |
| 1186 Convey("incremental result same build", func() { |
| 1187 aggr := &AggregateResult{ |
| 1188 BuilderInfo: &BuilderInfo{ |
| 1189 BuildNumbers: []Number{2, 1}, |
| 1190 Tests: AggregateTest{ |
| 1191 "001.html": &AggregateTestLeaf{ |
| 1192 Results: []ResultSummar
y{{5, "F"}}, |
| 1193 Runtimes: []RuntimeSumma
ry{{5, 0}}, |
| 1194 }, |
| 1195 }, |
| 1196 }, |
| 1197 } |
| 1198 x := &AggregateResult{ |
| 1199 BuilderInfo: &BuilderInfo{ |
| 1200 BuildNumbers: []Number{3, 2}, |
| 1201 Tests: AggregateTest{ |
| 1202 "001.html": &AggregateTestLeaf{ |
| 1203 Results: []ResultSummar
y{{2, "F"}}, |
| 1204 Runtimes: []RuntimeSumma
ry{{2, 0}}, |
| 1205 }, |
| 1206 }, |
| 1207 }, |
| 1208 } |
| 1209 expected := &AggregateResult{ |
| 1210 Version: ResultsVersion, |
| 1211 BuilderInfo: &BuilderInfo{ |
| 1212 FailureMap: FailureLongNames, |
| 1213 BuildNumbers: []Number{3, 2, 2, 1}, |
| 1214 Tests: AggregateTest{ |
| 1215 "001.html": &AggregateTestLeaf{ |
| 1216 Results: []ResultSummar
y{{7, "F"}}, |
| 1217 Runtimes: []RuntimeSumma
ry{{7, 0}}, |
| 1218 }, |
| 1219 }, |
| 1220 }, |
| 1221 } |
| 1222 So(aggr.Merge(x), ShouldBeNil) |
| 1223 So(aggr, ShouldResemble, expected) |
| 1224 }) |
| 1225 |
| 1226 Convey("remove new test", func() { |
| 1227 aggr := &AggregateResult{ |
| 1228 BuilderInfo: &BuilderInfo{ |
| 1229 BuildNumbers: []Number{2, 1}, |
| 1230 Tests: AggregateTest{ |
| 1231 "001.html": &AggregateTestLeaf{ |
| 1232 Results: []ResultSummar
y{{199, "F"}}, |
| 1233 Runtimes: []RuntimeSumma
ry{{199, 0}}, |
| 1234 }, |
| 1235 }, |
| 1236 }, |
| 1237 } |
| 1238 x := &AggregateResult{ |
| 1239 BuilderInfo: &BuilderInfo{ |
| 1240 BuildNumbers: []Number{3}, |
| 1241 Tests: AggregateTest{ |
| 1242 "001.html": &AggregateTestLeaf{ |
| 1243 Results: []ResultSummar
y{{1, "F"}}, |
| 1244 Runtimes: []RuntimeSumma
ry{{1, 0}}, |
| 1245 }, |
| 1246 "002.html": &AggregateTestLeaf{ |
| 1247 Results: []ResultSummar
y{{1, "P"}}, |
| 1248 Runtimes: []RuntimeSumma
ry{{1, 0}}, |
| 1249 }, |
| 1250 "notrun.html": &AggregateTestLea
f{ |
| 1251 Results: []ResultSummar
y{{1, "Y"}}, |
| 1252 Runtimes: []RuntimeSumma
ry{{1, 0}}, |
| 1253 }, |
| 1254 "003.html": &AggregateTestLeaf{ |
| 1255 Results: []ResultSummar
y{{1, "N"}}, |
| 1256 Runtimes: []RuntimeSumma
ry{{1, 0}}, |
| 1257 }, |
| 1258 }, |
| 1259 }, |
| 1260 } |
| 1261 expected := &AggregateResult{ |
| 1262 Version: ResultsVersion, |
| 1263 BuilderInfo: &BuilderInfo{ |
| 1264 FailureMap: FailureLongNames, |
| 1265 BuildNumbers: []Number{3, 2, 1}, |
| 1266 Tests: AggregateTest{ |
| 1267 "001.html": &AggregateTestLeaf{ |
| 1268 Results: []ResultSummar
y{{200, "F"}}, |
| 1269 Runtimes: []RuntimeSumma
ry{{200, 0}}, |
| 1270 }, |
| 1271 }, |
| 1272 }, |
| 1273 } |
| 1274 So(aggr.Merge(x), ShouldBeNil) |
| 1275 So(aggr.Trim(200), ShouldBeNil) |
| 1276 So(aggr, ShouldResemble, expected) |
| 1277 }) |
| 1278 |
| 1279 Convey("remove test", func() { |
| 1280 aggr := &AggregateResult{ |
| 1281 BuilderInfo: &BuilderInfo{ |
| 1282 BuildNumbers: []Number{2, 1}, |
| 1283 Tests: AggregateTest{ |
| 1284 "directory": AggregateTest{ |
| 1285 "directory": AggregateTe
st{ |
| 1286 "001.html": &Agg
regateTestLeaf{ |
| 1287 Results:
[]ResultSummary{{200, "P"}}, |
| 1288 Runtimes
: []RuntimeSummary{{200, 0}}, |
| 1289 }, |
| 1290 }, |
| 1291 }, |
| 1292 "002.html": &AggregateTestLeaf{ |
| 1293 Results: []ResultSummar
y{{10, "F"}}, |
| 1294 Runtimes: []RuntimeSumma
ry{{10, 0}}, |
| 1295 }, |
| 1296 "003.html": &AggregateTestLeaf{ |
| 1297 Results: []ResultSummar
y{{190, "P"}, {9, "N"}, {1, "F"}}, |
| 1298 Runtimes: []RuntimeSumma
ry{{200, 0}}, |
| 1299 }, |
| 1300 }, |
| 1301 }, |
| 1302 } |
| 1303 x := &AggregateResult{ |
| 1304 BuilderInfo: &BuilderInfo{ |
| 1305 BuildNumbers: []Number{3}, |
| 1306 Tests: AggregateTest{ |
| 1307 "directory": AggregateTest{ |
| 1308 "directory": AggregateTe
st{ |
| 1309 "001.html": &Agg
regateTestLeaf{ |
| 1310 Results:
[]ResultSummary{{1, "P"}}, |
| 1311 Runtimes
: []RuntimeSummary{{1, 0}}, |
| 1312 }, |
| 1313 }, |
| 1314 }, |
| 1315 "002.html": &AggregateTestLeaf{ |
| 1316 Results: []ResultSummar
y{{1, "P"}}, |
| 1317 Runtimes: []RuntimeSumma
ry{{1, 0}}, |
| 1318 }, |
| 1319 "003.html": &AggregateTestLeaf{ |
| 1320 Results: []ResultSummar
y{{1, "P"}}, |
| 1321 Runtimes: []RuntimeSumma
ry{{1, 0}}, |
| 1322 }, |
| 1323 }, |
| 1324 }, |
| 1325 } |
| 1326 expected := &AggregateResult{ |
| 1327 Version: ResultsVersion, |
| 1328 BuilderInfo: &BuilderInfo{ |
| 1329 FailureMap: FailureLongNames, |
| 1330 BuildNumbers: []Number{3, 2, 1}, |
| 1331 Tests: AggregateTest{ |
| 1332 "002.html": &AggregateTestLeaf{ |
| 1333 Results: []ResultSummar
y{{1, "P"}, {10, "F"}}, |
| 1334 Runtimes: []RuntimeSumma
ry{{11, 0}}, |
| 1335 }, |
| 1336 }, |
| 1337 }, |
| 1338 } |
| 1339 So(aggr.Merge(x), ShouldBeNil) |
| 1340 So(aggr.Trim(200), ShouldBeNil) |
| 1341 So(aggr, ShouldResemble, expected) |
| 1342 }) |
| 1343 |
| 1344 Convey("updates expected", func() { |
| 1345 aggr := &AggregateResult{ |
| 1346 BuilderInfo: &BuilderInfo{ |
| 1347 BuildNumbers: []Number{2, 1}, |
| 1348 Tests: AggregateTest{ |
| 1349 "directory": AggregateTest{ |
| 1350 "directory": AggregateTe
st{ |
| 1351 "001.html": &Agg
regateTestLeaf{ |
| 1352 Expected
: []string{"FAIL"}, |
| 1353 Results:
[]ResultSummary{{200, "P"}}, |
| 1354 Runtimes
: []RuntimeSummary{{200, 0}}, |
| 1355 }, |
| 1356 }, |
| 1357 }, |
| 1358 "002.html": &AggregateTestLeaf{ |
| 1359 Bugs: []string{"crbu
g.com/1234"}, |
| 1360 Expected: []string{"FAIL
"}, |
| 1361 Results: []ResultSummar
y{{10, "F"}}, |
| 1362 Runtimes: []RuntimeSumma
ry{{10, 0}}, |
| 1363 }, |
| 1364 "003.html": &AggregateTestLeaf{ |
| 1365 Expected: []string{"FAIL
"}, |
| 1366 Results: []ResultSummar
y{{190, "P"}, {9, "N"}, {1, "F"}}, |
| 1367 Runtimes: []RuntimeSumma
ry{{200, 0}}, |
| 1368 }, |
| 1369 "004.html": &AggregateTestLeaf{ |
| 1370 Results: []ResultSummar
y{{199, "P"}, {1, "F"}}, |
| 1371 Runtimes: []RuntimeSumma
ry{{200, 0}}, |
| 1372 }, |
| 1373 }, |
| 1374 }, |
| 1375 } |
| 1376 x := &AggregateResult{ |
| 1377 BuilderInfo: &BuilderInfo{ |
| 1378 BuildNumbers: []Number{3}, |
| 1379 Tests: AggregateTest{ |
| 1380 "002.html": &AggregateTestLeaf{ |
| 1381 Expected: []string{"PASS
"}, |
| 1382 Results: []ResultSummar
y{{1, "P"}}, |
| 1383 Runtimes: []RuntimeSumma
ry{{1, 0}}, |
| 1384 }, |
| 1385 "003.html": &AggregateTestLeaf{ |
| 1386 Expected: []string{"TIME
OUT"}, |
| 1387 Results: []ResultSummar
y{{1, "P"}}, |
| 1388 Runtimes: []RuntimeSumma
ry{{1, 0}}, |
| 1389 }, |
| 1390 "004.html": &AggregateTestLeaf{ |
| 1391 Bugs: []string{"crbu
g.com/1234"}, |
| 1392 Results: []ResultSummar
y{{1, "P"}}, |
| 1393 Runtimes: []RuntimeSumma
ry{{1, 0}}, |
| 1394 }, |
| 1395 }, |
| 1396 }, |
| 1397 } |
| 1398 expected := &AggregateResult{ |
| 1399 Version: ResultsVersion, |
| 1400 BuilderInfo: &BuilderInfo{ |
| 1401 FailureMap: FailureLongNames, |
| 1402 BuildNumbers: []Number{3, 2, 1}, |
| 1403 Tests: AggregateTest{ |
| 1404 "002.html": &AggregateTestLeaf{ |
| 1405 Results: []ResultSummar
y{{1, "P"}, {10, "F"}}, |
| 1406 Runtimes: []RuntimeSumma
ry{{11, 0}}, |
| 1407 }, |
| 1408 "003.html": &AggregateTestLeaf{ |
| 1409 Expected: []string{"TIME
OUT"}, |
| 1410 Results: []ResultSummar
y{{191, "P"}, {9, "N"}}, |
| 1411 Runtimes: []RuntimeSumma
ry{{200, 0}}, |
| 1412 }, |
| 1413 "004.html": &AggregateTestLeaf{ |
| 1414 Bugs: []string{"crbu
g.com/1234"}, |
| 1415 Results: []ResultSummar
y{{200, "P"}}, |
| 1416 Runtimes: []RuntimeSumma
ry{{200, 0}}, |
| 1417 }, |
| 1418 }, |
| 1419 }, |
| 1420 } |
| 1421 So(aggr.Merge(x), ShouldBeNil) |
| 1422 So(aggr.Trim(200), ShouldBeNil) |
| 1423 So(aggr, ShouldResemble, expected) |
| 1424 }) |
| 1425 |
| 1426 Convey("keep test with all pass but slow time", func() { |
| 1427 aggr := &AggregateResult{ |
| 1428 BuilderInfo: &BuilderInfo{ |
| 1429 BuildNumbers: []Number{2, 1}, |
| 1430 Tests: AggregateTest{ |
| 1431 "001.html": &AggregateTestLeaf{ |
| 1432 Results: []ResultSummar
y{{200, "P"}}, |
| 1433 Runtimes: []RuntimeSumma
ry{{200, runtimeThresholdNormal}}, |
| 1434 }, |
| 1435 "002.html": &AggregateTestLeaf{ |
| 1436 Results: []ResultSummar
y{{10, "F"}}, |
| 1437 Runtimes: []RuntimeSumma
ry{{10, 0}}, |
| 1438 }, |
| 1439 }, |
| 1440 }, |
| 1441 } |
| 1442 x := &AggregateResult{ |
| 1443 BuilderInfo: &BuilderInfo{ |
| 1444 BuildNumbers: []Number{3}, |
| 1445 Tests: AggregateTest{ |
| 1446 "001.html": &AggregateTestLeaf{ |
| 1447 Results: []ResultSummar
y{{1, "P"}}, |
| 1448 Runtimes: []RuntimeSumma
ry{{1, 1}}, |
| 1449 }, |
| 1450 "002.html": &AggregateTestLeaf{ |
| 1451 Results: []ResultSummar
y{{1, "P"}}, |
| 1452 Runtimes: []RuntimeSumma
ry{{1, 0}}, |
| 1453 }, |
| 1454 }, |
| 1455 }, |
| 1456 } |
| 1457 expected := &AggregateResult{ |
| 1458 Version: ResultsVersion, |
| 1459 BuilderInfo: &BuilderInfo{ |
| 1460 FailureMap: FailureLongNames, |
| 1461 BuildNumbers: []Number{3, 2, 1}, |
| 1462 Tests: AggregateTest{ |
| 1463 "001.html": &AggregateTestLeaf{ |
| 1464 Results: []ResultSummar
y{{201, "P"}}, |
| 1465 Runtimes: []RuntimeSumma
ry{{1, 1}, {200, runtimeThresholdNormal}}, |
| 1466 }, |
| 1467 "002.html": &AggregateTestLeaf{ |
| 1468 Results: []ResultSummar
y{{1, "P"}, {10, "F"}}, |
| 1469 Runtimes: []RuntimeSumma
ry{{11, 0}}, |
| 1470 }, |
| 1471 }, |
| 1472 }, |
| 1473 } |
| 1474 So(aggr.Merge(x), ShouldBeNil) |
| 1475 So(aggr, ShouldResemble, expected) |
| 1476 }) |
| 1477 |
| 1478 Convey("pruning slow tests for debug builders", func() { |
| 1479 aggr := &AggregateResult{ |
| 1480 Builder: "MockBuilder(dbg)", |
| 1481 BuilderInfo: &BuilderInfo{ |
| 1482 BuildNumbers: []Number{2, 1}, |
| 1483 Tests: AggregateTest{ |
| 1484 "001.html": &AggregateTestLeaf{ |
| 1485 Results: []ResultSummar
y{{200, "P"}}, |
| 1486 Runtimes: []RuntimeSumma
ry{{200, runtimeThresholdDebug}}, |
| 1487 }, |
| 1488 "002.html": &AggregateTestLeaf{ |
| 1489 Results: []ResultSummar
y{{10, "F"}}, |
| 1490 Runtimes: []RuntimeSumma
ry{{10, 0}}, |
| 1491 }, |
| 1492 }, |
| 1493 }, |
| 1494 } |
| 1495 x := &AggregateResult{ |
| 1496 Builder: "MockBuilder(dbg)", |
| 1497 BuilderInfo: &BuilderInfo{ |
| 1498 BuildNumbers: []Number{3}, |
| 1499 Tests: AggregateTest{ |
| 1500 "001.html": &AggregateTestLeaf{ |
| 1501 Results: []ResultSummar
y{{1, "P"}}, |
| 1502 Runtimes: []RuntimeSumma
ry{{1, 1}}, |
| 1503 }, |
| 1504 "002.html": &AggregateTestLeaf{ |
| 1505 Results: []ResultSummar
y{{1, "P"}}, |
| 1506 Runtimes: []RuntimeSumma
ry{{1, 0}}, |
| 1507 }, |
| 1508 "003.html": &AggregateTestLeaf{ |
| 1509 Results: []ResultSummar
y{{1, "P"}}, |
| 1510 Runtimes: []RuntimeSumma
ry{{1, 3}}, |
| 1511 }, |
| 1512 }, |
| 1513 }, |
| 1514 } |
| 1515 expected := &AggregateResult{ |
| 1516 Version: ResultsVersion, |
| 1517 Builder: "MockBuilder(dbg)", |
| 1518 BuilderInfo: &BuilderInfo{ |
| 1519 FailureMap: FailureLongNames, |
| 1520 BuildNumbers: []Number{3, 2, 1}, |
| 1521 Tests: AggregateTest{ |
| 1522 "001.html": &AggregateTestLeaf{ |
| 1523 Results: []ResultSummar
y{{201, "P"}}, |
| 1524 Runtimes: []RuntimeSumma
ry{{1, 1}, {200, runtimeThresholdDebug}}, |
| 1525 }, |
| 1526 "002.html": &AggregateTestLeaf{ |
| 1527 Results: []ResultSummar
y{{1, "P"}, {10, "F"}}, |
| 1528 Runtimes: []RuntimeSumma
ry{{11, 0}}, |
| 1529 }, |
| 1530 }, |
| 1531 }, |
| 1532 } |
| 1533 So(aggr.Merge(x), ShouldBeNil) |
| 1534 So(aggr.Trim(ResultsSize), ShouldBeNil) |
| 1535 So(aggr, ShouldResemble, expected) |
| 1536 }) |
| 1537 |
| 1538 Convey("prune extra results", func() { |
| 1539 size := ResultsSize |
| 1540 aggr := &AggregateResult{ |
| 1541 BuilderInfo: &BuilderInfo{ |
| 1542 BuildNumbers: []Number{2, 1}, |
| 1543 Tests: AggregateTest{ |
| 1544 "001.html": &AggregateTestLeaf{ |
| 1545 Results: []ResultSummar
y{{size, "F"}, {1, "I"}}, |
| 1546 Runtimes: []RuntimeSumma
ry{{size, 0}, {1, 1}}, |
| 1547 }, |
| 1548 }, |
| 1549 }, |
| 1550 } |
| 1551 x := &AggregateResult{ |
| 1552 BuilderInfo: &BuilderInfo{ |
| 1553 BuildNumbers: []Number{3}, |
| 1554 Tests: AggregateTest{ |
| 1555 "001.html": &AggregateTestLeaf{ |
| 1556 Results: []ResultSummar
y{{1, "T"}}, |
| 1557 Runtimes: []RuntimeSumma
ry{{1, 1}}, |
| 1558 }, |
| 1559 }, |
| 1560 }, |
| 1561 } |
| 1562 expected := &AggregateResult{ |
| 1563 Version: ResultsVersion, |
| 1564 BuilderInfo: &BuilderInfo{ |
| 1565 FailureMap: FailureLongNames, |
| 1566 BuildNumbers: []Number{3, 2, 1}, |
| 1567 Tests: AggregateTest{ |
| 1568 "001.html": &AggregateTestLeaf{ |
| 1569 Results: []ResultSummar
y{{1, "T"}, {size, "F"}}, |
| 1570 Runtimes: []RuntimeSumma
ry{{1, 1}, {size, 0}}, |
| 1571 }, |
| 1572 }, |
| 1573 }, |
| 1574 } |
| 1575 So(aggr.Merge(x), ShouldBeNil) |
| 1576 So(aggr.Trim(size), ShouldBeNil) |
| 1577 So(aggr, ShouldResemble, expected) |
| 1578 }) |
| 1579 |
| 1580 Convey("prune extra results small", func() { |
| 1581 size := ResultsSmallSize |
| 1582 aggr := &AggregateResult{ |
| 1583 BuilderInfo: &BuilderInfo{ |
| 1584 BuildNumbers: []Number{2, 1}, |
| 1585 Tests: AggregateTest{ |
| 1586 "001.html": &AggregateTestLeaf{ |
| 1587 Results: []ResultSummar
y{{size, "F"}, {1, "I"}}, |
| 1588 Runtimes: []RuntimeSumma
ry{{size, 0}, {1, 1}}, |
| 1589 }, |
| 1590 }, |
| 1591 }, |
| 1592 } |
| 1593 x := &AggregateResult{ |
| 1594 BuilderInfo: &BuilderInfo{ |
| 1595 BuildNumbers: []Number{3}, |
| 1596 Tests: AggregateTest{ |
| 1597 "001.html": &AggregateTestLeaf{ |
| 1598 Results: []ResultSummar
y{{1, "T"}}, |
| 1599 Runtimes: []RuntimeSumma
ry{{1, 1}}, |
| 1600 }, |
| 1601 }, |
| 1602 }, |
| 1603 } |
| 1604 expected := &AggregateResult{ |
| 1605 Version: ResultsVersion, |
| 1606 BuilderInfo: &BuilderInfo{ |
| 1607 FailureMap: FailureLongNames, |
| 1608 BuildNumbers: []Number{3, 2, 1}, |
| 1609 Tests: AggregateTest{ |
| 1610 "001.html": &AggregateTestLeaf{ |
| 1611 Results: []ResultSummar
y{{1, "T"}, {size, "F"}}, |
| 1612 Runtimes: []RuntimeSumma
ry{{1, 1}, {size, 0}}, |
| 1613 }, |
| 1614 }, |
| 1615 }, |
| 1616 } |
| 1617 So(aggr.Merge(x), ShouldBeNil) |
| 1618 So(aggr.Trim(size), ShouldBeNil) |
| 1619 So(aggr, ShouldResemble, expected) |
| 1620 }) |
| 1621 |
| 1622 Convey("prune extra results with new results of same type", func
() { |
| 1623 size := ResultsSmallSize |
| 1624 aggr := &AggregateResult{ |
| 1625 BuilderInfo: &BuilderInfo{ |
| 1626 BuildNumbers: []Number{2, 1}, |
| 1627 Tests: AggregateTest{ |
| 1628 "001.html": &AggregateTestLeaf{ |
| 1629 Results: []ResultSummar
y{{size, "F"}, {1, "N"}}, |
| 1630 Runtimes: []RuntimeSumma
ry{{size, 0}, {1, 1}}, |
| 1631 }, |
| 1632 }, |
| 1633 }, |
| 1634 } |
| 1635 x := &AggregateResult{ |
| 1636 BuilderInfo: &BuilderInfo{ |
| 1637 BuildNumbers: []Number{3}, |
| 1638 Tests: AggregateTest{ |
| 1639 "001.html": &AggregateTestLeaf{ |
| 1640 Results: []ResultSummar
y{{1, "F"}}, |
| 1641 Runtimes: []RuntimeSumma
ry{{1, 0}}, |
| 1642 }, |
| 1643 }, |
| 1644 }, |
| 1645 } |
| 1646 expected := &AggregateResult{ |
| 1647 Version: ResultsVersion, |
| 1648 BuilderInfo: &BuilderInfo{ |
| 1649 FailureMap: FailureLongNames, |
| 1650 BuildNumbers: []Number{3, 2, 1}, |
| 1651 Tests: AggregateTest{ |
| 1652 "001.html": &AggregateTestLeaf{ |
| 1653 Results: []ResultSummar
y{{size, "F"}}, |
| 1654 Runtimes: []RuntimeSumma
ry{{size, 0}}, |
| 1655 }, |
| 1656 }, |
| 1657 }, |
| 1658 } |
| 1659 So(aggr.Merge(x), ShouldBeNil) |
| 1660 So(aggr.Trim(size), ShouldBeNil) |
| 1661 So(aggr, ShouldResemble, expected) |
| 1662 }) |
| 1663 |
| 1664 Convey("merge build directory hierarchy", func() { |
| 1665 aggr := &AggregateResult{ |
| 1666 Version: ResultsVersion, |
| 1667 BuilderInfo: &BuilderInfo{ |
| 1668 BuildNumbers: []Number{2, 1}, |
| 1669 Tests: AggregateTest{ |
| 1670 "bar": AggregateTest{ |
| 1671 "baz": AggregateTest{ |
| 1672 "003.html": &Agg
regateTestLeaf{ |
| 1673 Results:
[]ResultSummary{{25, "F"}}, |
| 1674 Runtimes
: []RuntimeSummary{{25, 0}}, |
| 1675 }, |
| 1676 }, |
| 1677 }, |
| 1678 "foo": AggregateTest{ |
| 1679 "001.html": &AggregateTe
stLeaf{ |
| 1680 Results: []Resu
ltSummary{{50, "F"}}, |
| 1681 Runtimes: []Runt
imeSummary{{50, 0}}, |
| 1682 }, |
| 1683 "002.html": &AggregateTe
stLeaf{ |
| 1684 Results: []Resu
ltSummary{{100, "I"}}, |
| 1685 Runtimes: []Runt
imeSummary{{100, 0}}, |
| 1686 }, |
| 1687 }, |
| 1688 }, |
| 1689 }, |
| 1690 } |
| 1691 x := &AggregateResult{ |
| 1692 Version: ResultsVersion, |
| 1693 BuilderInfo: &BuilderInfo{ |
| 1694 BuildNumbers: []Number{3}, |
| 1695 Tests: AggregateTest{ |
| 1696 "baz": AggregateTest{ |
| 1697 "004.html": &AggregateTe
stLeaf{ |
| 1698 Results: []Resu
ltSummary{{1, "I"}}, |
| 1699 Runtimes: []Runt
imeSummary{{1, 0}}, |
| 1700 }, |
| 1701 }, |
| 1702 "foo": AggregateTest{ |
| 1703 "001.html": &AggregateTe
stLeaf{ |
| 1704 Results: []Resu
ltSummary{{1, "F"}}, |
| 1705 Runtimes: []Runt
imeSummary{{1, 0}}, |
| 1706 }, |
| 1707 "002.html": &AggregateTe
stLeaf{ |
| 1708 Results: []Resu
ltSummary{{1, "I"}}, |
| 1709 Runtimes: []Runt
imeSummary{{1, 0}}, |
| 1710 }, |
| 1711 }, |
| 1712 }, |
| 1713 }, |
| 1714 } |
| 1715 expected := &AggregateResult{ |
| 1716 Version: ResultsVersion, |
| 1717 BuilderInfo: &BuilderInfo{ |
| 1718 FailureMap: FailureLongNames, |
| 1719 BuildNumbers: []Number{3, 2, 1}, |
| 1720 Tests: AggregateTest{ |
| 1721 "bar": AggregateTest{ |
| 1722 "baz": AggregateTest{ |
| 1723 "003.html": &Agg
regateTestLeaf{ |
| 1724 Results:
[]ResultSummary{{1, "N"}, {25, "F"}}, |
| 1725 Runtimes
: []RuntimeSummary{{26, 0}}, |
| 1726 }, |
| 1727 }, |
| 1728 }, |
| 1729 "baz": AggregateTest{ |
| 1730 "004.html": &AggregateTe
stLeaf{ |
| 1731 Results: []Resu
ltSummary{{1, "I"}}, |
| 1732 Runtimes: []Runt
imeSummary{{1, 0}}, |
| 1733 }, |
| 1734 }, |
| 1735 "foo": AggregateTest{ |
| 1736 "001.html": &AggregateTe
stLeaf{ |
| 1737 Results: []Resu
ltSummary{{51, "F"}}, |
| 1738 Runtimes: []Runt
imeSummary{{51, 0}}, |
| 1739 }, |
| 1740 "002.html": &AggregateTe
stLeaf{ |
| 1741 Results: []Resu
ltSummary{{101, "I"}}, |
| 1742 Runtimes: []Runt
imeSummary{{101, 0}}, |
| 1743 }, |
| 1744 }, |
| 1745 }, |
| 1746 }, |
| 1747 } |
| 1748 So(aggr.Merge(x), ShouldBeNil) |
| 1749 So(aggr, ShouldResemble, expected) |
| 1750 }) |
| 1751 |
| 1752 Convey("treats multiple results as a unique type", func() { |
| 1753 aggr := &AggregateResult{ |
| 1754 BuilderInfo: &BuilderInfo{ |
| 1755 BuildNumbers: []Number{3, 1}, |
| 1756 Tests: AggregateTest{ |
| 1757 "001.html": &AggregateTestLeaf{ |
| 1758 Results: []ResultSummar
y{{5, "F"}}, |
| 1759 Runtimes: []RuntimeSumma
ry{{5, 0}}, |
| 1760 }, |
| 1761 "002.html": &AggregateTestLeaf{ |
| 1762 Results: []ResultSummar
y{{3, "FQ"}}, |
| 1763 Runtimes: []RuntimeSumma
ry{{3, 0}}, |
| 1764 }, |
| 1765 }, |
| 1766 }, |
| 1767 } |
| 1768 x := &AggregateResult{ |
| 1769 BuilderInfo: &BuilderInfo{ |
| 1770 BuildNumbers: []Number{2}, |
| 1771 Tests: AggregateTest{ |
| 1772 "001.html": &AggregateTestLeaf{ |
| 1773 Results: []ResultSummar
y{{1, "FIQ"}}, |
| 1774 Runtimes: []RuntimeSumma
ry{{1, 0}}, |
| 1775 }, |
| 1776 "002.html": &AggregateTestLeaf{ |
| 1777 Results: []ResultSummar
y{{1, "FQ"}}, |
| 1778 Runtimes: []RuntimeSumma
ry{{1, 0}}, |
| 1779 }, |
| 1780 }, |
| 1781 }, |
| 1782 } |
| 1783 expected := &AggregateResult{ |
| 1784 Version: ResultsVersion, |
| 1785 BuilderInfo: &BuilderInfo{ |
| 1786 FailureMap: FailureLongNames, |
| 1787 BuildNumbers: []Number{2, 3, 1}, |
| 1788 Tests: AggregateTest{ |
| 1789 "001.html": &AggregateTestLeaf{ |
| 1790 Results: []ResultSummar
y{{1, "FIQ"}, {5, "F"}}, |
| 1791 Runtimes: []RuntimeSumma
ry{{6, 0}}, |
| 1792 }, |
| 1793 "002.html": &AggregateTestLeaf{ |
| 1794 Results: []ResultSummar
y{{4, "FQ"}}, |
| 1795 Runtimes: []RuntimeSumma
ry{{4, 0}}, |
| 1796 }, |
| 1797 }, |
| 1798 }, |
| 1799 } |
| 1800 So(aggr.Merge(x), ShouldBeNil) |
| 1801 So(aggr, ShouldResemble, expected) |
| 1802 }) |
| 1803 |
| 1804 Convey("gtest", func() { |
| 1805 aggr := &AggregateResult{ |
| 1806 BuilderInfo: &BuilderInfo{ |
| 1807 BuildNumbers: []Number{2, 1}, |
| 1808 Tests: AggregateTest{ |
| 1809 "foo.bar": &AggregateTestLeaf{ |
| 1810 Results: []ResultSummar
y{{50, "F"}}, |
| 1811 Runtimes: []RuntimeSumma
ry{{50, 0}}, |
| 1812 }, |
| 1813 "foo.bar2": &AggregateTestLeaf{ |
| 1814 Results: []ResultSummar
y{{100, "I"}}, |
| 1815 Runtimes: []RuntimeSumma
ry{{100, 0}}, |
| 1816 }, |
| 1817 "test.failed": &AggregateTestLea
f{ |
| 1818 Results: []ResultSummar
y{{5, "Q"}}, |
| 1819 Runtimes: []RuntimeSumma
ry{{5, 0}}, |
| 1820 }, |
| 1821 }, |
| 1822 }, |
| 1823 } |
| 1824 x := &AggregateResult{ |
| 1825 BuilderInfo: &BuilderInfo{ |
| 1826 BuildNumbers: []Number{3}, |
| 1827 Tests: AggregateTest{ |
| 1828 "foo.bar2": &AggregateTestLeaf{ |
| 1829 Results: []ResultSummar
y{{1, "I"}}, |
| 1830 Runtimes: []RuntimeSumma
ry{{1, 0}}, |
| 1831 }, |
| 1832 "foo.bar3": &AggregateTestLeaf{ |
| 1833 Results: []ResultSummar
y{{1, "F"}}, |
| 1834 Runtimes: []RuntimeSumma
ry{{1, 0}}, |
| 1835 }, |
| 1836 "test.failed": &AggregateTestLea
f{ |
| 1837 Results: []ResultSummar
y{{5, "Q"}}, |
| 1838 Runtimes: []RuntimeSumma
ry{{5, 0}}, |
| 1839 }, |
| 1840 }, |
| 1841 }, |
| 1842 } |
| 1843 expected := &AggregateResult{ |
| 1844 Version: ResultsVersion, |
| 1845 BuilderInfo: &BuilderInfo{ |
| 1846 FailureMap: FailureLongNames, |
| 1847 BuildNumbers: []Number{3, 2, 1}, |
| 1848 Tests: AggregateTest{ |
| 1849 "foo.bar": &AggregateTestLeaf{ |
| 1850 Results: []ResultSummar
y{{1, "N"}, {50, "F"}}, |
| 1851 Runtimes: []RuntimeSumma
ry{{51, 0}}, |
| 1852 }, |
| 1853 "foo.bar2": &AggregateTestLeaf{ |
| 1854 Results: []ResultSummar
y{{101, "I"}}, |
| 1855 Runtimes: []RuntimeSumma
ry{{101, 0}}, |
| 1856 }, |
| 1857 "foo.bar3": &AggregateTestLeaf{ |
| 1858 Results: []ResultSummar
y{{1, "F"}}, |
| 1859 Runtimes: []RuntimeSumma
ry{{1, 0}}, |
| 1860 }, |
| 1861 "test.failed": &AggregateTestLea
f{ |
| 1862 Results: []ResultSummar
y{{10, "Q"}}, |
| 1863 Runtimes: []RuntimeSumma
ry{{10, 0}}, |
| 1864 }, |
| 1865 }, |
| 1866 }, |
| 1867 } |
| 1868 So(aggr.Merge(x), ShouldBeNil) |
| 1869 So(aggr, ShouldResemble, expected) |
| 1870 }) |
| 1871 }) |
| 1872 } |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 2231393002:
test-results: Add merge and trim methods (Closed)
Base URL: https://chromium.googlesource.com/infra/infra.git@master