LogDog: Update to archival V2.

appengine/logdog/coordinator/endpoints/services/terminateStream_test.go

 // Copyright 2015 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 package services
 
 import (
         "errors"
         "testing"
         "time"
 
         "github.com/luci/gae/filter/featureBreaker"
         "github.com/luci/gae/impl/memory"
         ds "github.com/luci/gae/service/datastore"
+        tq "github.com/luci/gae/service/taskqueue"
         "github.com/luci/luci-go/appengine/logdog/coordinator"
         ct "github.com/luci/luci-go/appengine/logdog/coordinator/coordinatorTest"
         "github.com/luci/luci-go/common/api/logdog_coordinator/services/v1"
         "github.com/luci/luci-go/common/clock/testclock"
+        "github.com/luci/luci-go/common/proto/google"
         "github.com/luci/luci-go/common/proto/logdog/svcconfig"
         "github.com/luci/luci-go/server/auth"
         "github.com/luci/luci-go/server/auth/authtest"
         "golang.org/x/net/context"
 
         . "github.com/luci/luci-go/common/testing/assertions"
         . "github.com/smartystreets/goconvey/convey"
 )
 
 func TestTerminateStream(t *testing.T) {
         t.Parallel()
 
         Convey(`With a testing configuration`, t, func() {
                 c, tc := testclock.UseTime(context.Background(), testclock.TestTimeLocal)
                 c = memory.Use(c)
                 be := Server{}
 
                 desc := ct.TestLogStreamDescriptor(c, "foo/bar")
                 ls := ct.TestLogStream(c, desc)
 
                 req := logdog.TerminateStreamRequest{
                         Path:          "testing/+/foo/bar",
                         Secret:        ls.Secret,
                         TerminalIndex: 1337,
                 }
 
-                c = ct.UseConfig(c, &svcconfig.Coordinator{
-                        ServiceAuthGroup: "test-services",
-                })
+                ccfg := svcconfig.Coordinator{
+                        ServiceAuthGroup:   "test-services",
+                        ArchiveTaskQueue:   "archive-task-queue",
+                        ArchiveSettleDelay: google.NewDuration(10 * time.Second),
+                        ArchiveDelayMax:    google.NewDuration(24 * time.Hour),
+                }
+                c = ct.UseConfig(c, &ccfg)
+
+                // Create an archival task queue.
+                tq.Get(c).Testable().CreateQueue(ccfg.ArchiveTaskQueue)
+                archiveTasks := func() []string {
+                        tasks, err := ct.GetArchiveTaskStreams(tq.Get(c), ccfg.ArchiveTaskQueue)
+                        if err != nil {
+                                panic(err)
+                        }
+                        return tasks
+                }
+
                 fs := authtest.FakeState{}
                 c = auth.WithState(c, &fs)
 
                 Convey(`Returns Forbidden error if not a service.`, func() {
                         _, err := be.TerminateStream(c, &req)
                         So(err, ShouldBeRPCPermissionDenied)
                 })
 
                 Convey(`When logged in as a service`, func() {
                         fs.IdentityGroups = []string{"test-services"}
 
                         Convey(`A non-terminal registered stream, "testing/+/foo/bar"`, func() {
-                                So(ls.Put(ds.Get(c)), ShouldBeNil)
-                                tc.Add(time.Second)
+                                So(ds.Get(c).Put(ls), ShouldBeNil)
 
-                                Convey(`Can be marked terminal.`, func() {
+                                Convey(`Can be marked terminal and schedules an archival task.`, func() {
                                         _, err := be.TerminateStream(c, &req)
                                         So(err, ShouldBeRPCOK)
 
                                         // Reload "ls" and confirm.
                                         So(ds.Get(c).Get(ls), ShouldBeNil)
                                         So(ls.TerminalIndex, ShouldEqual, 1337)
-                                        So(ls.State, ShouldEqual, coordinator.LSTerminated)
-                                        So(ls.Updated, ShouldResemble, ls.Created.Add(time.Second))
+                                        So(ls.State, ShouldEqual, coordinator.LSArchiveTasked)
+                                        So(ls.Terminated(), ShouldBeTrue)
+                                        So(archiveTasks(), ShouldResemble, []string{ls.Name})
+
+                                        // Assert that all archive tasks are scheduled ArchiveSettleDelay in
+                                        // the future.
+                                        for _, t := range tq.Get(c).Testable().GetScheduledTasks()[ccfg.ArchiveTaskQueue] {
+                                                So(t.ETA.After(tc.Now()), ShouldBeTrue)
+                                        }
 
                                         Convey(`Can be marked terminal again (idempotent).`, func() {
                                                 _, err := be.TerminateStream(c, &req)
                                                 So(err, ShouldBeRPCOK)
 
                                                 // Reload "ls" and confirm.
                                                 So(ds.Get(c).Get(ls), ShouldBeNil)
+
+                                                So(ls.Terminated(), ShouldBeTrue)
                                                 So(ls.TerminalIndex, ShouldEqual, 1337)
-                                                So(ls.State, ShouldEqual, coordinator.LSTerminated)
+                                                So(ls.State, ShouldEqual, coordinator.LSArchiveTasked)
+                                                So(archiveTasks(), ShouldResemble, []string{ls.Name})
                                         })
 
                                         Convey(`Will reject attempts to change the terminal index.`, func() {
                                                 req.TerminalIndex = 1338
                                                 _, err := be.TerminateStream(c, &req)
-                                                So(err, ShouldBeRPCAlreadyExists, "Terminal index is already set")
+                                                So(err, ShouldBeRPCFailedPrecondition, "Log stream is not in streaming state.")
 
                                                 // Reload "ls" and confirm.
                                                 So(ds.Get(c).Get(ls), ShouldBeNil)
-                                                So(ls.State, ShouldEqual, coordinator.LSTerminated)
+
+                                                So(ls.Terminated(), ShouldBeTrue)
+                                                So(ls.State, ShouldEqual, coordinator.LSArchiveTasked)
                                                 So(ls.TerminalIndex, ShouldEqual, 1337)
+                                                So(archiveTasks(), ShouldResemble, []string{ls.Name})
                                         })
 
                                         Convey(`Will reject attempts to clear the terminal index.`, func() {
                                                 req.TerminalIndex = -1
                                                 _, err := be.TerminateStream(c, &req)
                                                 So(err, ShouldBeRPCInvalidArgument, "Negative terminal index.")
 
                                                 // Reload "ls" and confirm.
                                                 So(ds.Get(c).Get(ls), ShouldBeNil)
-                                                So(ls.State, ShouldEqual, coordinator.LSTerminated)
+
+                                                So(ls.Terminated(), ShouldBeTrue)
+                                                So(ls.State, ShouldEqual, coordinator.LSArchiveTasked)
                                                 So(ls.TerminalIndex, ShouldEqual, 1337)
+                                                So(archiveTasks(), ShouldResemble, []string{ls.Name})
                                         })
                                 })
 
                                 Convey(`Will return an internal server error if Put() fails.`, func() {
                                         c, fb := featureBreaker.FilterRDS(c, nil)
                                         fb.BreakFeatures(errors.New("test error"), "PutMulti")
                                         _, err := be.TerminateStream(c, &req)
                                         So(err, ShouldBeRPCInternal)
                                 })
 
@@ skipping 17 matching lines @@
                                 So(err, ShouldBeRPCInvalidArgument, "Invalid path")
                         })
 
                         Convey(`Will fail if the stream is not registered.`, func() {
                                 _, err := be.TerminateStream(c, &req)
                                 So(err, ShouldBeRPCNotFound, "is not registered")
                         })
                 })
         })
 }