LogDog: Add Storage-layer data caching.

logdog/common/storage/bigtable/storage_test.go

 // Copyright 2015 The LUCI Authors. All rights reserved.
 // Use of this source code is governed under the Apache License, Version 2.0
 // that can be found in the LICENSE file.
 
 package bigtable
 
 import (
         "bytes"
-        "fmt"
         "strconv"
         "testing"
-        "time"
 
-        "github.com/luci/gkvlite"
         "github.com/luci/luci-go/common/config"
         "github.com/luci/luci-go/common/data/recordio"
         "github.com/luci/luci-go/logdog/common/storage"
         "github.com/luci/luci-go/logdog/common/types"
         "golang.org/x/net/context"
 
         . "github.com/luci/luci-go/common/testing/assertions"
         . "github.com/smartystreets/goconvey/convey"
 )
 
-// btTableTest is an in-memory implementation of btTable interface for testing.
-//
-// This is a simple implementation; not an efficient one.
-type btTableTest struct {
-        s *gkvlite.Store
-        c *gkvlite.Collection
-
-        // err, if true, is the error immediately returned by functions.
-        err error
-
-        // maxLogAge is the currently-configured maximum log age.
-        maxLogAge time.Duration
-}
-
-func (t *btTableTest) close() {
-        if t.s != nil {
-                t.s.Close()
-                t.s = nil
-        }
-}
-
-func (t *btTableTest) collection() *gkvlite.Collection {
-        if t.s == nil {
-                var err error
-                t.s, err = gkvlite.NewStore(nil)
-                if err != nil {
-                        panic(err)
-                }
-                t.c = t.s.MakePrivateCollection(bytes.Compare)
-        }
-        return t.c
-}
-
-func (t *btTableTest) putLogData(c context.Context, rk *rowKey, d []byte) error {
-        if t.err != nil {
-                return t.err
-        }
-
-        // Record/count sanity check.
-        records, err := recordio.Split(d)
-        if err != nil {
-                return err
-        }
-        if int64(len(records)) != rk.count {
-                return fmt.Errorf("count mismatch (%d != %d)", len(records), rk.count)
-        }
-
-        enc := []byte(rk.encode())
-        coll := t.collection()
-        if item, _ := coll.Get(enc); item != nil {
-                return storage.ErrExists
-        }
-
-        clone := make([]byte, len(d))
-        copy(clone, d)
-        if err := coll.Set(enc, clone); err != nil {
-                panic(err)
-        }
-        return nil
-}
-
-func (t *btTableTest) getLogData(c context.Context, rk *rowKey, limit int, keysOnly bool, cb btGetCallback) error {
-        if t.err != nil {
-                return t.err
-        }
-
-        enc := []byte(rk.encode())
-        prefix := rk.pathPrefix()
-        var ierr error
-        err := t.collection().VisitItemsAscend(enc, !keysOnly, func(i *gkvlite.Item) bool {
-                var drk *rowKey
-                drk, ierr = decodeRowKey(string(i.Key))
-                if ierr != nil {
-                        return false
-                }
-                if drk.pathPrefix() != prefix {
-                        return false
-                }
-
-                rowData := i.Val
-                if keysOnly {
-                        rowData = nil
-                }
-
-                if ierr = cb(drk, rowData); ierr != nil {
-                        if ierr == errStop {
-                                ierr = nil
-                        }
-                        return false
-                }
-
-                if limit > 0 {
-                        limit--
-                        if limit == 0 {
-                                return false
-                        }
-                }
-
-                return true
-        })
-        if err != nil {
-                panic(err)
-        }
-        return ierr
-}
-
-func (t *btTableTest) setMaxLogAge(c context.Context, d time.Duration) error {
-        if t.err != nil {
-                return t.err
-        }
-        t.maxLogAge = d
-        return nil
-}
-
-func (t *btTableTest) dataMap() map[string][]byte {
-        result := map[string][]byte{}
-
-        err := t.collection().VisitItemsAscend([]byte(nil), true, func(i *gkvlite.Item) bool {
-                result[string(i.Key)] = i.Val
-                return true
-        })
-        if err != nil {
-                panic(err)
-        }
-        return result
-}
-
 func mustGetIndex(e *storage.Entry) types.MessageIndex {
         idx, err := e.GetStreamIndex()
         if err != nil {
                 panic(err)
         }
         return idx
 }
 
 func TestStorage(t *testing.T) {
         t.Parallel()
 
         Convey(`A BigTable storage instance bound to a testing BigTable instance`, t, func() {
-                bt := btTableTest{}
-                defer bt.close()
-
-                s := newBTStorage(context.Background(), Options{
-                        Project:  "test-project",
-                        Instance: "test-instance",
-                        LogTable: "test-log-table",
-                }, nil, nil)
-
-                s.raw = &bt
+                s := NewMemoryInstance(context.Background(), Options{})
                 defer s.Close()
 
                 project := config.ProjectName("test-project")
                 get := func(path string, index int, limit int, keysOnly bool) ([]string, error) {
                         req := storage.GetRequest{
                                 Project:  project,
                                 Path:     types.StreamPath(path),
                                 Index:    types.MessageIndex(index),
                                 Limit:    limit,
                                 KeysOnly: keysOnly,
@@ skipping 39 matching lines @@
                                         panic(err)
                                 }
                         }
 
                         return buf.Bytes()
                 }
 
                 Convey(`With an artificial maximum BigTable row size of two records`, func() {
                         // Artificially constrain row size. 4 = 2*{size/1, data/1} RecordIO
                         // entries.
-                        s.maxRowSize = 4
+                        s.SetMaxRowSize(4)
 
                         Convey(`Will split row data that overflows the table into multiple rows.`, func() {
                                 So(put("A", 0, "0", "1", "2", "3"), ShouldBeNil)
 
-                                So(bt.dataMap(), ShouldResemble, map[string][]byte{
+                                So(s.DataMap(), ShouldResemble, map[string][]byte{
                                         ekey("A", 1, 2): records("0", "1"),
                                         ekey("A", 3, 2): records("2", "3"),
                                 })
                         })
 
                         Convey(`Loading a single row data beyond the maximum row size will fail.`, func() {
                                 So(put("A", 0, "0123"), ShouldErrLike, "single row entry exceeds maximum size")
                         })
                 })
 
                 Convey(`With row data: A{0, 1, 2, 3, 4}, B{10, 12, 13}`, func() {
                         So(put("A", 0, "0", "1", "2"), ShouldBeNil)
                         So(put("A", 3, "3", "4"), ShouldBeNil)
                         So(put("B", 10, "10"), ShouldBeNil)
                         So(put("B", 12, "12", "13"), ShouldBeNil)
 
                         Convey(`Testing "Put"...`, func() {
                                 Convey(`Loads the row data.`, func() {
-                                        So(bt.dataMap(), ShouldResemble, map[string][]byte{
+                                        So(s.DataMap(), ShouldResemble, map[string][]byte{
                                                 ekey("A", 2, 3):  records("0", "1", "2"),
                                                 ekey("A", 4, 2):  records("3", "4"),
                                                 ekey("B", 10, 1): records("10"),
                                                 ekey("B", 13, 2): records("12", "13"),
                                         })
                                 })
                         })
 
                         Convey(`Testing "Get"...`, func() {
                                 Convey(`Can fetch the full row, "A".`, func() {
@@ skipping 87 matching lines @@
                                 })
 
                                 Convey(`A tail request for "INVALID" errors NOT FOUND.`, func() {
                                         _, err := tail("INVALID")
                                         So(err, ShouldEqual, storage.ErrDoesNotExist)
                                 })
                         })
                 })
         })
 }