Refactor serialization helpers from impl/memory -> service/datastore/serialize

impl/memory/datastore_index.go

 // Copyright 2015 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 package memory
 
 import (
         "bytes"
         "fmt"
         "sort"
 
         ds "github.com/luci/gae/service/datastore"
         "github.com/luci/gae/service/datastore/serialize"
         "github.com/luci/gkvlite"
-        "github.com/luci/luci-go/common/stringset"
 )
 
 type qIndexSlice []*ds.IndexDefinition
 
 func (s qIndexSlice) Len() int           { return len(s) }
 func (s qIndexSlice) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
 func (s qIndexSlice) Less(i, j int) bool { return s[i].Less(s[j]) }
 
 func defaultIndexes(kind string, pmap ds.PropertyMap) []*ds.IndexDefinition {
         ret := make(qIndexSlice, 0, 2*len(pmap)+1)
@@ skipping 12 matching lines @@
                 ret = append(ret, &ds.IndexDefinition{Kind: kind, SortBy: []ds.IndexColumn{{Property: name}}})
                 ret = append(ret, &ds.IndexDefinition{Kind: kind, SortBy: []ds.IndexColumn{{Property: name, Descending: true}}})
         }
         if serializationDeterministic {
                 sort.Sort(ret)
         }
         return ret
 }
 
 func indexEntriesWithBuiltins(k *ds.Key, pm ds.PropertyMap, complexIdxs []*ds.IndexDefinition) *memStore {
-        sip := partiallySerialize(k, pm)
-        return sip.indexEntries(k.Namespace(), append(defaultIndexes(k.Kind(), pm), complexIdxs...))
-}
-
-// serializedPvals is all of the serialized DSProperty values in qASC order.
-type serializedPvals [][]byte
-
-func (s serializedPvals) Len() int           { return len(s) }
-func (s serializedPvals) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
-func (s serializedPvals) Less(i, j int) bool { return bytes.Compare(s[i], s[j]) < 0 }
-
-// prop name -> [<serialized DSProperty>, ...]
-// includes special values '__key__' and '__ancestor__' which contains all of
-// the ancestor entries for this key.
-type serializedIndexablePmap map[string]serializedPvals
-
-func serializeRow(vals []ds.Property) serializedPvals {
-        dups := stringset.New(0)
-        ret := make(serializedPvals, 0, len(vals))
-        for _, v := range vals {
-                if v.IndexSetting() == ds.NoIndex {
-                        continue
-                }
-                data := serialize.ToBytes(v.ForIndex())
-                dataS := string(data)
-                if !dups.Add(dataS) {
-                        continue
-                }
-                ret = append(ret, data)
-        }
-        return ret
-}
-
-func partiallySerialize(k *ds.Key, pm ds.PropertyMap) (ret serializedIndexablePmap) {
-        ret = make(serializedIndexablePmap, len(pm)+2)
-        if k == nil {
-                impossible(fmt.Errorf("key to partiallySerialize is nil"))
-        }
-        ret["__key__"] = [][]byte{serialize.ToBytes(ds.MkProperty(k))}
-        for k != nil {
-                ret["__ancestor__"] = append(ret["__ancestor__"], serialize.ToBytes(ds.MkProperty(k)))
-                k = k.Parent()
-        }
-        for k, vals := range pm {
-                newVals := serializeRow(vals)
-                if len(newVals) > 0 {
-                        ret[k] = newVals
-                }
-        }
-        return
+        sip := serialize.PropertyMapPartially(k, pm)
+        return indexEntries(sip, k.Namespace(), append(defaultIndexes(k.Kind(), pm), complexIdxs...))
 }
 
 // indexRowGen contains enough information to generate all of the index rows which
 // correspond with a propertyList and a ds.IndexDefinition.
 type indexRowGen struct {
-        propVec   []serializedPvals
+        propVec   []serialize.SerializedPslice
         decending []bool
 }
 
 // permute calls cb for each index row, in the sorted order of the rows.
 func (s indexRowGen) permute(collSetFn func(k, v []byte)) {
         iVec := make([]int, len(s.propVec))
         iVecLim := make([]int, len(s.propVec))
 
         incPos := func() bool {
                 for i := len(iVec) - 1; i >= 0; i-- {
@@ skipping 46 matching lines @@
         }
 }
 
 type matcher struct {
         buf indexRowGen
 }
 
 // matcher.match checks to see if the mapped, serialized property values
 // match the index. If they do, it returns a indexRowGen. Do not write or modify
 // the data in the indexRowGen.
-func (m *matcher) match(sortBy []ds.IndexColumn, sip serializedIndexablePmap) (indexRowGen, bool) {
+func (m *matcher) match(sortBy []ds.IndexColumn, sip serialize.SerializedPmap) (indexRowGen, bool) {
         m.buf.propVec = m.buf.propVec[:0]
         m.buf.decending = m.buf.decending[:0]
         for _, sb := range sortBy {
                 if pv, ok := sip[sb.Property]; ok {
                         m.buf.propVec = append(m.buf.propVec, pv)
                         m.buf.decending = append(m.buf.decending, sb.Descending)
                 } else {
                         return indexRowGen{}, false
                 }
         }
         return m.buf, true
 }
 
-func (sip serializedIndexablePmap) indexEntries(ns string, idxs []*ds.IndexDefinition) *memStore {
+func indexEntries(sip serialize.SerializedPmap, ns string, idxs []*ds.IndexDefinition) *memStore {
         ret := newMemStore()
         idxColl := ret.SetCollection("idx", nil)
 
         mtch := matcher{}
         for _, idx := range idxs {
                 idx = idx.Normalize()
                 if irg, ok := mtch.match(idx.GetFullSortOrder(), sip); ok {
                         idxBin := serialize.ToBytes(*idx.PrepForIdxTable())
                         idxColl.Set(idxBin, []byte{})
                         coll := ret.SetCollection(fmt.Sprintf("idx:%s:%s", ns, idxBin), nil)
@@ skipping 89 matching lines @@
         if allEnts := store.GetCollection("ents:" + ns); allEnts != nil {
                 allEnts.VisitItemsAscend(nil, true, func(i *gkvlite.Item) bool {
                         pm, err := rpmWoCtx(i.Val, ns)
                         memoryCorruption(err)
 
                         prop, err := serialize.ReadProperty(bytes.NewBuffer(i.Key), serialize.WithoutContext, globalAppID, ns)
                         memoryCorruption(err)
 
                         k := prop.Value().(*ds.Key)
 
-                        sip := partiallySerialize(k, pm)
+                        sip := serialize.PropertyMapPartially(k, pm)
 
                         mergeIndexes(ns, store,
                                 newMemStore(),
-                                sip.indexEntries(ns, normalized))
+                                indexEntries(sip, ns, normalized))
                         return true
                 })
         }
 }
 
 func updateIndexes(store *memStore, key *ds.Key, oldEnt, newEnt ds.PropertyMap) {
         // load all current complex query index definitions.
         compIdx := []*ds.IndexDefinition{}
         walkCompIdxs(store, nil, func(i *ds.IndexDefinition) bool {
                 compIdx = append(compIdx, i)
                 return true
         })
 
         mergeIndexes(key.Namespace(), store,
                 indexEntriesWithBuiltins(key, oldEnt, compIdx),
                 indexEntriesWithBuiltins(key, newEnt, compIdx))
 }