| OLD | NEW |
| 1 // Copyright 2015 The Chromium Authors. All rights reserved. | 1 // Copyright 2015 The Chromium Authors. All rights reserved. |
| 2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
| 3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
| 4 | 4 |
| 5 package memory | 5 package memory |
| 6 | 6 |
| 7 import ( | 7 import ( |
| 8 "bytes" | 8 "bytes" |
| 9 "fmt" | 9 "fmt" |
| 10 | 10 |
| 11 ds "github.com/luci/gae/service/datastore" | 11 ds "github.com/luci/gae/service/datastore" |
| 12 "github.com/luci/gae/service/datastore/serialize" | 12 "github.com/luci/gae/service/datastore/serialize" |
| 13 "github.com/luci/luci-go/common/cmpbin" | 13 "github.com/luci/luci-go/common/cmpbin" |
| 14 "github.com/luci/luci-go/common/stringset" |
| 14 ) | 15 ) |
| 15 | 16 |
| 16 type queryStrategy interface { | 17 type queryStrategy interface { |
| 17 // handle applies the strategy to the embedded user callback. | 18 // handle applies the strategy to the embedded user callback. |
| 18 // - rawData is the slice of encoded Properties from the index row | 19 // - rawData is the slice of encoded Properties from the index row |
| 19 // (correctly de-inverted). | 20 // (correctly de-inverted). |
| 20 // - decodedProps is the slice of decoded Properties from the index ro
w | 21 // - decodedProps is the slice of decoded Properties from the index ro
w |
| 21 // - key is the decoded Key from the index row (the last item in rawDa
ta and | 22 // - key is the decoded Key from the index row (the last item in rawDa
ta and |
| 22 // decodedProps) | 23 // decodedProps) |
| 23 // - gc is the getCursor function to be passed to the user's callback | 24 // - gc is the getCursor function to be passed to the user's callback |
| 24 handle(rawData [][]byte, decodedProps []ds.Property, key ds.Key, gc func
() (ds.Cursor, error)) bool | 25 handle(rawData [][]byte, decodedProps []ds.Property, key ds.Key, gc func
() (ds.Cursor, error)) bool |
| 25 } | 26 } |
| 26 | 27 |
| 27 type projectionLookup struct { | 28 type projectionLookup struct { |
| 28 suffixIndex int | 29 suffixIndex int |
| 29 propertyName string | 30 propertyName string |
| 30 } | 31 } |
| 31 | 32 |
| 32 type projectionStrategy struct { | 33 type projectionStrategy struct { |
| 33 cb ds.RawRunCB | 34 cb ds.RawRunCB |
| 34 | 35 |
| 35 project []projectionLookup | 36 project []projectionLookup |
| 36 » distinct stringSet | 37 » distinct stringset.Set |
| 37 } | 38 } |
| 38 | 39 |
| 39 func newProjectionStrategy(q *queryImpl, rq *reducedQuery, cb ds.RawRunCB) query
Strategy { | 40 func newProjectionStrategy(q *queryImpl, rq *reducedQuery, cb ds.RawRunCB) query
Strategy { |
| 40 projectionLookups := make([]projectionLookup, len(q.project)) | 41 projectionLookups := make([]projectionLookup, len(q.project)) |
| 41 for i, prop := range q.project { | 42 for i, prop := range q.project { |
| 42 projectionLookups[i].propertyName = prop | 43 projectionLookups[i].propertyName = prop |
| 43 lookupErr := fmt.Errorf("planning a strategy for an unfulfillabl
e query?") | 44 lookupErr := fmt.Errorf("planning a strategy for an unfulfillabl
e query?") |
| 44 for j, col := range rq.suffixFormat { | 45 for j, col := range rq.suffixFormat { |
| 45 if col.Property == prop { | 46 if col.Property == prop { |
| 46 projectionLookups[i].suffixIndex = j | 47 projectionLookups[i].suffixIndex = j |
| 47 lookupErr = nil | 48 lookupErr = nil |
| 48 break | 49 break |
| 49 } | 50 } |
| 50 } | 51 } |
| 51 impossible(lookupErr) | 52 impossible(lookupErr) |
| 52 } | 53 } |
| 53 ret := &projectionStrategy{cb: cb, project: projectionLookups} | 54 ret := &projectionStrategy{cb: cb, project: projectionLookups} |
| 54 if q.distinct { | 55 if q.distinct { |
| 55 » » ret.distinct = stringSet{} | 56 » » ret.distinct = stringset.New(0) |
| 56 } | 57 } |
| 57 return ret | 58 return ret |
| 58 } | 59 } |
| 59 | 60 |
| 60 func (s *projectionStrategy) handle(rawData [][]byte, decodedProps []ds.Property
, key ds.Key, gc func() (ds.Cursor, error)) bool { | 61 func (s *projectionStrategy) handle(rawData [][]byte, decodedProps []ds.Property
, key ds.Key, gc func() (ds.Cursor, error)) bool { |
| 61 projectedRaw := [][]byte(nil) | 62 projectedRaw := [][]byte(nil) |
| 62 if s.distinct != nil { | 63 if s.distinct != nil { |
| 63 projectedRaw = make([][]byte, len(decodedProps)) | 64 projectedRaw = make([][]byte, len(decodedProps)) |
| 64 } | 65 } |
| 65 pmap := make(ds.PropertyMap, len(s.project)) | 66 pmap := make(ds.PropertyMap, len(s.project)) |
| 66 for i, p := range s.project { | 67 for i, p := range s.project { |
| 67 if s.distinct != nil { | 68 if s.distinct != nil { |
| 68 projectedRaw[i] = rawData[p.suffixIndex] | 69 projectedRaw[i] = rawData[p.suffixIndex] |
| 69 } | 70 } |
| 70 pmap[p.propertyName] = []ds.Property{decodedProps[p.suffixIndex]
} | 71 pmap[p.propertyName] = []ds.Property{decodedProps[p.suffixIndex]
} |
| 71 } | 72 } |
| 72 if s.distinct != nil { | 73 if s.distinct != nil { |
| 73 » » if !s.distinct.add(string(bjoin(projectedRaw...))) { | 74 » » if !s.distinct.Add(string(bjoin(projectedRaw...))) { |
| 74 return true | 75 return true |
| 75 } | 76 } |
| 76 } | 77 } |
| 77 return s.cb(key, pmap, gc) | 78 return s.cb(key, pmap, gc) |
| 78 } | 79 } |
| 79 | 80 |
| 80 type keysOnlyStrategy struct { | 81 type keysOnlyStrategy struct { |
| 81 cb ds.RawRunCB | 82 cb ds.RawRunCB |
| 82 | 83 |
| 83 » dedup stringSet | 84 » dedup stringset.Set |
| 84 } | 85 } |
| 85 | 86 |
| 86 func (s *keysOnlyStrategy) handle(rawData [][]byte, _ []ds.Property, key ds.Key,
gc func() (ds.Cursor, error)) bool { | 87 func (s *keysOnlyStrategy) handle(rawData [][]byte, _ []ds.Property, key ds.Key,
gc func() (ds.Cursor, error)) bool { |
| 87 » if !s.dedup.add(string(rawData[len(rawData)-1])) { | 88 » if !s.dedup.Add(string(rawData[len(rawData)-1])) { |
| 88 return true | 89 return true |
| 89 } | 90 } |
| 90 return s.cb(key, nil, gc) | 91 return s.cb(key, nil, gc) |
| 91 } | 92 } |
| 92 | 93 |
| 93 type normalStrategy struct { | 94 type normalStrategy struct { |
| 94 cb ds.RawRunCB | 95 cb ds.RawRunCB |
| 95 | 96 |
| 96 ns string | 97 ns string |
| 97 head *memCollection | 98 head *memCollection |
| 98 » dedup stringSet | 99 » dedup stringset.Set |
| 99 } | 100 } |
| 100 | 101 |
| 101 func newNormalStrategy(ns string, cb ds.RawRunCB, head *memStore) queryStrategy
{ | 102 func newNormalStrategy(ns string, cb ds.RawRunCB, head *memStore) queryStrategy
{ |
| 102 coll := head.GetCollection("ents:" + ns) | 103 coll := head.GetCollection("ents:" + ns) |
| 103 if coll == nil { | 104 if coll == nil { |
| 104 return nil | 105 return nil |
| 105 } | 106 } |
| 106 » return &normalStrategy{cb, ns, coll, stringSet{}} | 107 » return &normalStrategy{cb, ns, coll, stringset.New(0)} |
| 107 } | 108 } |
| 108 | 109 |
| 109 func (s *normalStrategy) handle(rawData [][]byte, _ []ds.Property, key ds.Key, g
c func() (ds.Cursor, error)) bool { | 110 func (s *normalStrategy) handle(rawData [][]byte, _ []ds.Property, key ds.Key, g
c func() (ds.Cursor, error)) bool { |
| 110 rawKey := rawData[len(rawData)-1] | 111 rawKey := rawData[len(rawData)-1] |
| 111 » if !s.dedup.add(string(rawKey)) { | 112 » if !s.dedup.Add(string(rawKey)) { |
| 112 return true | 113 return true |
| 113 } | 114 } |
| 114 | 115 |
| 115 rawEnt := s.head.Get(rawKey) | 116 rawEnt := s.head.Get(rawKey) |
| 116 if rawEnt == nil { | 117 if rawEnt == nil { |
| 117 // entity doesn't exist at head | 118 // entity doesn't exist at head |
| 118 return true | 119 return true |
| 119 } | 120 } |
| 120 pm, err := serialize.ReadPropertyMap(bytes.NewBuffer(rawEnt), serialize.
WithoutContext, globalAppID, s.ns) | 121 pm, err := serialize.ReadPropertyMap(bytes.NewBuffer(rawEnt), serialize.
WithoutContext, globalAppID, s.ns) |
| 121 memoryCorruption(err) | 122 memoryCorruption(err) |
| 122 | 123 |
| 123 return s.cb(key, pm, gc) | 124 return s.cb(key, pm, gc) |
| 124 } | 125 } |
| 125 | 126 |
| 126 func pickQueryStrategy(q *queryImpl, rq *reducedQuery, cb ds.RawRunCB, head *mem
Store) queryStrategy { | 127 func pickQueryStrategy(q *queryImpl, rq *reducedQuery, cb ds.RawRunCB, head *mem
Store) queryStrategy { |
| 127 if q.keysOnly { | 128 if q.keysOnly { |
| 128 » » return &keysOnlyStrategy{cb, stringSet{}} | 129 » » return &keysOnlyStrategy{cb, stringset.New(0)} |
| 129 } | 130 } |
| 130 if len(q.project) > 0 { | 131 if len(q.project) > 0 { |
| 131 return newProjectionStrategy(q, rq, cb) | 132 return newProjectionStrategy(q, rq, cb) |
| 132 } | 133 } |
| 133 return newNormalStrategy(rq.ns, cb, head) | 134 return newNormalStrategy(rq.ns, cb, head) |
| 134 } | 135 } |
| 135 | 136 |
| 136 func parseSuffix(ns string, suffixFormat []ds.IndexColumn, suffix []byte, count
int) (raw [][]byte, decoded []ds.Property) { | 137 func parseSuffix(ns string, suffixFormat []ds.IndexColumn, suffix []byte, count
int) (raw [][]byte, decoded []ds.Property) { |
| 137 buf := serialize.Invertible(bytes.NewBuffer(suffix)) | 138 buf := serialize.Invertible(bytes.NewBuffer(suffix)) |
| 138 decoded = make([]ds.Property, len(suffixFormat)) | 139 decoded = make([]ds.Property, len(suffixFormat)) |
| (...skipping 90 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 229 impossible(fmt.Errorf("decoded index row doesn't end wit
h a Key: %#v", keyProp)) | 230 impossible(fmt.Errorf("decoded index row doesn't end wit
h a Key: %#v", keyProp)) |
| 230 } | 231 } |
| 231 | 232 |
| 232 return strategy.handle( | 233 return strategy.handle( |
| 233 rawData, decodedProps, keyProp.Value().(ds.Key), | 234 rawData, decodedProps, keyProp.Value().(ds.Key), |
| 234 getCursorFn(suffix)) | 235 getCursorFn(suffix)) |
| 235 }) | 236 }) |
| 236 | 237 |
| 237 return nil | 238 return nil |
| 238 } | 239 } |
| OLD | NEW |