OLD | NEW |
1 // Copyright 2015 The Chromium Authors. All rights reserved. | 1 // Copyright 2015 The Chromium Authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
4 | 4 |
5 package memory | 5 package memory |
6 | 6 |
7 import ( | 7 import ( |
8 "bytes" | 8 "bytes" |
9 "fmt" | 9 "fmt" |
10 | 10 |
11 ds "github.com/luci/gae/service/datastore" | 11 ds "github.com/luci/gae/service/datastore" |
12 "github.com/luci/gae/service/datastore/serialize" | 12 "github.com/luci/gae/service/datastore/serialize" |
13 "github.com/luci/luci-go/common/cmpbin" | 13 "github.com/luci/luci-go/common/cmpbin" |
| 14 "github.com/luci/luci-go/common/stringset" |
14 ) | 15 ) |
15 | 16 |
16 type queryStrategy interface { | 17 type queryStrategy interface { |
17 // handle applies the strategy to the embedded user callback. | 18 // handle applies the strategy to the embedded user callback. |
18 // - rawData is the slice of encoded Properties from the index row | 19 // - rawData is the slice of encoded Properties from the index row |
19 // (correctly de-inverted). | 20 // (correctly de-inverted). |
20 // - decodedProps is the slice of decoded Properties from the index ro
w | 21 // - decodedProps is the slice of decoded Properties from the index ro
w |
21 // - key is the decoded Key from the index row (the last item in rawDa
ta and | 22 // - key is the decoded Key from the index row (the last item in rawDa
ta and |
22 // decodedProps) | 23 // decodedProps) |
23 // - gc is the getCursor function to be passed to the user's callback | 24 // - gc is the getCursor function to be passed to the user's callback |
24 handle(rawData [][]byte, decodedProps []ds.Property, key ds.Key, gc func
() (ds.Cursor, error)) bool | 25 handle(rawData [][]byte, decodedProps []ds.Property, key ds.Key, gc func
() (ds.Cursor, error)) bool |
25 } | 26 } |
26 | 27 |
27 type projectionLookup struct { | 28 type projectionLookup struct { |
28 suffixIndex int | 29 suffixIndex int |
29 propertyName string | 30 propertyName string |
30 } | 31 } |
31 | 32 |
32 type projectionStrategy struct { | 33 type projectionStrategy struct { |
33 cb ds.RawRunCB | 34 cb ds.RawRunCB |
34 | 35 |
35 project []projectionLookup | 36 project []projectionLookup |
36 » distinct stringSet | 37 » distinct stringset.Set |
37 } | 38 } |
38 | 39 |
39 func newProjectionStrategy(q *queryImpl, rq *reducedQuery, cb ds.RawRunCB) query
Strategy { | 40 func newProjectionStrategy(q *queryImpl, rq *reducedQuery, cb ds.RawRunCB) query
Strategy { |
40 projectionLookups := make([]projectionLookup, len(q.project)) | 41 projectionLookups := make([]projectionLookup, len(q.project)) |
41 for i, prop := range q.project { | 42 for i, prop := range q.project { |
42 projectionLookups[i].propertyName = prop | 43 projectionLookups[i].propertyName = prop |
43 lookupErr := fmt.Errorf("planning a strategy for an unfulfillabl
e query?") | 44 lookupErr := fmt.Errorf("planning a strategy for an unfulfillabl
e query?") |
44 for j, col := range rq.suffixFormat { | 45 for j, col := range rq.suffixFormat { |
45 if col.Property == prop { | 46 if col.Property == prop { |
46 projectionLookups[i].suffixIndex = j | 47 projectionLookups[i].suffixIndex = j |
47 lookupErr = nil | 48 lookupErr = nil |
48 break | 49 break |
49 } | 50 } |
50 } | 51 } |
51 impossible(lookupErr) | 52 impossible(lookupErr) |
52 } | 53 } |
53 ret := &projectionStrategy{cb: cb, project: projectionLookups} | 54 ret := &projectionStrategy{cb: cb, project: projectionLookups} |
54 if q.distinct { | 55 if q.distinct { |
55 » » ret.distinct = stringSet{} | 56 » » ret.distinct = stringset.New(0) |
56 } | 57 } |
57 return ret | 58 return ret |
58 } | 59 } |
59 | 60 |
60 func (s *projectionStrategy) handle(rawData [][]byte, decodedProps []ds.Property
, key ds.Key, gc func() (ds.Cursor, error)) bool { | 61 func (s *projectionStrategy) handle(rawData [][]byte, decodedProps []ds.Property
, key ds.Key, gc func() (ds.Cursor, error)) bool { |
61 projectedRaw := [][]byte(nil) | 62 projectedRaw := [][]byte(nil) |
62 if s.distinct != nil { | 63 if s.distinct != nil { |
63 projectedRaw = make([][]byte, len(decodedProps)) | 64 projectedRaw = make([][]byte, len(decodedProps)) |
64 } | 65 } |
65 pmap := make(ds.PropertyMap, len(s.project)) | 66 pmap := make(ds.PropertyMap, len(s.project)) |
66 for i, p := range s.project { | 67 for i, p := range s.project { |
67 if s.distinct != nil { | 68 if s.distinct != nil { |
68 projectedRaw[i] = rawData[p.suffixIndex] | 69 projectedRaw[i] = rawData[p.suffixIndex] |
69 } | 70 } |
70 pmap[p.propertyName] = []ds.Property{decodedProps[p.suffixIndex]
} | 71 pmap[p.propertyName] = []ds.Property{decodedProps[p.suffixIndex]
} |
71 } | 72 } |
72 if s.distinct != nil { | 73 if s.distinct != nil { |
73 » » if !s.distinct.add(string(bjoin(projectedRaw...))) { | 74 » » if !s.distinct.Add(string(bjoin(projectedRaw...))) { |
74 return true | 75 return true |
75 } | 76 } |
76 } | 77 } |
77 return s.cb(key, pmap, gc) | 78 return s.cb(key, pmap, gc) |
78 } | 79 } |
79 | 80 |
80 type keysOnlyStrategy struct { | 81 type keysOnlyStrategy struct { |
81 cb ds.RawRunCB | 82 cb ds.RawRunCB |
82 | 83 |
83 » dedup stringSet | 84 » dedup stringset.Set |
84 } | 85 } |
85 | 86 |
86 func (s *keysOnlyStrategy) handle(rawData [][]byte, _ []ds.Property, key ds.Key,
gc func() (ds.Cursor, error)) bool { | 87 func (s *keysOnlyStrategy) handle(rawData [][]byte, _ []ds.Property, key ds.Key,
gc func() (ds.Cursor, error)) bool { |
87 » if !s.dedup.add(string(rawData[len(rawData)-1])) { | 88 » if !s.dedup.Add(string(rawData[len(rawData)-1])) { |
88 return true | 89 return true |
89 } | 90 } |
90 return s.cb(key, nil, gc) | 91 return s.cb(key, nil, gc) |
91 } | 92 } |
92 | 93 |
93 type normalStrategy struct { | 94 type normalStrategy struct { |
94 cb ds.RawRunCB | 95 cb ds.RawRunCB |
95 | 96 |
96 ns string | 97 ns string |
97 head *memCollection | 98 head *memCollection |
98 » dedup stringSet | 99 » dedup stringset.Set |
99 } | 100 } |
100 | 101 |
101 func newNormalStrategy(ns string, cb ds.RawRunCB, head *memStore) queryStrategy
{ | 102 func newNormalStrategy(ns string, cb ds.RawRunCB, head *memStore) queryStrategy
{ |
102 coll := head.GetCollection("ents:" + ns) | 103 coll := head.GetCollection("ents:" + ns) |
103 if coll == nil { | 104 if coll == nil { |
104 return nil | 105 return nil |
105 } | 106 } |
106 » return &normalStrategy{cb, ns, coll, stringSet{}} | 107 » return &normalStrategy{cb, ns, coll, stringset.New(0)} |
107 } | 108 } |
108 | 109 |
109 func (s *normalStrategy) handle(rawData [][]byte, _ []ds.Property, key ds.Key, g
c func() (ds.Cursor, error)) bool { | 110 func (s *normalStrategy) handle(rawData [][]byte, _ []ds.Property, key ds.Key, g
c func() (ds.Cursor, error)) bool { |
110 rawKey := rawData[len(rawData)-1] | 111 rawKey := rawData[len(rawData)-1] |
111 » if !s.dedup.add(string(rawKey)) { | 112 » if !s.dedup.Add(string(rawKey)) { |
112 return true | 113 return true |
113 } | 114 } |
114 | 115 |
115 rawEnt := s.head.Get(rawKey) | 116 rawEnt := s.head.Get(rawKey) |
116 if rawEnt == nil { | 117 if rawEnt == nil { |
117 // entity doesn't exist at head | 118 // entity doesn't exist at head |
118 return true | 119 return true |
119 } | 120 } |
120 pm, err := serialize.ReadPropertyMap(bytes.NewBuffer(rawEnt), serialize.
WithoutContext, globalAppID, s.ns) | 121 pm, err := serialize.ReadPropertyMap(bytes.NewBuffer(rawEnt), serialize.
WithoutContext, globalAppID, s.ns) |
121 memoryCorruption(err) | 122 memoryCorruption(err) |
122 | 123 |
123 return s.cb(key, pm, gc) | 124 return s.cb(key, pm, gc) |
124 } | 125 } |
125 | 126 |
126 func pickQueryStrategy(q *queryImpl, rq *reducedQuery, cb ds.RawRunCB, head *mem
Store) queryStrategy { | 127 func pickQueryStrategy(q *queryImpl, rq *reducedQuery, cb ds.RawRunCB, head *mem
Store) queryStrategy { |
127 if q.keysOnly { | 128 if q.keysOnly { |
128 » » return &keysOnlyStrategy{cb, stringSet{}} | 129 » » return &keysOnlyStrategy{cb, stringset.New(0)} |
129 } | 130 } |
130 if len(q.project) > 0 { | 131 if len(q.project) > 0 { |
131 return newProjectionStrategy(q, rq, cb) | 132 return newProjectionStrategy(q, rq, cb) |
132 } | 133 } |
133 return newNormalStrategy(rq.ns, cb, head) | 134 return newNormalStrategy(rq.ns, cb, head) |
134 } | 135 } |
135 | 136 |
136 func parseSuffix(ns string, suffixFormat []ds.IndexColumn, suffix []byte, count
int) (raw [][]byte, decoded []ds.Property) { | 137 func parseSuffix(ns string, suffixFormat []ds.IndexColumn, suffix []byte, count
int) (raw [][]byte, decoded []ds.Property) { |
137 buf := serialize.Invertible(bytes.NewBuffer(suffix)) | 138 buf := serialize.Invertible(bytes.NewBuffer(suffix)) |
138 decoded = make([]ds.Property, len(suffixFormat)) | 139 decoded = make([]ds.Property, len(suffixFormat)) |
(...skipping 90 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
229 impossible(fmt.Errorf("decoded index row doesn't end wit
h a Key: %#v", keyProp)) | 230 impossible(fmt.Errorf("decoded index row doesn't end wit
h a Key: %#v", keyProp)) |
230 } | 231 } |
231 | 232 |
232 return strategy.handle( | 233 return strategy.handle( |
233 rawData, decodedProps, keyProp.Value().(ds.Key), | 234 rawData, decodedProps, keyProp.Value().(ds.Key), |
234 getCursorFn(suffix)) | 235 getCursorFn(suffix)) |
235 }) | 236 }) |
236 | 237 |
237 return nil | 238 return nil |
238 } | 239 } |
OLD | NEW |