OLD | NEW |
1 // Copyright 2015 The Chromium Authors. All rights reserved. | 1 // Copyright 2015 The Chromium Authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
4 | 4 |
5 package memory | 5 package memory |
6 | 6 |
7 import ( | 7 import ( |
8 "bytes" | 8 "bytes" |
9 "errors" | |
10 "fmt" | 9 "fmt" |
11 "sync" | 10 "sync" |
12 "sync/atomic" | 11 "sync/atomic" |
13 | 12 |
14 "github.com/luci/gae" | |
15 rds "github.com/luci/gae/service/rawdatastore" | 13 rds "github.com/luci/gae/service/rawdatastore" |
| 14 "github.com/luci/luci-go/common/errors" |
16 "golang.org/x/net/context" | 15 "golang.org/x/net/context" |
17 ) | 16 ) |
18 | 17 |
19 //////////////////////////////// dataStoreData ///////////////////////////////// | 18 //////////////////////////////// dataStoreData ///////////////////////////////// |
20 | 19 |
21 type dataStoreData struct { | 20 type dataStoreData struct { |
22 rwlock sync.RWMutex | 21 rwlock sync.RWMutex |
23 // See README.md for store schema. | 22 // See README.md for store schema. |
24 store *memStore | 23 store *memStore |
25 snap *memStore | 24 snap *memStore |
(...skipping 80 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
106 } | 105 } |
107 | 106 |
108 return ents, key | 107 return ents, key |
109 } | 108 } |
110 | 109 |
111 func (d *dataStoreData) put(ns string, key rds.Key, pls rds.PropertyLoadSaver) (
rds.Key, error) { | 110 func (d *dataStoreData) put(ns string, key rds.Key, pls rds.PropertyLoadSaver) (
rds.Key, error) { |
112 keys, errs := d.putMulti(ns, []rds.Key{key}, []rds.PropertyLoadSaver{pls
}) | 111 keys, errs := d.putMulti(ns, []rds.Key{key}, []rds.PropertyLoadSaver{pls
}) |
113 if errs == nil { | 112 if errs == nil { |
114 return keys[0], nil | 113 return keys[0], nil |
115 } | 114 } |
116 » return nil, gae.SingleError(errs) | 115 » return nil, errors.SingleError(errs) |
117 } | 116 } |
118 | 117 |
119 func (d *dataStoreData) putMulti(ns string, keys []rds.Key, plss []rds.PropertyL
oadSaver) ([]rds.Key, error) { | 118 func (d *dataStoreData) putMulti(ns string, keys []rds.Key, plss []rds.PropertyL
oadSaver) ([]rds.Key, error) { |
120 pmaps, err := putMultiPrelim(ns, keys, plss) | 119 pmaps, err := putMultiPrelim(ns, keys, plss) |
121 if err != nil { | 120 if err != nil { |
122 return nil, err | 121 return nil, err |
123 } | 122 } |
124 return d.putMultiInner(keys, pmaps) | 123 return d.putMultiInner(keys, pmaps) |
125 } | 124 } |
126 | 125 |
127 func putMultiPrelim(ns string, keys []rds.Key, plss []rds.PropertyLoadSaver) ([]
rds.PropertyMap, error) { | 126 func putMultiPrelim(ns string, keys []rds.Key, plss []rds.PropertyLoadSaver) ([]
rds.PropertyMap, error) { |
128 err := multiValid(keys, plss, ns, true, false) | 127 err := multiValid(keys, plss, ns, true, false) |
129 if err != nil { | 128 if err != nil { |
130 return nil, err | 129 return nil, err |
131 } | 130 } |
132 pmaps := make([]rds.PropertyMap, len(keys)) | 131 pmaps := make([]rds.PropertyMap, len(keys)) |
133 » lme := gae.LazyMultiError{Size: len(keys)} | 132 » lme := errors.LazyMultiError{Size: len(keys)} |
134 for i, pls := range plss { | 133 for i, pls := range plss { |
135 pm, err := pls.Save(false) | 134 pm, err := pls.Save(false) |
136 lme.Assign(i, err) | 135 lme.Assign(i, err) |
137 pmaps[i] = pm | 136 pmaps[i] = pm |
138 } | 137 } |
139 return pmaps, lme.Get() | 138 return pmaps, lme.Get() |
140 } | 139 } |
141 | 140 |
142 func (d *dataStoreData) putMultiInner(keys []rds.Key, data []rds.PropertyMap) ([
]rds.Key, error) { | 141 func (d *dataStoreData) putMultiInner(keys []rds.Key, data []rds.PropertyMap) ([
]rds.Key, error) { |
143 retKeys := make([]rds.Key, len(keys)) | 142 retKeys := make([]rds.Key, len(keys)) |
144 » lme := gae.LazyMultiError{Size: len(keys)} | 143 » lme := errors.LazyMultiError{Size: len(keys)} |
145 for i, k := range keys { | 144 for i, k := range keys { |
146 buf := &bytes.Buffer{} | 145 buf := &bytes.Buffer{} |
147 data[i].Write(buf, rds.WithoutContext) | 146 data[i].Write(buf, rds.WithoutContext) |
148 dataBytes := buf.Bytes() | 147 dataBytes := buf.Bytes() |
149 | 148 |
150 rKey, err := func() (ret rds.Key, err error) { | 149 rKey, err := func() (ret rds.Key, err error) { |
151 d.rwlock.Lock() | 150 d.rwlock.Lock() |
152 defer d.rwlock.Unlock() | 151 defer d.rwlock.Unlock() |
153 | 152 |
154 ents, ret := d.entsKeyLocked(k) | 153 ents, ret := d.entsKeyLocked(k) |
(...skipping 14 matching lines...) Expand all Loading... |
169 retKeys[i] = rKey | 168 retKeys[i] = rKey |
170 } | 169 } |
171 return retKeys, lme.Get() | 170 return retKeys, lme.Get() |
172 } | 171 } |
173 | 172 |
174 func getMultiInner(ns string, keys []rds.Key, plss []rds.PropertyLoadSaver, getC
oll func() (*memCollection, error)) error { | 173 func getMultiInner(ns string, keys []rds.Key, plss []rds.PropertyLoadSaver, getC
oll func() (*memCollection, error)) error { |
175 if err := multiValid(keys, plss, ns, false, true); err != nil { | 174 if err := multiValid(keys, plss, ns, false, true); err != nil { |
176 return err | 175 return err |
177 } | 176 } |
178 | 177 |
179 » lme := gae.LazyMultiError{Size: len(keys)} | 178 » lme := errors.LazyMultiError{Size: len(keys)} |
180 | 179 |
181 ents, err := getColl() | 180 ents, err := getColl() |
182 if err != nil { | 181 if err != nil { |
183 return err | 182 return err |
184 } | 183 } |
185 if ents == nil { | 184 if ents == nil { |
186 for i := range keys { | 185 for i := range keys { |
187 lme.Assign(i, rds.ErrNoSuchEntity) | 186 lme.Assign(i, rds.ErrNoSuchEntity) |
188 } | 187 } |
189 return lme.Get() | 188 return lme.Get() |
(...skipping 11 matching lines...) Expand all Loading... |
201 lme.Assign(i, err) | 200 lme.Assign(i, err) |
202 continue | 201 continue |
203 } | 202 } |
204 | 203 |
205 lme.Assign(i, plss[i].Load(got)) | 204 lme.Assign(i, plss[i].Load(got)) |
206 } | 205 } |
207 return lme.Get() | 206 return lme.Get() |
208 } | 207 } |
209 | 208 |
210 func (d *dataStoreData) get(ns string, key rds.Key, pls rds.PropertyLoadSaver) e
rror { | 209 func (d *dataStoreData) get(ns string, key rds.Key, pls rds.PropertyLoadSaver) e
rror { |
211 » return gae.SingleError(d.getMulti(ns, []rds.Key{key}, []rds.PropertyLoad
Saver{pls})) | 210 » return errors.SingleError(d.getMulti(ns, []rds.Key{key}, []rds.PropertyL
oadSaver{pls})) |
212 } | 211 } |
213 | 212 |
214 func (d *dataStoreData) getMulti(ns string, keys []rds.Key, plss []rds.PropertyL
oadSaver) error { | 213 func (d *dataStoreData) getMulti(ns string, keys []rds.Key, plss []rds.PropertyL
oadSaver) error { |
215 return getMultiInner(ns, keys, plss, func() (*memCollection, error) { | 214 return getMultiInner(ns, keys, plss, func() (*memCollection, error) { |
216 d.rwlock.RLock() | 215 d.rwlock.RLock() |
217 s := d.store.Snapshot() | 216 s := d.store.Snapshot() |
218 d.rwlock.RUnlock() | 217 d.rwlock.RUnlock() |
219 | 218 |
220 return s.GetCollection("ents:" + ns), nil | 219 return s.GetCollection("ents:" + ns), nil |
221 }) | 220 }) |
222 } | 221 } |
223 | 222 |
224 func (d *dataStoreData) del(ns string, key rds.Key) (err error) { | 223 func (d *dataStoreData) del(ns string, key rds.Key) (err error) { |
225 » return gae.SingleError(d.delMulti(ns, []rds.Key{key})) | 224 » return errors.SingleError(d.delMulti(ns, []rds.Key{key})) |
226 } | 225 } |
227 | 226 |
228 func (d *dataStoreData) delMulti(ns string, keys []rds.Key) error { | 227 func (d *dataStoreData) delMulti(ns string, keys []rds.Key) error { |
229 » lme := gae.LazyMultiError{Size: len(keys)} | 228 » lme := errors.LazyMultiError{Size: len(keys)} |
230 toDel := make([][]byte, 0, len(keys)) | 229 toDel := make([][]byte, 0, len(keys)) |
231 for i, k := range keys { | 230 for i, k := range keys { |
232 if !rds.KeyValid(k, ns, false) { | 231 if !rds.KeyValid(k, ns, false) { |
233 lme.Assign(i, rds.ErrInvalidKey) | 232 lme.Assign(i, rds.ErrInvalidKey) |
234 continue | 233 continue |
235 } | 234 } |
236 toDel = append(toDel, keyBytes(rds.WithoutContext, k)) | 235 toDel = append(toDel, keyBytes(rds.WithoutContext, k)) |
237 } | 236 } |
238 err := lme.Get() | 237 err := lme.Get() |
239 if err != nil { | 238 if err != nil { |
(...skipping 171 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
411 } | 410 } |
412 | 411 |
413 return nil | 412 return nil |
414 } | 413 } |
415 | 414 |
416 func (td *txnDataStoreData) put(ns string, key rds.Key, pls rds.PropertyLoadSave
r) (rds.Key, error) { | 415 func (td *txnDataStoreData) put(ns string, key rds.Key, pls rds.PropertyLoadSave
r) (rds.Key, error) { |
417 keys, errs := td.putMulti(ns, []rds.Key{key}, []rds.PropertyLoadSaver{pl
s}) | 416 keys, errs := td.putMulti(ns, []rds.Key{key}, []rds.PropertyLoadSaver{pl
s}) |
418 if errs == nil { | 417 if errs == nil { |
419 return keys[0], nil | 418 return keys[0], nil |
420 } | 419 } |
421 » return nil, gae.SingleError(errs) | 420 » return nil, errors.SingleError(errs) |
422 } | 421 } |
423 | 422 |
424 func (td *txnDataStoreData) putMulti(ns string, keys []rds.Key, plss []rds.Prope
rtyLoadSaver) ([]rds.Key, error) { | 423 func (td *txnDataStoreData) putMulti(ns string, keys []rds.Key, plss []rds.Prope
rtyLoadSaver) ([]rds.Key, error) { |
425 pmaps, err := putMultiPrelim(ns, keys, plss) | 424 pmaps, err := putMultiPrelim(ns, keys, plss) |
426 if err != nil { | 425 if err != nil { |
427 return nil, err | 426 return nil, err |
428 } | 427 } |
429 | 428 |
430 retKeys := make([]rds.Key, len(keys)) | 429 retKeys := make([]rds.Key, len(keys)) |
431 » lme := gae.LazyMultiError{Size: len(keys)} | 430 » lme := errors.LazyMultiError{Size: len(keys)} |
432 for i, k := range keys { | 431 for i, k := range keys { |
433 func() { | 432 func() { |
434 td.parent.Lock() | 433 td.parent.Lock() |
435 defer td.parent.Unlock() | 434 defer td.parent.Unlock() |
436 _, k = td.parent.entsKeyLocked(k) | 435 _, k = td.parent.entsKeyLocked(k) |
437 }() | 436 }() |
438 lme.Assign(i, td.writeMutation(false, k, pmaps[i])) | 437 lme.Assign(i, td.writeMutation(false, k, pmaps[i])) |
439 retKeys[i] = k | 438 retKeys[i] = k |
440 } | 439 } |
441 | 440 |
442 return retKeys, lme.Get() | 441 return retKeys, lme.Get() |
443 } | 442 } |
444 | 443 |
445 func (td *txnDataStoreData) get(ns string, key rds.Key, pls rds.PropertyLoadSave
r) error { | 444 func (td *txnDataStoreData) get(ns string, key rds.Key, pls rds.PropertyLoadSave
r) error { |
446 » return gae.SingleError(td.getMulti(ns, []rds.Key{key}, []rds.PropertyLoa
dSaver{pls})) | 445 » return errors.SingleError(td.getMulti(ns, []rds.Key{key}, []rds.Property
LoadSaver{pls})) |
447 } | 446 } |
448 | 447 |
449 func (td *txnDataStoreData) getMulti(ns string, keys []rds.Key, plss []rds.Prope
rtyLoadSaver) error { | 448 func (td *txnDataStoreData) getMulti(ns string, keys []rds.Key, plss []rds.Prope
rtyLoadSaver) error { |
450 return getMultiInner(ns, keys, plss, func() (*memCollection, error) { | 449 return getMultiInner(ns, keys, plss, func() (*memCollection, error) { |
451 » » lme := gae.LazyMultiError{Size: len(keys)} | 450 » » lme := errors.LazyMultiError{Size: len(keys)} |
452 for i, k := range keys { | 451 for i, k := range keys { |
453 lme.Assign(i, td.writeMutation(true, k, nil)) | 452 lme.Assign(i, td.writeMutation(true, k, nil)) |
454 } | 453 } |
455 return td.snap.GetCollection("ents:" + ns), lme.Get() | 454 return td.snap.GetCollection("ents:" + ns), lme.Get() |
456 }) | 455 }) |
457 } | 456 } |
458 | 457 |
459 func (td *txnDataStoreData) del(ns string, key rds.Key) error { | 458 func (td *txnDataStoreData) del(ns string, key rds.Key) error { |
460 » return gae.SingleError(td.delMulti(ns, []rds.Key{key})) | 459 » return errors.SingleError(td.delMulti(ns, []rds.Key{key})) |
461 } | 460 } |
462 | 461 |
463 func (td *txnDataStoreData) delMulti(ns string, keys []rds.Key) error { | 462 func (td *txnDataStoreData) delMulti(ns string, keys []rds.Key) error { |
464 » lme := gae.LazyMultiError{Size: len(keys)} | 463 » lme := errors.LazyMultiError{Size: len(keys)} |
465 for i, k := range keys { | 464 for i, k := range keys { |
466 if !rds.KeyValid(k, ns, false) { | 465 if !rds.KeyValid(k, ns, false) { |
467 lme.Assign(i, rds.ErrInvalidKey) | 466 lme.Assign(i, rds.ErrInvalidKey) |
468 } else { | 467 } else { |
469 lme.Assign(i, td.writeMutation(false, k, nil)) | 468 lme.Assign(i, td.writeMutation(false, k, nil)) |
470 } | 469 } |
471 } | 470 } |
472 return lme.Get() | 471 return lme.Get() |
473 } | 472 } |
474 | 473 |
(...skipping 28 matching lines...) Expand all Loading... |
503 return rds.KeyValid(k, ns, allowSpecial) | 502 return rds.KeyValid(k, ns, allowSpecial) |
504 } | 503 } |
505 } | 504 } |
506 | 505 |
507 if keys == nil || plss == nil { | 506 if keys == nil || plss == nil { |
508 return errors.New("gae: key or plss slices were nil") | 507 return errors.New("gae: key or plss slices were nil") |
509 } | 508 } |
510 if len(keys) != len(plss) { | 509 if len(keys) != len(plss) { |
511 return errors.New("gae: key and dst slices have different length
") | 510 return errors.New("gae: key and dst slices have different length
") |
512 } | 511 } |
513 » lme := gae.LazyMultiError{Size: len(keys)} | 512 » lme := errors.LazyMultiError{Size: len(keys)} |
514 for i, k := range keys { | 513 for i, k := range keys { |
515 if !vfn(k) { | 514 if !vfn(k) { |
516 lme.Assign(i, rds.ErrInvalidKey) | 515 lme.Assign(i, rds.ErrInvalidKey) |
517 } | 516 } |
518 } | 517 } |
519 return lme.Get() | 518 return lme.Get() |
520 } | 519 } |
OLD | NEW |