Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(38)

Issues Search
    My Issues | Starred     Open | Closed | All

Side by Side Diff: service/datastore/datastore.go

Issue 2302743002: Interface update, per-method Contexts. (Closed)
Patch Set: Created 4 years, 3 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
« service/datastore/context.go ('K') | « service/datastore/context_test.go ('k') | service/datastore/datastore_test.go » ('j') | service/datastore/interface.go » ('J')
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 // Copyright 2015 The LUCI Authors. All rights reserved. 1 // Copyright 2015 The LUCI Authors. All rights reserved.
2 // Use of this source code is governed under the Apache License, Version 2.0 2 // Use of this source code is governed under the Apache License, Version 2.0
3 // that can be found in the LICENSE file. 3 // that can be found in the LICENSE file.
4 4
5 package datastore 5 package datastore
6 6
7 import ( 7 import (
8 "fmt" 8 "fmt"
9 "io" 9 "io"
10 "io/ioutil" 10 "io/ioutil"
11 "os" 11 "os"
12 "path/filepath" 12 "path/filepath"
13 "reflect"
14 "runtime" 13 "runtime"
15 "strings" 14 "strings"
16 15
17 "github.com/luci/luci-go/common/errors"
18
19 "gopkg.in/yaml.v2" 16 "gopkg.in/yaml.v2"
20 ) 17 )
21 18
22 type datastoreImpl struct {
23 RawInterface
24
25 aid string
26 ns string
27 }
28
29 var _ Interface = (*datastoreImpl)(nil)
30
31 func (d *datastoreImpl) KeyForObj(src interface{}) *Key {
32 ret, err := d.KeyForObjErr(src)
33 if err != nil {
34 panic(err)
35 }
36 return ret
37 }
38
39 func (d *datastoreImpl) KeyForObjErr(src interface{}) (*Key, error) {
dnj 2016/09/01 15:25:40 These all moved into key.go.
40 return newKeyObjErr(d.aid, d.ns, getMGS(src))
41 }
42
43 func (d *datastoreImpl) MakeKey(elems ...interface{}) *Key {
44 return MakeKey(d.aid, d.ns, elems...)
45 }
46
47 func (d *datastoreImpl) NewKey(kind, stringID string, intID int64, parent *Key) *Key {
48 return NewKey(d.aid, d.ns, kind, stringID, intID, parent)
49 }
50
51 func (d *datastoreImpl) NewIncompleteKeys(count int, kind string, parent *Key) ( keys []*Key) {
52 if count > 0 {
53 keys = make([]*Key, count)
54 for i := range keys {
55 keys[i] = d.NewKey(kind, "", 0, parent)
56 }
57 }
58 return
59 }
60
61 func (d *datastoreImpl) NewKeyToks(toks []KeyTok) *Key {
62 return NewKeyToks(d.aid, d.ns, toks)
63 }
64
65 // PopulateKey loads key into obj.
66 //
67 // obj is any object that Interface.Get is able to accept.
68 //
69 // Upon successful application, this method will return true. If the key could
70 // not be applied to the object, this method will return false. It will panic if
71 // obj is an invalid datastore model.
72 //
73 // This method will panic if obj is an invalid datastore model. If the key could
74 // not be applied to the object, nothing will happen.
75 func PopulateKey(obj interface{}, key *Key) bool {
dnj 2016/09/01 15:25:40 This all moved into "interface.go".
76 return populateKeyMGS(getMGS(obj), key)
77 }
78
79 func populateKeyMGS(mgs MetaGetterSetter, key *Key) bool {
80 if mgs.SetMeta("key", key) {
81 return true
82 }
83
84 lst := key.LastTok()
85 if lst.StringID != "" {
86 if !mgs.SetMeta("id", lst.StringID) {
87 return false
88 }
89 } else {
90 if !mgs.SetMeta("id", lst.IntID) {
91 return false
92 }
93 }
94
95 mgs.SetMeta("kind", lst.Kind)
96 mgs.SetMeta("parent", key.Parent())
97 return true
98 }
99
100 func checkMultiSliceType(v interface{}) error {
101 if reflect.TypeOf(v).Kind() == reflect.Slice {
102 return nil
103 }
104 return fmt.Errorf("argument must be a slice, not %T", v)
105
106 }
107
108 func runParseCallback(cbIface interface{}) (isKey, hasErr, hasCursorCB bool, mat *multiArgType) {
109 badSig := func() {
110 panic(fmt.Errorf(
111 "cb does not match the required callback signature: `%T` != `func(TYPE, [CursorCB]) [error]`",
112 cbIface))
113 }
114
115 if cbIface == nil {
116 badSig()
117 }
118
119 // TODO(riannucci): Profile and determine if any of this is causing a re al
120 // slowdown. Could potentially cache reflection stuff by cbTyp?
121 cbTyp := reflect.TypeOf(cbIface)
122
123 if cbTyp.Kind() != reflect.Func {
124 badSig()
125 }
126
127 numIn := cbTyp.NumIn()
128 if numIn != 1 && numIn != 2 {
129 badSig()
130 }
131
132 firstArg := cbTyp.In(0)
133 if firstArg == typeOfKey {
134 isKey = true
135 } else {
136 mat = mustParseArg(firstArg, false)
137 if mat.newElem == nil {
138 badSig()
139 }
140 }
141
142 hasCursorCB = numIn == 2
143 if hasCursorCB && cbTyp.In(1) != typeOfCursorCB {
144 badSig()
145 }
146
147 if cbTyp.NumOut() > 1 {
148 badSig()
149 } else if cbTyp.NumOut() == 1 && cbTyp.Out(0) != typeOfError {
150 badSig()
151 }
152 hasErr = cbTyp.NumOut() == 1
153
154 return
155 }
156
157 func (d *datastoreImpl) AllocateIDs(ent ...interface{}) error {
158 if len(ent) == 0 {
159 return nil
160 }
161
162 mma, err := makeMetaMultiArg(ent, mmaWriteKeys)
163 if err != nil {
164 panic(err)
165 }
166
167 keys, _, err := mma.getKeysPMs(d.aid, d.ns, false)
168 if err != nil {
169 return err
170 }
171 if len(keys) == 0 {
172 return nil
173 }
174
175 // Convert each key to be partial valid, assigning an integer ID of 0. C onfirm
176 // that each object can be populated with such a key.
177 for i, key := range keys {
178 keys[i] = key.Incomplete()
179 }
180
181 var et errorTracker
182 it := mma.iterator(et.init(mma))
183 err = filterStop(d.RawInterface.AllocateIDs(keys, func(key *Key, err err or) error {
184 it.next(func(mat *multiArgType, v reflect.Value) error {
185 if err != nil {
186 return err
187 }
188
189 if !mat.setKey(v, key) {
190 return ErrInvalidKey
191 }
192 return nil
193 })
194
195 return nil
196 }))
197 if err == nil {
198 err = et.error()
199
200 if err != nil && len(ent) == 1 {
201 // Single-argument Exists will return a single error.
202 err = errors.SingleError(err)
203 }
204 }
205 return err
206 }
207
208 func (d *datastoreImpl) Run(q *Query, cbIface interface{}) error {
209 isKey, hasErr, hasCursorCB, mat := runParseCallback(cbIface)
210
211 if isKey {
212 q = q.KeysOnly(true)
213 }
214 fq, err := q.Finalize()
215 if err != nil {
216 return err
217 }
218
219 cbVal := reflect.ValueOf(cbIface)
220 var cb func(reflect.Value, CursorCB) error
221 switch {
222 case hasErr && hasCursorCB:
223 cb = func(v reflect.Value, cb CursorCB) error {
224 err := cbVal.Call([]reflect.Value{v, reflect.ValueOf(cb) })[0].Interface()
225 if err != nil {
226 return err.(error)
227 }
228 return nil
229 }
230
231 case hasErr && !hasCursorCB:
232 cb = func(v reflect.Value, _ CursorCB) error {
233 err := cbVal.Call([]reflect.Value{v})[0].Interface()
234 if err != nil {
235 return err.(error)
236 }
237 return nil
238 }
239
240 case !hasErr && hasCursorCB:
241 cb = func(v reflect.Value, cb CursorCB) error {
242 cbVal.Call([]reflect.Value{v, reflect.ValueOf(cb)})
243 return nil
244 }
245
246 case !hasErr && !hasCursorCB:
247 cb = func(v reflect.Value, _ CursorCB) error {
248 cbVal.Call([]reflect.Value{v})
249 return nil
250 }
251 }
252
253 if isKey {
254 err = d.RawInterface.Run(fq, func(k *Key, _ PropertyMap, gc Curs orCB) error {
255 return cb(reflect.ValueOf(k), gc)
256 })
257 } else {
258 err = d.RawInterface.Run(fq, func(k *Key, pm PropertyMap, gc Cur sorCB) error {
259 itm := mat.newElem()
260 if err := mat.setPM(itm, pm); err != nil {
261 return err
262 }
263 mat.setKey(itm, k)
264 return cb(itm, gc)
265 })
266 }
267 return filterStop(err)
268 }
269
270 func (d *datastoreImpl) Count(q *Query) (int64, error) {
271 fq, err := q.Finalize()
272 if err != nil {
273 return 0, err
274 }
275 v, err := d.RawInterface.Count(fq)
276 return v, filterStop(err)
277 }
278
279 func (d *datastoreImpl) GetAll(q *Query, dst interface{}) error {
280 v := reflect.ValueOf(dst)
281 if v.Kind() != reflect.Ptr {
282 panic(fmt.Errorf("invalid GetAll dst: must have a ptr-to-slice: %T", dst))
283 }
284 if !v.IsValid() || v.IsNil() {
285 panic(errors.New("invalid GetAll dst: <nil>"))
286 }
287
288 if keys, ok := dst.(*[]*Key); ok {
289 fq, err := q.KeysOnly(true).Finalize()
290 if err != nil {
291 return err
292 }
293
294 return d.RawInterface.Run(fq, func(k *Key, _ PropertyMap, _ Curs orCB) error {
295 *keys = append(*keys, k)
296 return nil
297 })
298 }
299 fq, err := q.Finalize()
300 if err != nil {
301 return err
302 }
303
304 slice := v.Elem()
305 mat := mustParseMultiArg(slice.Type())
306 if mat.newElem == nil {
307 panic(fmt.Errorf("invalid GetAll dst (non-concrete element type) : %T", dst))
308 }
309
310 errs := map[int]error{}
311 i := 0
312 err = filterStop(d.RawInterface.Run(fq, func(k *Key, pm PropertyMap, _ C ursorCB) error {
313 slice.Set(reflect.Append(slice, mat.newElem()))
314 itm := slice.Index(i)
315 mat.setKey(itm, k)
316 err := mat.setPM(itm, pm)
317 if err != nil {
318 errs[i] = err
319 }
320 i++
321 return nil
322 }))
323 if err == nil {
324 if len(errs) > 0 {
325 me := make(errors.MultiError, slice.Len())
326 for i, e := range errs {
327 me[i] = e
328 }
329 err = me
330 }
331 }
332 return err
333 }
334
335 func (d *datastoreImpl) Exists(ent ...interface{}) (*ExistsResult, error) {
336 if len(ent) == 0 {
337 return nil, nil
338 }
339
340 mma, err := makeMetaMultiArg(ent, mmaKeysOnly)
341 if err != nil {
342 panic(err)
343 }
344
345 keys, _, err := mma.getKeysPMs(d.aid, d.ns, false)
346 if err != nil {
347 return nil, err
348 }
349 if len(keys) == 0 {
350 return nil, nil
351 }
352
353 var bt boolTracker
354 it := mma.iterator(bt.init(mma))
355 err = filterStop(d.RawInterface.GetMulti(keys, nil, func(_ PropertyMap, err error) error {
356 it.next(func(*multiArgType, reflect.Value) error {
357 return err
358 })
359 return nil
360 }))
361 if err == nil {
362 err = bt.error()
363
364 if err != nil && len(ent) == 1 {
365 // Single-argument Exists will return a single error.
366 err = errors.SingleError(err)
367 }
368 }
369 return bt.result(), err
370 }
371
372 func (d *datastoreImpl) ExistsMulti(keys []*Key) (BoolList, error) {
373 v, err := d.Exists(keys)
374 if err != nil {
375 return nil, err
376 }
377 return v.List(0), nil
378 }
379
380 func (d *datastoreImpl) Get(dst ...interface{}) (err error) {
381 if len(dst) == 0 {
382 return nil
383 }
384
385 mma, err := makeMetaMultiArg(dst, mmaReadWrite)
386 if err != nil {
387 panic(err)
388 }
389
390 keys, pms, err := mma.getKeysPMs(d.aid, d.ns, true)
391 if err != nil {
392 return err
393 }
394 if len(keys) == 0 {
395 return nil
396 }
397
398 var et errorTracker
399 it := mma.iterator(et.init(mma))
400 meta := NewMultiMetaGetter(pms)
401 err = filterStop(d.RawInterface.GetMulti(keys, meta, func(pm PropertyMap , err error) error {
402 it.next(func(mat *multiArgType, slot reflect.Value) error {
403 if err != nil {
404 return err
405 }
406 return mat.setPM(slot, pm)
407 })
408 return nil
409 }))
410
411 if err == nil {
412 err = et.error()
413
414 if err != nil && len(dst) == 1 {
415 // Single-argument Get will return a single error.
416 err = errors.SingleError(err)
417 }
418 }
419 return err
420 }
421
422 func (d *datastoreImpl) GetMulti(dst interface{}) error {
423 if err := checkMultiSliceType(dst); err != nil {
424 panic(err)
425 }
426 return d.Get(dst)
427 }
428
429 func (d *datastoreImpl) Put(src ...interface{}) (err error) {
430 if len(src) == 0 {
431 return nil
432 }
433
434 mma, err := makeMetaMultiArg(src, mmaReadWrite)
435 if err != nil {
436 panic(err)
437 }
438
439 keys, vals, err := mma.getKeysPMs(d.aid, d.ns, false)
440 if err != nil {
441 return err
442 }
443 if len(keys) == 0 {
444 return nil
445 }
446
447 i := 0
448 var et errorTracker
449 it := mma.iterator(et.init(mma))
450 err = filterStop(d.RawInterface.PutMulti(keys, vals, func(key *Key, err error) error {
451 it.next(func(mat *multiArgType, slot reflect.Value) error {
452 if err != nil {
453 return err
454 }
455 if key != keys[i] {
456 mat.setKey(slot, key)
457 }
458 return nil
459 })
460
461 i++
462 return nil
463 }))
464
465 if err == nil {
466 err = et.error()
467
468 if err != nil && len(src) == 1 {
469 // Single-argument Put will return a single error.
470 err = errors.SingleError(err)
471 }
472 }
473 return err
474 }
475
476 func (d *datastoreImpl) PutMulti(src interface{}) error {
477 if err := checkMultiSliceType(src); err != nil {
478 panic(err)
479 }
480 return d.Put(src)
481 }
482
483 func (d *datastoreImpl) Delete(ent ...interface{}) error {
484 if len(ent) == 0 {
485 return nil
486 }
487
488 mma, err := makeMetaMultiArg(ent, mmaKeysOnly)
489 if err != nil {
490 panic(err)
491 }
492
493 keys, _, err := mma.getKeysPMs(d.aid, d.ns, false)
494 if err != nil {
495 return err
496 }
497 if len(keys) == 0 {
498 return nil
499 }
500
501 var et errorTracker
502 it := mma.iterator(et.init(mma))
503 err = filterStop(d.RawInterface.DeleteMulti(keys, func(err error) error {
504 it.next(func(*multiArgType, reflect.Value) error {
505 return err
506 })
507
508 return nil
509 }))
510 if err == nil {
511 err = et.error()
512
513 if err != nil && len(ent) == 1 {
514 // Single-argument Delete will return a single error.
515 err = errors.SingleError(err)
516 }
517 }
518 return err
519 }
520
521 func (d *datastoreImpl) DeleteMulti(keys []*Key) error {
522 return d.Delete(keys)
523 }
524
525 func (d *datastoreImpl) Raw() RawInterface {
526 return d.RawInterface
527 }
528
529 // ParseIndexYAML parses the contents of a index YAML file into a list of 19 // ParseIndexYAML parses the contents of a index YAML file into a list of
530 // IndexDefinitions. 20 // IndexDefinitions.
531 func ParseIndexYAML(content io.Reader) ([]*IndexDefinition, error) { 21 func ParseIndexYAML(content io.Reader) ([]*IndexDefinition, error) {
532 serialized, err := ioutil.ReadAll(content) 22 serialized, err := ioutil.ReadAll(content)
533 if err != nil { 23 if err != nil {
534 return nil, err 24 return nil, err
535 } 25 }
536 26
537 var m map[string][]*IndexDefinition 27 var m map[string][]*IndexDefinition
538 if err := yaml.Unmarshal(serialized, &m); err != nil { 28 if err := yaml.Unmarshal(serialized, &m); err != nil {
(...skipping 62 matching lines...) Expand 10 before | Expand all | Expand 10 after
601 } 91 }
602 } 92 }
603 93
604 if isRoot(currentDir) { 94 if isRoot(currentDir) {
605 return nil, fmt.Errorf("datastore: failed to find index YAML file") 95 return nil, fmt.Errorf("datastore: failed to find index YAML file")
606 } 96 }
607 97
608 currentDir = filepath.Dir(currentDir) 98 currentDir = filepath.Dir(currentDir)
609 } 99 }
610 } 100 }
611
612 func filterStop(err error) error {
613 if err == Stop {
614 err = nil
615 }
616 return err
617 }
OLDNEW
« service/datastore/context.go ('K') | « service/datastore/context_test.go ('k') | service/datastore/datastore_test.go » ('j') | service/datastore/interface.go » ('J')

Powered by Google App Engine
This is Rietveld 408576698