| OLD | NEW |
| 1 // Copyright 2015 The Chromium Authors. All rights reserved. | 1 // Copyright 2015 The Chromium Authors. All rights reserved. |
| 2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
| 3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
| 4 | 4 |
| 5 package memory | 5 package memory |
| 6 | 6 |
| 7 import ( | 7 import ( |
| 8 "bytes" | 8 "bytes" |
| 9 "errors" | |
| 10 "fmt" | 9 "fmt" |
| 11 "sync" | 10 "sync" |
| 12 "sync/atomic" | 11 "sync/atomic" |
| 13 | 12 |
| 14 "github.com/luci/gae" | |
| 15 rds "github.com/luci/gae/service/rawdatastore" | 13 rds "github.com/luci/gae/service/rawdatastore" |
| 14 "github.com/luci/luci-go/common/errors" |
| 16 "golang.org/x/net/context" | 15 "golang.org/x/net/context" |
| 17 ) | 16 ) |
| 18 | 17 |
| 19 //////////////////////////////// dataStoreData ///////////////////////////////// | 18 //////////////////////////////// dataStoreData ///////////////////////////////// |
| 20 | 19 |
| 21 type dataStoreData struct { | 20 type dataStoreData struct { |
| 22 rwlock sync.RWMutex | 21 rwlock sync.RWMutex |
| 23 // See README.md for store schema. | 22 // See README.md for store schema. |
| 24 store *memStore | 23 store *memStore |
| 25 snap *memStore | 24 snap *memStore |
| (...skipping 80 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 106 } | 105 } |
| 107 | 106 |
| 108 return ents, key | 107 return ents, key |
| 109 } | 108 } |
| 110 | 109 |
| 111 func (d *dataStoreData) put(ns string, key rds.Key, pls rds.PropertyLoadSaver) (
rds.Key, error) { | 110 func (d *dataStoreData) put(ns string, key rds.Key, pls rds.PropertyLoadSaver) (
rds.Key, error) { |
| 112 keys, errs := d.putMulti(ns, []rds.Key{key}, []rds.PropertyLoadSaver{pls
}) | 111 keys, errs := d.putMulti(ns, []rds.Key{key}, []rds.PropertyLoadSaver{pls
}) |
| 113 if errs == nil { | 112 if errs == nil { |
| 114 return keys[0], nil | 113 return keys[0], nil |
| 115 } | 114 } |
| 116 » return nil, gae.SingleError(errs) | 115 » return nil, errors.SingleError(errs) |
| 117 } | 116 } |
| 118 | 117 |
| 119 func (d *dataStoreData) putMulti(ns string, keys []rds.Key, plss []rds.PropertyL
oadSaver) ([]rds.Key, error) { | 118 func (d *dataStoreData) putMulti(ns string, keys []rds.Key, plss []rds.PropertyL
oadSaver) ([]rds.Key, error) { |
| 120 pmaps, err := putMultiPrelim(ns, keys, plss) | 119 pmaps, err := putMultiPrelim(ns, keys, plss) |
| 121 if err != nil { | 120 if err != nil { |
| 122 return nil, err | 121 return nil, err |
| 123 } | 122 } |
| 124 return d.putMultiInner(keys, pmaps) | 123 return d.putMultiInner(keys, pmaps) |
| 125 } | 124 } |
| 126 | 125 |
| 127 func putMultiPrelim(ns string, keys []rds.Key, plss []rds.PropertyLoadSaver) ([]
rds.PropertyMap, error) { | 126 func putMultiPrelim(ns string, keys []rds.Key, plss []rds.PropertyLoadSaver) ([]
rds.PropertyMap, error) { |
| 128 err := multiValid(keys, plss, ns, true, false) | 127 err := multiValid(keys, plss, ns, true, false) |
| 129 if err != nil { | 128 if err != nil { |
| 130 return nil, err | 129 return nil, err |
| 131 } | 130 } |
| 132 pmaps := make([]rds.PropertyMap, len(keys)) | 131 pmaps := make([]rds.PropertyMap, len(keys)) |
| 133 » lme := gae.LazyMultiError{Size: len(keys)} | 132 » lme := errors.LazyMultiError{Size: len(keys)} |
| 134 for i, pls := range plss { | 133 for i, pls := range plss { |
| 135 pm, err := pls.Save(false) | 134 pm, err := pls.Save(false) |
| 136 lme.Assign(i, err) | 135 lme.Assign(i, err) |
| 137 pmaps[i] = pm | 136 pmaps[i] = pm |
| 138 } | 137 } |
| 139 return pmaps, lme.Get() | 138 return pmaps, lme.Get() |
| 140 } | 139 } |
| 141 | 140 |
| 142 func (d *dataStoreData) putMultiInner(keys []rds.Key, data []rds.PropertyMap) ([
]rds.Key, error) { | 141 func (d *dataStoreData) putMultiInner(keys []rds.Key, data []rds.PropertyMap) ([
]rds.Key, error) { |
| 143 retKeys := make([]rds.Key, len(keys)) | 142 retKeys := make([]rds.Key, len(keys)) |
| 144 » lme := gae.LazyMultiError{Size: len(keys)} | 143 » lme := errors.LazyMultiError{Size: len(keys)} |
| 145 for i, k := range keys { | 144 for i, k := range keys { |
| 146 buf := &bytes.Buffer{} | 145 buf := &bytes.Buffer{} |
| 147 data[i].Write(buf, rds.WithoutContext) | 146 data[i].Write(buf, rds.WithoutContext) |
| 148 dataBytes := buf.Bytes() | 147 dataBytes := buf.Bytes() |
| 149 | 148 |
| 150 rKey, err := func() (ret rds.Key, err error) { | 149 rKey, err := func() (ret rds.Key, err error) { |
| 151 d.rwlock.Lock() | 150 d.rwlock.Lock() |
| 152 defer d.rwlock.Unlock() | 151 defer d.rwlock.Unlock() |
| 153 | 152 |
| 154 ents, ret := d.entsKeyLocked(k) | 153 ents, ret := d.entsKeyLocked(k) |
| (...skipping 14 matching lines...) Expand all Loading... |
| 169 retKeys[i] = rKey | 168 retKeys[i] = rKey |
| 170 } | 169 } |
| 171 return retKeys, lme.Get() | 170 return retKeys, lme.Get() |
| 172 } | 171 } |
| 173 | 172 |
| 174 func getMultiInner(ns string, keys []rds.Key, plss []rds.PropertyLoadSaver, getC
oll func() (*memCollection, error)) error { | 173 func getMultiInner(ns string, keys []rds.Key, plss []rds.PropertyLoadSaver, getC
oll func() (*memCollection, error)) error { |
| 175 if err := multiValid(keys, plss, ns, false, true); err != nil { | 174 if err := multiValid(keys, plss, ns, false, true); err != nil { |
| 176 return err | 175 return err |
| 177 } | 176 } |
| 178 | 177 |
| 179 » lme := gae.LazyMultiError{Size: len(keys)} | 178 » lme := errors.LazyMultiError{Size: len(keys)} |
| 180 | 179 |
| 181 ents, err := getColl() | 180 ents, err := getColl() |
| 182 if err != nil { | 181 if err != nil { |
| 183 return err | 182 return err |
| 184 } | 183 } |
| 185 if ents == nil { | 184 if ents == nil { |
| 186 for i := range keys { | 185 for i := range keys { |
| 187 lme.Assign(i, rds.ErrNoSuchEntity) | 186 lme.Assign(i, rds.ErrNoSuchEntity) |
| 188 } | 187 } |
| 189 return lme.Get() | 188 return lme.Get() |
| (...skipping 11 matching lines...) Expand all Loading... |
| 201 lme.Assign(i, err) | 200 lme.Assign(i, err) |
| 202 continue | 201 continue |
| 203 } | 202 } |
| 204 | 203 |
| 205 lme.Assign(i, plss[i].Load(got)) | 204 lme.Assign(i, plss[i].Load(got)) |
| 206 } | 205 } |
| 207 return lme.Get() | 206 return lme.Get() |
| 208 } | 207 } |
| 209 | 208 |
| 210 func (d *dataStoreData) get(ns string, key rds.Key, pls rds.PropertyLoadSaver) e
rror { | 209 func (d *dataStoreData) get(ns string, key rds.Key, pls rds.PropertyLoadSaver) e
rror { |
| 211 » return gae.SingleError(d.getMulti(ns, []rds.Key{key}, []rds.PropertyLoad
Saver{pls})) | 210 » return errors.SingleError(d.getMulti(ns, []rds.Key{key}, []rds.PropertyL
oadSaver{pls})) |
| 212 } | 211 } |
| 213 | 212 |
| 214 func (d *dataStoreData) getMulti(ns string, keys []rds.Key, plss []rds.PropertyL
oadSaver) error { | 213 func (d *dataStoreData) getMulti(ns string, keys []rds.Key, plss []rds.PropertyL
oadSaver) error { |
| 215 return getMultiInner(ns, keys, plss, func() (*memCollection, error) { | 214 return getMultiInner(ns, keys, plss, func() (*memCollection, error) { |
| 216 d.rwlock.RLock() | 215 d.rwlock.RLock() |
| 217 s := d.store.Snapshot() | 216 s := d.store.Snapshot() |
| 218 d.rwlock.RUnlock() | 217 d.rwlock.RUnlock() |
| 219 | 218 |
| 220 return s.GetCollection("ents:" + ns), nil | 219 return s.GetCollection("ents:" + ns), nil |
| 221 }) | 220 }) |
| 222 } | 221 } |
| 223 | 222 |
| 224 func (d *dataStoreData) del(ns string, key rds.Key) (err error) { | 223 func (d *dataStoreData) del(ns string, key rds.Key) (err error) { |
| 225 » return gae.SingleError(d.delMulti(ns, []rds.Key{key})) | 224 » return errors.SingleError(d.delMulti(ns, []rds.Key{key})) |
| 226 } | 225 } |
| 227 | 226 |
| 228 func (d *dataStoreData) delMulti(ns string, keys []rds.Key) error { | 227 func (d *dataStoreData) delMulti(ns string, keys []rds.Key) error { |
| 229 » lme := gae.LazyMultiError{Size: len(keys)} | 228 » lme := errors.LazyMultiError{Size: len(keys)} |
| 230 toDel := make([][]byte, 0, len(keys)) | 229 toDel := make([][]byte, 0, len(keys)) |
| 231 for i, k := range keys { | 230 for i, k := range keys { |
| 232 if !rds.KeyValid(k, ns, false) { | 231 if !rds.KeyValid(k, ns, false) { |
| 233 lme.Assign(i, rds.ErrInvalidKey) | 232 lme.Assign(i, rds.ErrInvalidKey) |
| 234 continue | 233 continue |
| 235 } | 234 } |
| 236 toDel = append(toDel, keyBytes(rds.WithoutContext, k)) | 235 toDel = append(toDel, keyBytes(rds.WithoutContext, k)) |
| 237 } | 236 } |
| 238 err := lme.Get() | 237 err := lme.Get() |
| 239 if err != nil { | 238 if err != nil { |
| (...skipping 171 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 411 } | 410 } |
| 412 | 411 |
| 413 return nil | 412 return nil |
| 414 } | 413 } |
| 415 | 414 |
| 416 func (td *txnDataStoreData) put(ns string, key rds.Key, pls rds.PropertyLoadSave
r) (rds.Key, error) { | 415 func (td *txnDataStoreData) put(ns string, key rds.Key, pls rds.PropertyLoadSave
r) (rds.Key, error) { |
| 417 keys, errs := td.putMulti(ns, []rds.Key{key}, []rds.PropertyLoadSaver{pl
s}) | 416 keys, errs := td.putMulti(ns, []rds.Key{key}, []rds.PropertyLoadSaver{pl
s}) |
| 418 if errs == nil { | 417 if errs == nil { |
| 419 return keys[0], nil | 418 return keys[0], nil |
| 420 } | 419 } |
| 421 » return nil, gae.SingleError(errs) | 420 » return nil, errors.SingleError(errs) |
| 422 } | 421 } |
| 423 | 422 |
| 424 func (td *txnDataStoreData) putMulti(ns string, keys []rds.Key, plss []rds.Prope
rtyLoadSaver) ([]rds.Key, error) { | 423 func (td *txnDataStoreData) putMulti(ns string, keys []rds.Key, plss []rds.Prope
rtyLoadSaver) ([]rds.Key, error) { |
| 425 pmaps, err := putMultiPrelim(ns, keys, plss) | 424 pmaps, err := putMultiPrelim(ns, keys, plss) |
| 426 if err != nil { | 425 if err != nil { |
| 427 return nil, err | 426 return nil, err |
| 428 } | 427 } |
| 429 | 428 |
| 430 retKeys := make([]rds.Key, len(keys)) | 429 retKeys := make([]rds.Key, len(keys)) |
| 431 » lme := gae.LazyMultiError{Size: len(keys)} | 430 » lme := errors.LazyMultiError{Size: len(keys)} |
| 432 for i, k := range keys { | 431 for i, k := range keys { |
| 433 func() { | 432 func() { |
| 434 td.parent.Lock() | 433 td.parent.Lock() |
| 435 defer td.parent.Unlock() | 434 defer td.parent.Unlock() |
| 436 _, k = td.parent.entsKeyLocked(k) | 435 _, k = td.parent.entsKeyLocked(k) |
| 437 }() | 436 }() |
| 438 lme.Assign(i, td.writeMutation(false, k, pmaps[i])) | 437 lme.Assign(i, td.writeMutation(false, k, pmaps[i])) |
| 439 retKeys[i] = k | 438 retKeys[i] = k |
| 440 } | 439 } |
| 441 | 440 |
| 442 return retKeys, lme.Get() | 441 return retKeys, lme.Get() |
| 443 } | 442 } |
| 444 | 443 |
| 445 func (td *txnDataStoreData) get(ns string, key rds.Key, pls rds.PropertyLoadSave
r) error { | 444 func (td *txnDataStoreData) get(ns string, key rds.Key, pls rds.PropertyLoadSave
r) error { |
| 446 » return gae.SingleError(td.getMulti(ns, []rds.Key{key}, []rds.PropertyLoa
dSaver{pls})) | 445 » return errors.SingleError(td.getMulti(ns, []rds.Key{key}, []rds.Property
LoadSaver{pls})) |
| 447 } | 446 } |
| 448 | 447 |
| 449 func (td *txnDataStoreData) getMulti(ns string, keys []rds.Key, plss []rds.Prope
rtyLoadSaver) error { | 448 func (td *txnDataStoreData) getMulti(ns string, keys []rds.Key, plss []rds.Prope
rtyLoadSaver) error { |
| 450 return getMultiInner(ns, keys, plss, func() (*memCollection, error) { | 449 return getMultiInner(ns, keys, plss, func() (*memCollection, error) { |
| 451 » » lme := gae.LazyMultiError{Size: len(keys)} | 450 » » lme := errors.LazyMultiError{Size: len(keys)} |
| 452 for i, k := range keys { | 451 for i, k := range keys { |
| 453 lme.Assign(i, td.writeMutation(true, k, nil)) | 452 lme.Assign(i, td.writeMutation(true, k, nil)) |
| 454 } | 453 } |
| 455 return td.snap.GetCollection("ents:" + ns), lme.Get() | 454 return td.snap.GetCollection("ents:" + ns), lme.Get() |
| 456 }) | 455 }) |
| 457 } | 456 } |
| 458 | 457 |
| 459 func (td *txnDataStoreData) del(ns string, key rds.Key) error { | 458 func (td *txnDataStoreData) del(ns string, key rds.Key) error { |
| 460 » return gae.SingleError(td.delMulti(ns, []rds.Key{key})) | 459 » return errors.SingleError(td.delMulti(ns, []rds.Key{key})) |
| 461 } | 460 } |
| 462 | 461 |
| 463 func (td *txnDataStoreData) delMulti(ns string, keys []rds.Key) error { | 462 func (td *txnDataStoreData) delMulti(ns string, keys []rds.Key) error { |
| 464 » lme := gae.LazyMultiError{Size: len(keys)} | 463 » lme := errors.LazyMultiError{Size: len(keys)} |
| 465 for i, k := range keys { | 464 for i, k := range keys { |
| 466 if !rds.KeyValid(k, ns, false) { | 465 if !rds.KeyValid(k, ns, false) { |
| 467 lme.Assign(i, rds.ErrInvalidKey) | 466 lme.Assign(i, rds.ErrInvalidKey) |
| 468 } else { | 467 } else { |
| 469 lme.Assign(i, td.writeMutation(false, k, nil)) | 468 lme.Assign(i, td.writeMutation(false, k, nil)) |
| 470 } | 469 } |
| 471 } | 470 } |
| 472 return lme.Get() | 471 return lme.Get() |
| 473 } | 472 } |
| 474 | 473 |
| (...skipping 28 matching lines...) Expand all Loading... |
| 503 return rds.KeyValid(k, ns, allowSpecial) | 502 return rds.KeyValid(k, ns, allowSpecial) |
| 504 } | 503 } |
| 505 } | 504 } |
| 506 | 505 |
| 507 if keys == nil || plss == nil { | 506 if keys == nil || plss == nil { |
| 508 return errors.New("gae: key or plss slices were nil") | 507 return errors.New("gae: key or plss slices were nil") |
| 509 } | 508 } |
| 510 if len(keys) != len(plss) { | 509 if len(keys) != len(plss) { |
| 511 return errors.New("gae: key and dst slices have different length
") | 510 return errors.New("gae: key and dst slices have different length
") |
| 512 } | 511 } |
| 513 » lme := gae.LazyMultiError{Size: len(keys)} | 512 » lme := errors.LazyMultiError{Size: len(keys)} |
| 514 for i, k := range keys { | 513 for i, k := range keys { |
| 515 if !vfn(k) { | 514 if !vfn(k) { |
| 516 lme.Assign(i, rds.ErrInvalidKey) | 515 lme.Assign(i, rds.ErrInvalidKey) |
| 517 } | 516 } |
| 518 } | 517 } |
| 519 return lme.Get() | 518 return lme.Get() |
| 520 } | 519 } |
| OLD | NEW |