| OLD | NEW |
|---|
| (Empty) |
| 1 // Copyright 2015 The Chromium Authors. All rights reserved. | |
| 2 // Use of this source code is governed by a BSD-style license that can be | |
| 3 // found in the LICENSE file. | |
| 4 | |
| 5 package datastore | |
| 6 | |
| 7 import ( | |
| 8 "bytes" | |
| 9 "errors" | |
| 10 "fmt" | |
| 11 "sort" | |
| 12 "time" | |
| 13 | |
| 14 "github.com/luci/gae/service/blobstore" | |
| 15 "github.com/luci/luci-go/common/cmpbin" | |
| 16 ) | |
| 17 | |
| 18 const MaxIndexColumns = 64 | |
| 19 | |
| 20 // WritePropertyMapDeterministic allows tests to make WritePropertyMap | |
| 21 // deterministic. | |
| 22 var WritePropertyMapDeterministic = false | |
| 23 | |
| 24 // ReadPropertyMapReasonableLimit sets a limit on the number of rows and | |
| 25 // number of properties per row which can be read by ReadPropertyMap. The | |
| 26 // total number of Property objects readable by this method is this number | |
| 27 // squared (e.g. Limit rows * Limit properties) | |
| 28 const ReadPropertyMapReasonableLimit uint64 = 30000 | |
| 29 | |
| 30 // ReadKeyNumToksReasonableLimit is the maximum number of Key tokens that | |
| 31 // ReadKey is willing to read for a single key. | |
| 32 const ReadKeyNumToksReasonableLimit uint64 = 50 | |
| 33 | |
| 34 // KeyContext controls whether the various Write and Read serializtion | |
| 35 // routines should encode the context of Keys (read: the appid and namespace). | |
| 36 // Frequently the appid and namespace of keys are known in advance and so there'
s | |
| 37 // no reason to redundantly encode them. | |
| 38 type KeyContext bool | |
| 39 | |
| 40 // With- and WithoutContext indicate if the serialization method should include | |
| 41 // context for Keys. See KeyContext for more information. | |
| 42 const ( | |
| 43 WithContext KeyContext = true | |
| 44 WithoutContext = false | |
| 45 ) | |
| 46 | |
| 47 // WriteKey encodes a key to the buffer. If context is WithContext, then this | |
| 48 // encoded value will include the appid and namespace of the key. | |
| 49 func WriteKey(buf Buffer, context KeyContext, k Key) (err error) { | |
| 50 // [appid ++ namespace]? ++ #tokens ++ tokens* | |
| 51 defer recoverTo(&err) | |
| 52 appid, namespace, toks := KeySplit(k) | |
| 53 if context == WithContext { | |
| 54 panicIf(buf.WriteByte(1)) | |
| 55 _, e := cmpbin.WriteString(buf, appid) | |
| 56 panicIf(e) | |
| 57 _, e = cmpbin.WriteString(buf, namespace) | |
| 58 panicIf(e) | |
| 59 } else { | |
| 60 panicIf(buf.WriteByte(0)) | |
| 61 } | |
| 62 _, e := cmpbin.WriteUint(buf, uint64(len(toks))) | |
| 63 panicIf(e) | |
| 64 for _, tok := range toks { | |
| 65 panicIf(WriteKeyTok(buf, tok)) | |
| 66 } | |
| 67 return nil | |
| 68 } | |
| 69 | |
| 70 // ReadKey deserializes a key from the buffer. The value of context must match | |
| 71 // the value of context that was passed to WriteKey when the key was encoded. | |
| 72 // If context == WithoutContext, then the appid and namespace parameters are | |
| 73 // used in the decoded Key. Otherwise they're ignored. | |
| 74 func ReadKey(buf Buffer, context KeyContext, appid, namespace string) (ret Key,
err error) { | |
| 75 defer recoverTo(&err) | |
| 76 actualCtx, e := buf.ReadByte() | |
| 77 panicIf(e) | |
| 78 | |
| 79 actualAid, actualNS := "", "" | |
| 80 if actualCtx == 1 { | |
| 81 actualAid, _, e = cmpbin.ReadString(buf) | |
| 82 panicIf(e) | |
| 83 actualNS, _, e = cmpbin.ReadString(buf) | |
| 84 panicIf(e) | |
| 85 } else if actualCtx != 0 { | |
| 86 err = fmt.Errorf("helper: expected actualCtx to be 0 or 1, got %
d", actualCtx) | |
| 87 return | |
| 88 } | |
| 89 | |
| 90 if context == WithoutContext { | |
| 91 // overrwrite with the supplied ones | |
| 92 actualAid = appid | |
| 93 actualNS = namespace | |
| 94 } | |
| 95 | |
| 96 numToks, _, e := cmpbin.ReadUint(buf) | |
| 97 panicIf(e) | |
| 98 if numToks > ReadKeyNumToksReasonableLimit { | |
| 99 err = fmt.Errorf("helper: tried to decode huge key of length %d"
, numToks) | |
| 100 return | |
| 101 } | |
| 102 | |
| 103 toks := make([]KeyTok, numToks) | |
| 104 for i := uint64(0); i < numToks; i++ { | |
| 105 toks[i], e = ReadKeyTok(buf) | |
| 106 panicIf(e) | |
| 107 } | |
| 108 | |
| 109 return NewKeyToks(actualAid, actualNS, toks), nil | |
| 110 } | |
| 111 | |
| 112 // WriteKeyTok writes a KeyTok to the buffer. You usually want WriteKey | |
| 113 // instead of this. | |
| 114 func WriteKeyTok(buf Buffer, tok KeyTok) (err error) { | |
| 115 // tok.kind ++ typ ++ [tok.stringID || tok.intID] | |
| 116 defer recoverTo(&err) | |
| 117 _, e := cmpbin.WriteString(buf, tok.Kind) | |
| 118 panicIf(e) | |
| 119 if tok.StringID != "" { | |
| 120 panicIf(buf.WriteByte(byte(PTString))) | |
| 121 _, e := cmpbin.WriteString(buf, tok.StringID) | |
| 122 panicIf(e) | |
| 123 } else { | |
| 124 panicIf(buf.WriteByte(byte(PTInt))) | |
| 125 _, e := cmpbin.WriteInt(buf, tok.IntID) | |
| 126 panicIf(e) | |
| 127 } | |
| 128 return nil | |
| 129 } | |
| 130 | |
| 131 // ReadKeyTok reads a KeyTok from the buffer. You usually want ReadKey | |
| 132 // instead of this. | |
| 133 func ReadKeyTok(buf Buffer) (ret KeyTok, err error) { | |
| 134 defer recoverTo(&err) | |
| 135 e := error(nil) | |
| 136 ret.Kind, _, e = cmpbin.ReadString(buf) | |
| 137 panicIf(e) | |
| 138 | |
| 139 typ, e := buf.ReadByte() | |
| 140 panicIf(e) | |
| 141 | |
| 142 switch PropertyType(typ) { | |
| 143 case PTString: | |
| 144 ret.StringID, _, err = cmpbin.ReadString(buf) | |
| 145 case PTInt: | |
| 146 ret.IntID, _, err = cmpbin.ReadInt(buf) | |
| 147 if err == nil && ret.IntID <= 0 { | |
| 148 err = errors.New("helper: decoded key with empty stringI
D and zero/negative intID") | |
| 149 } | |
| 150 default: | |
| 151 err = fmt.Errorf("helper: invalid type %s", PropertyType(typ)) | |
| 152 } | |
| 153 return | |
| 154 } | |
| 155 | |
| 156 // Write writes a GeoPoint to the buffer. | |
| 157 func (gp GeoPoint) Write(buf Buffer) (err error) { | |
| 158 defer recoverTo(&err) | |
| 159 _, e := cmpbin.WriteFloat64(buf, gp.Lat) | |
| 160 panicIf(e) | |
| 161 _, e = cmpbin.WriteFloat64(buf, gp.Lng) | |
| 162 return e | |
| 163 } | |
| 164 | |
| 165 // Read reads a GeoPoint from the buffer. | |
| 166 func (gp *GeoPoint) Read(buf Buffer) (err error) { | |
| 167 defer recoverTo(&err) | |
| 168 e := error(nil) | |
| 169 gp.Lat, _, e = cmpbin.ReadFloat64(buf) | |
| 170 panicIf(e) | |
| 171 | |
| 172 gp.Lng, _, e = cmpbin.ReadFloat64(buf) | |
| 173 panicIf(e) | |
| 174 | |
| 175 if !gp.Valid() { | |
| 176 err = fmt.Errorf("helper: decoded invalid GeoPoint: %v", gp) | |
| 177 } | |
| 178 return | |
| 179 } | |
| 180 | |
| 181 // WriteTime writes a time.Time in a byte-sortable way. | |
| 182 // | |
| 183 // This method truncates the time to microseconds and drops the timezone, | |
| 184 // because that's the (undocumented) way that the appengine SDK does it. | |
| 185 func WriteTime(buf Buffer, t time.Time) error { | |
| 186 name, off := t.Zone() | |
| 187 if name != "UTC" || off != 0 { | |
| 188 panic(fmt.Errorf("helper: UTC OR DEATH: %s", t)) | |
| 189 } | |
| 190 _, err := cmpbin.WriteUint(buf, uint64(t.Unix())*1e6+uint64(t.Nanosecond
()/1e3)) | |
| 191 return err | |
| 192 } | |
| 193 | |
| 194 // ReadTime reads a time.Time from the buffer. | |
| 195 func ReadTime(buf Buffer) (time.Time, error) { | |
| 196 v, _, err := cmpbin.ReadUint(buf) | |
| 197 if err != nil { | |
| 198 return time.Time{}, err | |
| 199 } | |
| 200 return time.Unix(int64(v/1e6), int64((v%1e6)*1e3)).UTC(), nil | |
| 201 } | |
| 202 | |
| 203 // Write writes a Property to the buffer. `context` behaves the same | |
| 204 // way that it does for WriteKey, but only has an effect if `p` contains a | |
| 205 // Key as its Value. | |
| 206 func (p *Property) Write(buf Buffer, context KeyContext) (err error) { | |
| 207 defer recoverTo(&err) | |
| 208 typb := byte(p.Type()) | |
| 209 if p.IndexSetting() == NoIndex { | |
| 210 typb |= 0x80 | |
| 211 } | |
| 212 panicIf(buf.WriteByte(typb)) | |
| 213 switch p.Type() { | |
| 214 case PTNull, PTBoolTrue, PTBoolFalse: | |
| 215 case PTInt: | |
| 216 _, err = cmpbin.WriteInt(buf, p.Value().(int64)) | |
| 217 case PTFloat: | |
| 218 _, err = cmpbin.WriteFloat64(buf, p.Value().(float64)) | |
| 219 case PTString: | |
| 220 _, err = cmpbin.WriteString(buf, p.Value().(string)) | |
| 221 case PTBytes: | |
| 222 if p.IndexSetting() == NoIndex { | |
| 223 _, err = cmpbin.WriteBytes(buf, p.Value().([]byte)) | |
| 224 } else { | |
| 225 _, err = cmpbin.WriteBytes(buf, p.Value().(ByteString)) | |
| 226 } | |
| 227 case PTTime: | |
| 228 err = WriteTime(buf, p.Value().(time.Time)) | |
| 229 case PTGeoPoint: | |
| 230 err = p.Value().(GeoPoint).Write(buf) | |
| 231 case PTKey: | |
| 232 err = WriteKey(buf, context, p.Value().(Key)) | |
| 233 case PTBlobKey: | |
| 234 _, err = cmpbin.WriteString(buf, string(p.Value().(blobstore.Key
))) | |
| 235 } | |
| 236 return | |
| 237 } | |
| 238 | |
| 239 // Read reads a Property from the buffer. `context`, `appid`, and | |
| 240 // `namespace` behave the same way they do for ReadKey, but only have an | |
| 241 // effect if the decoded property has a Key value. | |
| 242 func (p *Property) Read(buf Buffer, context KeyContext, appid, namespace string)
(err error) { | |
| 243 val := interface{}(nil) | |
| 244 typb, err := buf.ReadByte() | |
| 245 if err != nil { | |
| 246 return | |
| 247 } | |
| 248 is := ShouldIndex | |
| 249 if (typb & 0x80) != 0 { | |
| 250 is = NoIndex | |
| 251 } | |
| 252 switch PropertyType(typb & 0x7f) { | |
| 253 case PTNull: | |
| 254 case PTBoolTrue: | |
| 255 val = true | |
| 256 case PTBoolFalse: | |
| 257 val = false | |
| 258 case PTInt: | |
| 259 val, _, err = cmpbin.ReadInt(buf) | |
| 260 case PTFloat: | |
| 261 val, _, err = cmpbin.ReadFloat64(buf) | |
| 262 case PTString: | |
| 263 val, _, err = cmpbin.ReadString(buf) | |
| 264 case PTBytes: | |
| 265 b := []byte(nil) | |
| 266 if b, _, err = cmpbin.ReadBytes(buf); err != nil { | |
| 267 break | |
| 268 } | |
| 269 if is == NoIndex { | |
| 270 val = b | |
| 271 } else { | |
| 272 val = ByteString(b) | |
| 273 } | |
| 274 case PTTime: | |
| 275 val, err = ReadTime(buf) | |
| 276 case PTGeoPoint: | |
| 277 gp := GeoPoint{} | |
| 278 err = gp.Read(buf) | |
| 279 val = gp | |
| 280 case PTKey: | |
| 281 val, err = ReadKey(buf, context, appid, namespace) | |
| 282 case PTBlobKey: | |
| 283 s := "" | |
| 284 if s, _, err = cmpbin.ReadString(buf); err != nil { | |
| 285 break | |
| 286 } | |
| 287 val = blobstore.Key(s) | |
| 288 default: | |
| 289 err = fmt.Errorf("read: unknown type! %v", typb) | |
| 290 } | |
| 291 if err == nil { | |
| 292 err = p.SetValue(val, is) | |
| 293 } | |
| 294 return | |
| 295 } | |
| 296 | |
| 297 // Write writes an entire PropertyMap to the buffer. `context` behaves the same | |
| 298 // way that it does for WriteKey. If WritePropertyMapDeterministic is true, then | |
| 299 // the rows will be sorted by property name before they're serialized to buf | |
| 300 // (mostly useful for testing, but also potentially useful if you need to make | |
| 301 // a hash of the property data). | |
| 302 // | |
| 303 // Write skips metadata keys. | |
| 304 func (pm PropertyMap) Write(buf Buffer, context KeyContext) (err error) { | |
| 305 defer recoverTo(&err) | |
| 306 rows := make(sort.StringSlice, 0, len(pm)) | |
| 307 tmpBuf := &bytes.Buffer{} | |
| 308 for name, vals := range pm { | |
| 309 if isMetaKey(name) { | |
| 310 continue | |
| 311 } | |
| 312 tmpBuf.Reset() | |
| 313 _, e := cmpbin.WriteString(tmpBuf, name) | |
| 314 panicIf(e) | |
| 315 _, e = cmpbin.WriteUint(tmpBuf, uint64(len(vals))) | |
| 316 panicIf(e) | |
| 317 for _, p := range vals { | |
| 318 panicIf(p.Write(tmpBuf, context)) | |
| 319 } | |
| 320 rows = append(rows, tmpBuf.String()) | |
| 321 } | |
| 322 | |
| 323 if WritePropertyMapDeterministic { | |
| 324 rows.Sort() | |
| 325 } | |
| 326 | |
| 327 _, e := cmpbin.WriteUint(buf, uint64(len(pm))) | |
| 328 panicIf(e) | |
| 329 for _, r := range rows { | |
| 330 _, e := buf.WriteString(r) | |
| 331 panicIf(e) | |
| 332 } | |
| 333 return | |
| 334 } | |
| 335 | |
| 336 // Read reads a PropertyMap from the buffer. `context` and | |
| 337 // friends behave the same way that they do for ReadKey. | |
| 338 func (pm PropertyMap) Read(buf Buffer, context KeyContext, appid, namespace stri
ng) (err error) { | |
| 339 defer recoverTo(&err) | |
| 340 | |
| 341 numRows := uint64(0) | |
| 342 numRows, _, e := cmpbin.ReadUint(buf) | |
| 343 panicIf(e) | |
| 344 if numRows > ReadPropertyMapReasonableLimit { | |
| 345 err = fmt.Errorf("helper: tried to decode map with huge number o
f rows %d", numRows) | |
| 346 return | |
| 347 } | |
| 348 | |
| 349 name, prop := "", Property{} | |
| 350 for i := uint64(0); i < numRows; i++ { | |
| 351 name, _, e = cmpbin.ReadString(buf) | |
| 352 panicIf(e) | |
| 353 | |
| 354 numProps, _, e := cmpbin.ReadUint(buf) | |
| 355 panicIf(e) | |
| 356 if numProps > ReadPropertyMapReasonableLimit { | |
| 357 err = fmt.Errorf("helper: tried to decode map with huge
number of properties %d", numProps) | |
| 358 return | |
| 359 } | |
| 360 props := make([]Property, 0, numProps) | |
| 361 for j := uint64(0); j < numProps; j++ { | |
| 362 panicIf(prop.Read(buf, context, appid, namespace)) | |
| 363 props = append(props, prop) | |
| 364 } | |
| 365 pm[name] = props | |
| 366 } | |
| 367 return | |
| 368 } | |
| 369 | |
| 370 func (c *IndexColumn) Write(buf Buffer) (err error) { | |
| 371 defer recoverTo(&err) | |
| 372 | |
| 373 if c.Direction == ASCENDING { | |
| 374 panicIf(buf.WriteByte(0)) | |
| 375 } else { | |
| 376 panicIf(buf.WriteByte(1)) | |
| 377 } | |
| 378 _, err = cmpbin.WriteString(buf, c.Property) | |
| 379 return | |
| 380 } | |
| 381 | |
| 382 func (c *IndexColumn) Read(buf Buffer) (err error) { | |
| 383 defer recoverTo(&err) | |
| 384 | |
| 385 dir, err := buf.ReadByte() | |
| 386 panicIf(err) | |
| 387 | |
| 388 c.Direction = dir != 0 | |
| 389 c.Property, _, err = cmpbin.ReadString(buf) | |
| 390 return err | |
| 391 } | |
| 392 | |
| 393 func (i *IndexDefinition) Write(buf Buffer) (err error) { | |
| 394 defer recoverTo(&err) | |
| 395 | |
| 396 if i.Builtin() { | |
| 397 panicIf(buf.WriteByte(0)) | |
| 398 } else { | |
| 399 panicIf(buf.WriteByte(1)) | |
| 400 } | |
| 401 _, err = cmpbin.WriteString(buf, i.Kind) | |
| 402 panicIf(err) | |
| 403 if !i.Ancestor { | |
| 404 panicIf(buf.WriteByte(0)) | |
| 405 } else { | |
| 406 panicIf(buf.WriteByte(1)) | |
| 407 } | |
| 408 _, err = cmpbin.WriteUint(buf, uint64(len(i.SortBy))) | |
| 409 panicIf(err) | |
| 410 for _, sb := range i.SortBy { | |
| 411 panicIf(sb.Write(buf)) | |
| 412 } | |
| 413 return | |
| 414 } | |
| 415 | |
| 416 func (i *IndexDefinition) Read(buf Buffer) (err error) { | |
| 417 defer recoverTo(&err) | |
| 418 | |
| 419 // discard builtin/complex byte | |
| 420 _, err = buf.ReadByte() | |
| 421 panicIf(err) | |
| 422 | |
| 423 i.Kind, _, err = cmpbin.ReadString(buf) | |
| 424 panicIf(err) | |
| 425 | |
| 426 anc, err := buf.ReadByte() | |
| 427 panicIf(err) | |
| 428 | |
| 429 i.Ancestor = anc == 1 | |
| 430 | |
| 431 numSorts, _, err := cmpbin.ReadUint(buf) | |
| 432 panicIf(err) | |
| 433 | |
| 434 if numSorts > MaxIndexColumns { | |
| 435 return fmt.Errorf("datastore: Got over %d sort orders: %d", | |
| 436 MaxIndexColumns, numSorts) | |
| 437 } | |
| 438 | |
| 439 if numSorts > 0 { | |
| 440 i.SortBy = make([]IndexColumn, numSorts) | |
| 441 for idx := range i.SortBy { | |
| 442 panicIf(i.SortBy[idx].Read(buf)) | |
| 443 } | |
| 444 } | |
| 445 | |
| 446 return | |
| 447 } | |
| 448 | |
| 449 type parseError error | |
| 450 | |
| 451 func panicIf(err error) { | |
| 452 if err != nil { | |
| 453 panic(parseError(err)) | |
| 454 } | |
| 455 } | |
| 456 | |
| 457 func recoverTo(err *error) { | |
| 458 if r := recover(); r != nil { | |
| 459 if rerr := r.(parseError); rerr != nil { | |
| 460 *err = error(rerr) | |
| 461 } | |
| 462 } | |
| 463 } | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 1292913002:
Split off serialization and key functions to their own packages. (Closed)
Base URL: https://github.com/luci/gae.git@make_queries_better