| OLD | NEW |
| 1 // Copyright 2015 The LUCI Authors. All rights reserved. | 1 // Copyright 2015 The LUCI Authors. All rights reserved. |
| 2 // Use of this source code is governed under the Apache License, Version 2.0 | 2 // Use of this source code is governed under the Apache License, Version 2.0 |
| 3 // that can be found in the LICENSE file. | 3 // that can be found in the LICENSE file. |
| 4 | 4 |
| 5 package serialize | 5 package serialize |
| 6 | 6 |
| 7 import ( | 7 import ( |
| 8 "bytes" | 8 "bytes" |
| 9 "errors" | 9 "errors" |
| 10 "fmt" | 10 "fmt" |
| (...skipping 319 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 330 // If WritePropertyMapDeterministic is true, then the rows will be sorted by | 330 // If WritePropertyMapDeterministic is true, then the rows will be sorted by |
| 331 // property name before they're serialized to buf (mostly useful for testing, | 331 // property name before they're serialized to buf (mostly useful for testing, |
| 332 // but also potentially useful if you need to make a hash of the property data). | 332 // but also potentially useful if you need to make a hash of the property data). |
| 333 // | 333 // |
| 334 // Write skips metadata keys. | 334 // Write skips metadata keys. |
| 335 func WritePropertyMap(buf Buffer, context KeyContext, pm ds.PropertyMap) (err er
ror) { | 335 func WritePropertyMap(buf Buffer, context KeyContext, pm ds.PropertyMap) (err er
ror) { |
| 336 defer recoverTo(&err) | 336 defer recoverTo(&err) |
| 337 rows := make(sort.StringSlice, 0, len(pm)) | 337 rows := make(sort.StringSlice, 0, len(pm)) |
| 338 tmpBuf := &bytes.Buffer{} | 338 tmpBuf := &bytes.Buffer{} |
| 339 pm, _ = pm.Save(false) | 339 pm, _ = pm.Save(false) |
| 340 » for name, vals := range pm { | 340 » for name, pdata := range pm { |
| 341 tmpBuf.Reset() | 341 tmpBuf.Reset() |
| 342 _, e := cmpbin.WriteString(tmpBuf, name) | 342 _, e := cmpbin.WriteString(tmpBuf, name) |
| 343 panicIf(e) | 343 panicIf(e) |
| 344 » » _, e = cmpbin.WriteUint(tmpBuf, uint64(len(vals))) | 344 |
| 345 » » panicIf(e) | 345 » » switch t := pdata.(type) { |
| 346 » » for _, p := range vals { | 346 » » case ds.Property: |
| 347 » » » panicIf(WriteProperty(tmpBuf, context, p)) | 347 » » » _, e = cmpbin.WriteInt(tmpBuf, -1) |
| 348 » » » panicIf(e) |
| 349 » » » panicIf(WriteProperty(tmpBuf, context, t)) |
| 350 |
| 351 » » case ds.PropertySlice: |
| 352 » » » _, e = cmpbin.WriteInt(tmpBuf, int64(len(t))) |
| 353 » » » panicIf(e) |
| 354 » » » for _, p := range t { |
| 355 » » » » panicIf(WriteProperty(tmpBuf, context, p)) |
| 356 » » » } |
| 357 |
| 358 » » default: |
| 359 » » » return fmt.Errorf("unknown PropertyData type %T", t) |
| 348 } | 360 } |
| 349 rows = append(rows, tmpBuf.String()) | 361 rows = append(rows, tmpBuf.String()) |
| 350 } | 362 } |
| 351 | 363 |
| 352 if WritePropertyMapDeterministic { | 364 if WritePropertyMapDeterministic { |
| 353 rows.Sort() | 365 rows.Sort() |
| 354 } | 366 } |
| 355 | 367 |
| 356 _, e := cmpbin.WriteUint(buf, uint64(len(pm))) | 368 _, e := cmpbin.WriteUint(buf, uint64(len(pm))) |
| 357 panicIf(e) | 369 panicIf(e) |
| (...skipping 17 matching lines...) Expand all Loading... |
| 375 return | 387 return |
| 376 } | 388 } |
| 377 | 389 |
| 378 pm = make(ds.PropertyMap, numRows) | 390 pm = make(ds.PropertyMap, numRows) |
| 379 | 391 |
| 380 name, prop := "", ds.Property{} | 392 name, prop := "", ds.Property{} |
| 381 for i := uint64(0); i < numRows; i++ { | 393 for i := uint64(0); i < numRows; i++ { |
| 382 name, _, e = cmpbin.ReadString(buf) | 394 name, _, e = cmpbin.ReadString(buf) |
| 383 panicIf(e) | 395 panicIf(e) |
| 384 | 396 |
| 385 » » numProps, _, e := cmpbin.ReadUint(buf) | 397 » » numProps, _, e := cmpbin.ReadInt(buf) |
| 386 panicIf(e) | 398 panicIf(e) |
| 387 » » if numProps > ReadPropertyMapReasonableLimit { | 399 » » switch { |
| 400 » » case numProps < 0: |
| 401 » » » // Single property. |
| 402 » » » prop, err = ReadProperty(buf, context, appid, namespace) |
| 403 » » » panicIf(err) |
| 404 » » » pm[name] = prop |
| 405 |
| 406 » » case uint64(numProps) > ReadPropertyMapReasonableLimit: |
| 388 err = fmt.Errorf("helper: tried to decode map with huge
number of properties %d", numProps) | 407 err = fmt.Errorf("helper: tried to decode map with huge
number of properties %d", numProps) |
| 389 return | 408 return |
| 409 |
| 410 default: |
| 411 props := make(ds.PropertySlice, 0, numProps) |
| 412 for j := int64(0); j < numProps; j++ { |
| 413 prop, err = ReadProperty(buf, context, appid, na
mespace) |
| 414 panicIf(err) |
| 415 props = append(props, prop) |
| 416 } |
| 417 pm[name] = props |
| 390 } | 418 } |
| 391 props := make([]ds.Property, 0, numProps) | |
| 392 for j := uint64(0); j < numProps; j++ { | |
| 393 prop, err = ReadProperty(buf, context, appid, namespace) | |
| 394 panicIf(err) | |
| 395 props = append(props, prop) | |
| 396 } | |
| 397 pm[name] = props | |
| 398 } | 419 } |
| 399 return | 420 return |
| 400 } | 421 } |
| 401 | 422 |
| 402 // WriteIndexColumn writes an IndexColumn to the buffer. | 423 // WriteIndexColumn writes an IndexColumn to the buffer. |
| 403 func WriteIndexColumn(buf Buffer, c ds.IndexColumn) (err error) { | 424 func WriteIndexColumn(buf Buffer, c ds.IndexColumn) (err error) { |
| 404 defer recoverTo(&err) | 425 defer recoverTo(&err) |
| 405 | 426 |
| 406 if !c.Descending { | 427 if !c.Descending { |
| 407 panicIf(buf.WriteByte(0)) | 428 panicIf(buf.WriteByte(0)) |
| (...skipping 60 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 468 | 489 |
| 469 sb, err := ReadIndexColumn(buf) | 490 sb, err := ReadIndexColumn(buf) |
| 470 panicIf(err) | 491 panicIf(err) |
| 471 | 492 |
| 472 i.SortBy = append(i.SortBy, sb) | 493 i.SortBy = append(i.SortBy, sb) |
| 473 } | 494 } |
| 474 | 495 |
| 475 return | 496 return |
| 476 } | 497 } |
| 477 | 498 |
| 478 // SerializedPslice is all of the serialized DSProperty values in qASC order. | 499 // SerializedPslice is all of the serialized DSProperty values in ASC order. |
| 479 type SerializedPslice [][]byte | 500 type SerializedPslice [][]byte |
| 480 | 501 |
| 481 func (s SerializedPslice) Len() int { return len(s) } | 502 func (s SerializedPslice) Len() int { return len(s) } |
| 482 func (s SerializedPslice) Swap(i, j int) { s[i], s[j] = s[j], s[i] } | 503 func (s SerializedPslice) Swap(i, j int) { s[i], s[j] = s[j], s[i] } |
| 483 func (s SerializedPslice) Less(i, j int) bool { return bytes.Compare(s[i], s[j])
< 0 } | 504 func (s SerializedPslice) Less(i, j int) bool { return bytes.Compare(s[i], s[j])
< 0 } |
| 484 | 505 |
| 485 // PropertySlice serializes a single row of a DSProperty map. | 506 // PropertySlice serializes a single row of a DSProperty map. |
| 507 // |
| 508 // It does not differentiate between single- and multi- properties. |
| 486 func PropertySlice(vals ds.PropertySlice) SerializedPslice { | 509 func PropertySlice(vals ds.PropertySlice) SerializedPslice { |
| 487 dups := stringset.New(0) | 510 dups := stringset.New(0) |
| 488 ret := make(SerializedPslice, 0, len(vals)) | 511 ret := make(SerializedPslice, 0, len(vals)) |
| 489 for _, v := range vals { | 512 for _, v := range vals { |
| 490 if v.IndexSetting() == ds.NoIndex { | 513 if v.IndexSetting() == ds.NoIndex { |
| 491 continue | 514 continue |
| 492 } | 515 } |
| 493 | 516 |
| 494 data := ToBytes(v) | 517 data := ToBytes(v) |
| 495 dataS := string(data) | 518 dataS := string(data) |
| (...skipping 16 matching lines...) Expand all Loading... |
| 512 // datastore/serialize's encodings. | 535 // datastore/serialize's encodings. |
| 513 func PropertyMapPartially(k *ds.Key, pm ds.PropertyMap) (ret SerializedPmap) { | 536 func PropertyMapPartially(k *ds.Key, pm ds.PropertyMap) (ret SerializedPmap) { |
| 514 ret = make(SerializedPmap, len(pm)+2) | 537 ret = make(SerializedPmap, len(pm)+2) |
| 515 if k != nil { | 538 if k != nil { |
| 516 ret["__key__"] = [][]byte{ToBytes(ds.MkProperty(k))} | 539 ret["__key__"] = [][]byte{ToBytes(ds.MkProperty(k))} |
| 517 for k != nil { | 540 for k != nil { |
| 518 ret["__ancestor__"] = append(ret["__ancestor__"], ToByte
s(ds.MkProperty(k))) | 541 ret["__ancestor__"] = append(ret["__ancestor__"], ToByte
s(ds.MkProperty(k))) |
| 519 k = k.Parent() | 542 k = k.Parent() |
| 520 } | 543 } |
| 521 } | 544 } |
| 522 » for k, vals := range pm { | 545 » for k := range pm { |
| 523 » » newVals := PropertySlice(vals) | 546 » » newVals := PropertySlice(pm.Slice(k)) |
| 524 if len(newVals) > 0 { | 547 if len(newVals) > 0 { |
| 525 ret[k] = newVals | 548 ret[k] = newVals |
| 526 } | 549 } |
| 527 } | 550 } |
| 528 return | 551 return |
| 529 } | 552 } |
| 530 | 553 |
| 531 func toBytesErr(i interface{}, ctx KeyContext) (ret []byte, err error) { | 554 func toBytesErr(i interface{}, ctx KeyContext) (ret []byte, err error) { |
| 532 buf := bytes.Buffer{} | 555 buf := bytes.Buffer{} |
| 533 | 556 |
| (...skipping 78 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 612 } | 635 } |
| 613 } | 636 } |
| 614 | 637 |
| 615 func recoverTo(err *error) { | 638 func recoverTo(err *error) { |
| 616 if r := recover(); r != nil { | 639 if r := recover(); r != nil { |
| 617 if rerr := r.(parseError); rerr != nil { | 640 if rerr := r.(parseError); rerr != nil { |
| 618 *err = error(rerr) | 641 *err = error(rerr) |
| 619 } | 642 } |
| 620 } | 643 } |
| 621 } | 644 } |
| OLD | NEW |