Simple memory testing for gae/wrapper

go/src/infra/gae/libs/wrapper/memory/plist.go

+// Copyright 2015 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package memory
+
+import (
+        "appengine"
+        "bytes"
+        "fmt"
+        "reflect"
+        "time"
+
+        "appengine/datastore"
+
+        "github.com/luci/luci-go/common/funnybase"
+)
+
+type typData struct {
+        typ  propValType
+        data interface{}
+}
+
+func newTypData(v interface{}) (ret *typData, err error) {
+        typ := pvUNKNOWN
+
+        switch x := v.(type) {
+        case nil:
+                typ = pvNull
+        case time.Time:
+                // toMicro return t.Unix()*1e6 + int64(t.Nanosecond()/1e3)
+                // fromMicro return time.Unix(t/1e6, (t%1e6)*1e3)
+                typ = pvTime
+        case int, int8, int16, int32, int64:
+                typ = pvInt
+        case float32, float64:
+                typ = pvFloat
+        case bool:
+                if x {
+                        typ = pvBoolTrue
+                } else {
+                        typ = pvBoolFalse
+                }
+        case []byte, datastore.ByteString:
+                typ = pvBytes
+        case appengine.BlobKey:
+                typ = pvBlobKey
+        case string:
+                typ = pvStr
+        case appengine.GeoPoint:
+                typ = pvGeoPoint
+        case *datastore.Key:
+                typ = pvKey
+        }
+        if typ == pvUNKNOWN {
+                err = fmt.Errorf("propValTypeOf: unknown type of %v", v)

[M-A Ruel, 2015/05/25 18:21:10]

%#v

[iannucci, 2015/05/27 19:33:32]

done

+        }
+
+        return &typData{typ, v}, err
+}
+
+func (td *typData) WriteBinary(buf *bytes.Buffer) error {
+        buf.WriteByte(byte(td.typ))
+        switch td.typ {
+        case pvNull, pvBoolFalse, pvBoolTrue:
+                return nil
+        case pvInt:
+                funnybase.Write(buf, reflect.ValueOf(td.data).Int())
+        case pvFloat:
+                writeFloat64(buf, reflect.ValueOf(td.data).Float())
+        case pvStr:
+                writeString(buf, td.data.(string))
+        case pvBytes:
+                // []byte or datastore.ByteString
+                b := reflect.ValueOf(td.data).Convert(byteSliceType).Interface().([]byte)
+                writeBytes(buf, b)
+        case pvTime:
+                t := td.data.(time.Time)
+                funnybase.WriteUint(buf, uint64(t.Unix())*1e6+uint64(t.Nanosecond()/1e3))
+        case pvGeoPoint:
+                t := td.data.(appengine.GeoPoint)
+                writeFloat64(buf, t.Lat)
+                writeFloat64(buf, t.Lng)
+        case pvKey:
+                writeKey(buf, withNS, td.data.(*datastore.Key))
+        case pvBlobKey:
+                writeString(buf, string(td.data.(appengine.BlobKey)))
+        default:
+                return fmt.Errorf("write: unknown type! %v", td)
+        }
+        return nil
+}
+
+func (td *typData) ReadBinary(buf *bytes.Buffer, indexed bool) error {
+        typb, err := buf.ReadByte()
+        if err != nil {
+                return err
+        }
+        td.typ = propValType(typb)
+        switch td.typ {
+        case pvNull:
+                td.data = nil
+        case pvBoolTrue:
+                td.data = true
+        case pvBoolFalse:
+                td.data = false
+        case pvInt:
+                v, err := funnybase.Read(buf)
+                if err != nil {
+                        return err
+                }
+                td.data = v
+        case pvFloat:
+                td.data, err = readFloat64(buf)
+                if err != nil {
+                        return err
+                }
+        case pvStr:
+                td.data, err = readString(buf)
+                if err != nil {
+                        return err
+                }
+        case pvBytes:
+                b, err := readBytes(buf)
+                if err != nil {
+                        return err
+                }
+
+                if indexed {
+                        td.data = datastore.ByteString(b)
+                } else {
+                        td.data = b
+                }
+        case pvTime:
+                v, err := funnybase.ReadUint(buf)
+                if err != nil {
+                        return err
+                }
+                td.data = time.Unix(int64(v/1e6), int64((v%1e6)*1e3))
+        case pvGeoPoint:
+                pt := appengine.GeoPoint{}
+                pt.Lat, err = readFloat64(buf)
+                if err != nil {
+                        return err
+                }
+                pt.Lng, err = readFloat64(buf)
+                if err != nil {
+                        return err
+                }
+                td.data = pt
+        case pvKey:
+                td.data, err = readKey(buf, true)
+                if err != nil {
+                        return err
+                }
+        case pvBlobKey:

[M-A Ruel, 2015/05/25 18:21:10]

Do we need to implement this at all?

[iannucci, 2015/05/27 19:33:32]

maybe not... should I just panic for BlobKey's? I'm tempted to do that for
*Key's too... that would maybe let me decouple from "appengine/datastore".
Followup CL? Added TODO's

[M-A Ruel, 2015/05/27 20:14:47]

I don't mind much.

+                s, err := readString(buf)
+                if err != nil {
+                        return err
+                }
+                td.data = appengine.BlobKey(s)
+        default:
+                return fmt.Errorf("read: unknown type! %v", td)
+        }
+
+        return nil
+}
+
+type pval struct {
+        name    string
+        noIndex bool
+        multi   bool
+        vals    []*typData
+}
+
+type propertyList []datastore.Property
+
+func (pl *propertyList) Load(ch <-chan datastore.Property) error {
+        return (*datastore.PropertyList)(pl).Load(ch)
+}
+
+func (pl *propertyList) Save(ch chan<- datastore.Property) error {
+        return (*datastore.PropertyList)(pl).Save(ch)
+}
+
+func (pl *propertyList) collate() ([]*pval, error) {
+        if pl == nil || len(*pl) == 0 {
+                return nil, nil
+        }
+
+        cols := []*pval{}
+        colIdx := map[string]int{}
+
+        for _, p := range *pl {
+                var ok bool
+                idx := 0
+                if idx, ok = colIdx[p.Name]; !ok {

[M-A Ruel, 2015/05/25 18:21:10]

if idx, ok := colIdx[p.Name]; !ok {

and removing lines 195-196 would work just fine.

[iannucci, 2015/05/27 19:33:32]

yay!

+                        colIdx[p.Name] = len(cols)
+                        td, err := newTypData(p.Value)
+                        if err != nil {
+                                return nil, err
+                        }
+                        cols = append(cols, &pval{p.Name, p.NoIndex, p.Multiple, []*typData{td}})
+                } else {
+                        c := cols[idx]
+                        if c.noIndex != p.NoIndex {
+                                return nil, fmt.Errorf(
+                                        "propertyList.MarshalBinary: field %q has conflicting values of NoIndex", p.Name)
+                        }
+                        if c.multi != p.Multiple {
+                                return nil, fmt.Errorf(
+                                        "propertyList.MarshalBinary: field %q has conflicting values of Multiple", p.Name)
+                        }
+                        td, err := newTypData(p.Value)
+                        if err != nil {
+                                return nil, err
+                        }
+                        c.vals = append(c.vals, td)
+                }
+        }
+
+        return cols, nil
+}
+
+func (pl *propertyList) addCollated(pv *pval) {
+        for _, v := range pv.vals {
+                *pl = append(*pl, datastore.Property{
+                        Name:     pv.name,
+                        Multiple: pv.multi,
+                        NoIndex:  pv.noIndex,
+                        Value:    v.data,
+                })
+        }
+}
+
+func (pl *propertyList) MarshalBinary() ([]byte, error) {
+        cols, err := pl.collate()
+        if err != nil || len(cols) == 0 {
+                return nil, err
+        }
+
+        pieces := make([][]byte, 0, len(*pl)*2+1)
+        for _, pv := range cols {
+                // TODO(riannucci): estimate buffer size better.
+                buf := bytes.NewBuffer(make([]byte, 0, funnybase.MaxFunnyBaseLen64+len(pv.name)))
+                writeString(buf, pv.name)
+                err := pv.WriteBinary(buf)
+                if err != nil {
+                        return nil, err
+                }
+                pieces = append(pieces, buf.Bytes())
+        }
+        return bytes.Join(pieces, nil), nil
+}
+
+func (pl *propertyList) UnmarshalBinary(data []byte) error {
+        buf := bytes.NewBuffer(data)
+        for buf.Len() > 0 {
+                name, err := readString(buf)
+                if err != nil {
+                        return err
+                }
+
+                pv := &pval{name: name}
+                err = pv.ReadBinary(buf)
+                if err != nil {
+                        return err
+                }
+                pl.addCollated(pv)
+        }
+
+        return nil
+}
+
+func toPL(src interface{}) (ret *propertyList, err error) {
+        propchan := make(chan datastore.Property)
+        ret = &propertyList{}
+        go func() { err = datastore.SaveStruct(src, propchan) }()
+        err2 := ret.Load(propchan)
+        if err != nil {
+                return
+        }
+        return ret, err2
+}
+
+func fromPL(props *propertyList, dst interface{}) (err error) {
+        propchan := make(chan datastore.Property)
+        go func() { err = props.Save(propchan) }()
+        err2 := datastore.LoadStruct(dst, propchan)
+        if err != nil {
+                return err
+        }
+        return err2
+}
+
+type propValType byte
+
+var byteSliceType = reflect.TypeOf([]byte(nil))
+
+// These constants are in the order described by
+//   https://cloud.google.com/appengine/docs/go/datastore/entities#Go_Value_type_ordering
+// with a slight divergence for the Int/Time split.
+const (
+        pvNull propValType = iota
+        pvInt
+
+        // NOTE: this is a slight divergence; times and integers actually sort
+        // together (apparently?) in datastore. This is probably insane, and I don't
+        // want to add the complexity of field 'meaning' as a sparate concept from the
+        // field's 'type' (which is what datastore seems to do, judging from the
+        // protobufs). So if you're here because you implemented an app which relies
+        // on time.Time and int64 sorting together, then this is why your app acts
+        // differently in production. My advice is to NOT DO THAT. If you really want
+        // this (and you probably don't), you should take care of the time.Time <->
+        // int64 conversion in your app and just use a property type of int64.
+        pvTime
+
+        // NOTE: this is also a slight divergence, but not a semantic one. IIUC, in
+        // datastore 'bool' is actually the type and the value is either 0 or
+        // 1 (taking another byte to store). Since we have plenty of space in this
+        // type byte, I just merge the value into the type for booleans. If this
+        // becomes problematic, consider changing this to just pvBool, and then
+        // encoding a 0 or 1 as a byte in the relevant marshalling routines.
+        pvBoolFalse
+        pvBoolTrue
+        pvBytes // []byte or datastore.ByteString
+        pvStr   // string or string noindex
+        pvFloat
+        pvGeoPoint
+        pvKey
+        pvBlobKey
+
+        pvUNKNOWN
+)
+
+func (p *pval) ReadBinary(buf *bytes.Buffer) error {
+        prefix, err := buf.ReadByte()
+        if err != nil {
+                return err
+        }
+        p.multi = (prefix & 1) == 1
+        p.noIndex = ((prefix >> 1) & 1) == 1
+        n, err := funnybase.ReadUint(buf)
+        if err != nil {
+                return err
+        }
+
+        p.vals = make([]*typData, n)
+        for i := range p.vals {
+                p.vals[i] = &typData{}
+                err := p.vals[i].ReadBinary(buf, !p.noIndex)
+                if err != nil {
+                        return err
+                }
+        }
+
+        return nil
+}
+
+func (p *pval) WriteBinary(buf *bytes.Buffer) error {
+        buf.WriteByte(btoi(p.noIndex)<<1 | btoi(p.multi))
+        funnybase.WriteUint(buf, uint64(len(p.vals)))
+        for _, v := range p.vals {
+                err := v.WriteBinary(buf)
+                if err != nil {
+                        return err
+                }
+        }
+        return nil
+}