Simple memory testing for gae/wrapper

go/src/infra/gae/libs/wrapper/memory/plist.go

+// Copyright 2015 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package memory
+
+import (
+        "appengine"
+        "bytes"
+        "fmt"
+        "reflect"
+        "time"
+
+        "appengine/datastore"
+
+        "github.com/luci/luci-go/common/funnybase"
+)
+
+type typData struct {
+        typ  propValType
+        data interface{}
+}
+
+func newTypData(v interface{}) (ret *typData, err error) {
+        typ := pvUNKNOWN
+
+        switch x := v.(type) {
+        case nil:
+                typ = pvNull
+        case time.Time:
+                // toMicro return t.Unix()*1e6 + int64(t.Nanosecond()/1e3)
+                // fromMicro return time.Unix(t/1e6, (t%1e6)*1e3)
+                typ = pvTime
+        case int, int8, int16, int32, int64:
+                typ = pvInt
+        case float32, float64:
+                typ = pvFloat
+        case bool:
+                if x {
+                        typ = pvBoolTrue
+                } else {
+                        typ = pvBoolFalse
+                }
+        case []byte, datastore.ByteString:
+                typ = pvBytes
+        case appengine.BlobKey:
+                typ = pvBlobKey
+        case string:
+                typ = pvStr
+        case appengine.GeoPoint:
+                typ = pvGeoPoint
+        case *datastore.Key:
+                typ = pvKey
+        }
+        if typ == pvUNKNOWN {
+                err = fmt.Errorf("propValTypeOf: unknown type of %v", v)
+        }
+
+        return &typData{typ, v}, err
+}
+
+func (td *typData) WriteBinary(buf *bytes.Buffer) error {
+        buf.WriteByte(byte(td.typ))
+        switch td.typ {
+        case pvNull, pvBoolFalse, pvBoolTrue:
+                return nil
+        case pvInt:
+                funnybase.Write(buf, reflect.ValueOf(td.data).Int())
+        case pvFloat:
+                writeFloat64(buf, reflect.ValueOf(td.data).Float())
+        case pvStr:
+                writeString(buf, td.data.(string))
+        case pvBytes:
+                // []byte or datastore.ByteString
+                b := reflect.ValueOf(td.data).Convert(byteSliceType).Interface().([]byte)
+                writeBytes(buf, b)
+        case pvTime:
+                t := td.data.(time.Time)
+                funnybase.WriteUint(buf, uint64(t.Unix())*1e6+uint64(t.Nanosecond()/1e3))
+        case pvGeoPoint:
+                t := td.data.(appengine.GeoPoint)
+                writeFloat64(buf, t.Lat)
+                writeFloat64(buf, t.Lng)
+        case pvKey:
+                writeKey(buf, withNS, td.data.(*datastore.Key))
+        case pvBlobKey:
+                writeString(buf, string(td.data.(appengine.BlobKey)))
+        default:
+                return fmt.Errorf("write: unknown type! %v", td)
+        }
+        return nil
+}
+
+func (td *typData) ReadBinary(buf *bytes.Buffer, indexed bool) error {
+        typb, err := buf.ReadByte()
+        if err != nil {
+                return err
+        }
+        td.typ = propValType(typb)
+        switch td.typ {
+        case pvNull:
+                td.data = nil
+        case pvBoolTrue:
+                td.data = true
+        case pvBoolFalse:
+                td.data = false
+        case pvInt:
+                v, err := funnybase.Read(buf)
+                if err != nil {
+                        return err
+                }
+                td.data = v
+        case pvFloat:
+                td.data, err = readFloat64(buf)
+                if err != nil {
+                        return err
+                }
+        case pvStr:
+                td.data, err = readString(buf)
+                if err != nil {
+                        return err
+                }
+        case pvBytes:
+                b, err := readBytes(buf)
+                if err != nil {
+                        return err
+                }
+
+                if indexed {
+                        td.data = datastore.ByteString(b)
+                } else {
+                        td.data = b
+                }
+        case pvTime:
+                v, err := funnybase.ReadUint(buf)
+                if err != nil {
+                        return err
+                }
+                td.data = time.Unix(int64(v/1e6), int64((v%1e6)*1e3))
+        case pvGeoPoint:
+                pt := appengine.GeoPoint{}
+                pt.Lat, err = readFloat64(buf)
+                if err != nil {
+                        return err
+                }
+                pt.Lng, err = readFloat64(buf)
+                if err != nil {
+                        return err
+                }
+                td.data = pt
+        case pvKey:
+                td.data, err = readKey(buf, true)
+                if err != nil {
+                        return err
+                }
+        case pvBlobKey:
+                s, err := readString(buf)
+                if err != nil {
+                        return err
+                }
+                td.data = appengine.BlobKey(s)
+        default:
+                return fmt.Errorf("read: unknown type! %v", td)
+        }
+
+        return nil
+}
+
+type pval struct {
+        name    string
+        noIndex bool
+        multi   bool
+        vals    []*typData
+}
+
+type propertyList []datastore.Property
+
+func (pl *propertyList) Load(ch <-chan datastore.Property) error {
+        return (*datastore.PropertyList)(pl).Load(ch)
+}
+
+func (pl *propertyList) Save(ch chan<- datastore.Property) error {
+        return (*datastore.PropertyList)(pl).Save(ch)
+}
+
+func (pl *propertyList) collate() ([]*pval, error) {
+        if pl == nil || len(*pl) == 0 {
+                return nil, nil
+        }
+
+        cols := []*pval{}
+        colIdx := map[string]int{}
+
+        for _, p := range *pl {
+                var ok bool
+                idx := 0
+                if idx, ok = colIdx[p.Name]; !ok {
+                        colIdx[p.Name] = len(cols)
+                        td, err := newTypData(p.Value)
+                        if err != nil {
+                                return nil, err
+                        }
+                        cols = append(cols, &pval{p.Name, p.NoIndex, p.Multiple, []*typData{td}})
+                } else {
+                        c := cols[idx]
+                        if c.noIndex != p.NoIndex {
+                                return nil, fmt.Errorf(
+                                        "propertyList.MarshalBinary: field %q has conflicting values of NoIndex", p.Name)
+                        }
+                        if c.multi != p.Multiple {
+                                return nil, fmt.Errorf(
+                                        "propertyList.MarshalBinary: field %q has conflicting values of Multiple", p.Name)
+                        }
+                        td, err := newTypData(p.Value)
+                        if err != nil {
+                                return nil, err
+                        }
+                        c.vals = append(c.vals, td)
+                }
+        }
+
+        return cols, nil
+}
+
+func (pl *propertyList) addCollated(pv *pval) {
+        for _, v := range pv.vals {
+                *pl = append(*pl, datastore.Property{
+                        Name:     pv.name,
+                        Multiple: pv.multi,
+                        NoIndex:  pv.noIndex,
+                        Value:    v.data,
+                })
+        }
+}
+
+func (pl *propertyList) MarshalBinary() ([]byte, error) {
+        cols, err := pl.collate()
+        if err != nil || len(cols) == 0 {
+                return nil, err
+        }
+
+        pieces := make([][]byte, 0, len(*pl)*2+1)
+        for _, pv := range cols {
+                // TODO(riannucci): estimate buffer size better.
+                buf := bytes.NewBuffer(make([]byte, 0, funnybase.MaxFunnyBaseLen64+len(pv.name)))
+                writeString(buf, pv.name)
+                err := pv.WriteBinary(buf)
+                if err != nil {
+                        return nil, err
+                }
+                pieces = append(pieces, buf.Bytes())
+        }
+        return bytes.Join(pieces, nil), nil
+}
+
+func (pl *propertyList) UnmarshalBinary(data []byte) error {
+        buf := bytes.NewBuffer(data)
+        for buf.Len() > 0 {
+                name, err := readString(buf)
+                if err != nil {
+                        return err
+                }
+
+                pv := &pval{name: name}
+                err = pv.ReadBinary(buf)
+                if err != nil {
+                        return err
+                }
+                pl.addCollated(pv)
+        }
+
+        return nil
+}
+
+func toPL(src interface{}) (ret *propertyList, err error) {
+        propchan := make(chan datastore.Property)
+        ret = &propertyList{}
+        go func() { err = datastore.SaveStruct(src, propchan) }()
+        err2 := ret.Load(propchan)
+        if err != nil {
+                return
+        }
+        return ret, err2
+}
+
+func fromPL(props *propertyList, dst interface{}) (err error) {
+        propchan := make(chan datastore.Property)
+        go func() { err = props.Save(propchan) }()
+        err2 := datastore.LoadStruct(dst, propchan)
+        if err != nil {
+                return err
+        }
+        return err2
+}
+
+type propValType byte
+
+var byteSliceType = reflect.TypeOf([]byte(nil))
+
+// These constants are in the order described by
+//   https://cloud.google.com/appengine/docs/go/datastore/entities#Go_Value_type_ordering
+// with a slight divergence for the Int/Time split.
+const (
+        pvNull propValType = iota
+        pvInt
+
+        // NOTE: this is a slight divergence; times and integers actually sort
+        // together (apparently?) in datastore. This is probably insane, and I don't
+        // want to add the complexity of field 'meaning' as a sparate concept from the
+        // field's 'type' (which is what datastore seems to do, judging from the
+        // protobufs). So if you're here because you implemented an app which relies
+        // on time.Time and int64 sorting together, then this is why your app acts
+        // differently in production. My advice is to NOT DO THAT. If you really want
+        // this (and you probably don't), you should take care of the time.Time <->
+        // int64 conversion in your app and just use a property type of int64.
+        pvTime
+
+        // NOTE: this is also a slight divergence, but not a semantic one. IIUC, in

[Vadim Sh., 2015/05/24 19:43:27]

does datastore support "<=" index on bools? :) 

[iannucci, 2015/05/24 20:33:54]

I think it throws the filter out in the query optimizer. But that should still
work here anyway, since the equality prefix doesn't include inequality filters,
so the index scan will see them both.

But, yes, boolean values have a defined sort order, and you could do e.g. `>
false` (though you'd be a moron to do that). It's mostly important for Sort()
clauses, of which there may be multiple, and it could make sense to have
something sort by a boolean (getting all 'false's before all 'true's)

+        // datastore 'bool' is actually the type and the value is either 0 or
+        // 1 (taking another byte to store). Since we have plenty of space in this
+        // type byte, I just merge the value into the type for booleans. If this
+        // becomes problematic, consider changing this to just pvBool, and then
+        // encoding a 0 or 1 as a byte in the relevant marshalling routines.
+        pvBoolFalse
+        pvBoolTrue
+        pvBytes // []byte or datastore.ByteString
+        pvStr   // string or string noindex
+        pvFloat
+        pvGeoPoint
+        pvKey
+        pvBlobKey
+
+        pvUNKNOWN
+)
+
+func (p *pval) ReadBinary(buf *bytes.Buffer) error {
+        prefix, err := buf.ReadByte()
+        if err != nil {
+                return err
+        }
+        p.multi = (prefix & 1) == 1
+        p.noIndex = ((prefix >> 1) & 1) == 1
+        n, err := funnybase.ReadUint(buf)
+        if err != nil {
+                return err
+        }
+
+        p.vals = make([]*typData, n)
+        for i := range p.vals {
+                p.vals[i] = &typData{}
+                err := p.vals[i].ReadBinary(buf, !p.noIndex)
+                if err != nil {
+                        return err
+                }
+        }
+
+        return nil
+}
+
+func (p *pval) WriteBinary(buf *bytes.Buffer) error {
+        buf.WriteByte(btoi(p.noIndex)<<1 | btoi(p.multi))
+        funnybase.WriteUint(buf, uint64(len(p.vals)))
+        for _, v := range p.vals {
+                err := v.WriteBinary(buf)
+                if err != nil {
+                        return err
+                }
+        }
+        return nil
+}