Implement projection queries correctly.

service/datastore/properties.go

 // Copyright 2015 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 package datastore
 
 import (
         "errors"
         "fmt"
         "math"
@@ skipping 88 matching lines @@
         // a path in a struct (in the struct loading routine in helper).
 
         ToProperty() (Property, error)
         FromProperty(Property) error
 }
 
 // PropertyType is a single-byte representation of the type of data contained
 // in a Property. The specific values of this type information are chosen so
 // that the types sort according to the order of types as sorted by the
 // datastore.
+//
+// Note that indexes may only contain values of the following types:
+//   PTNull
+//   PTInt
+//   PTBool
+//   PTFloat
+//   PTString
+//   PTGeoPoint
+//   PTKey
+//
+// The biggest impact of this is that if you do a Projection query, you'll only
+// get back Properties with the above types (e.g. if you store a PTTime value,
+// then Project on it, you'll get back a PTInt value). For convenience, Property
+// has a Project(PropertyType) method which will side-cast to your intended
+// type. If you project into a structure with the high-level Interface
+// implementation, or use StructPLS, this conversion will be done for you
+// automatically, using the type of the destination field to cast.
 type PropertyType byte
 
 // These constants are in the order described by
 //   https://cloud.google.com/appengine/docs/go/datastore/entities#Go_Value_type_ordering
 // with a slight divergence for the Int/Time split.
 //
 // NOTE: this enum can only occupy 7 bits, because we use the high bit to encode
 // indexed/non-indexed, and we additionally require that all valid values and
 // all INVERTED valid values must never equal 0xFF or 0x00. The reason for this
 // constraint is that we must always be able to create a byte that sorts before
 // and after it.
 //
 // See "./serialize".WriteProperty and "impl/memory".increment for more info.
 const (
+        // PTNull represents the 'nil' value. This is only directly visible when
+        // reading/writing a PropertyMap. If a PTNull value is loaded into a struct
+        // field, the field will be initialized with its zero value. If a struct with
+        // a zero value is saved from a struct, it will still retain the field's type,
+        // not the 'nil' type. This is in contrast to other GAE languages such as
+        // python where 'None' is a distinct value than the 'zero' value (e.g. a
+        // StringProperty can have the value "" OR None).
+        //
+        // PTNull is a Projection-query type
         PTNull PropertyType = iota
+
+        // PTInt is always an int64.
+        //
+        // This is a Projection-query type, and may be projected to PTTime.
         PTInt
-
-        // PTTime is a slight divergence from the way that datastore natively stores
-        // time. In datastore, times and integers actually sort together
-        // (apparently?). This is probably insane, and I don't want to add the
-        // complexity of field 'meaning' as a sparate concept from the field's 'type'
-        // (which is what datastore seems to do, judging from the protobufs). So if
-        // you're here because you implemented an app which relies on time.Time and
-        // int64 sorting together, then this is why your app acts differently in
-        // production. My advice is to NOT DO THAT. If you really want this (and you
-        // probably don't), you should take care of the time.Time <-> int64 conversion
-        // in your app and just use a property type of int64 (consider using
-        // PropertyConverter).
         PTTime
 
-        // PTBoolFalse and True are also a slight divergence, but not a semantic
-        // one. IIUC, in datastore 'bool' is actually the type and the value is either
-        // 0 or 1 (taking another byte to store). Since we have plenty of space in
-        // this type byte, I just merge the value into the type for booleans. If this
-        // becomes problematic, consider changing this to just pvBool, and then
-        // encoding a 0 or 1 as a byte in the relevant marshalling routines.
-        PTBoolFalse
-        PTBoolTrue
+        // PTBool represents true or false
+        //
+        // This is a Projection-query type.
+        PTBool
 
-        PTBytes  // []byte or datastore.ByteString
-        PTString // string or string noindex
+        // PTBytes represents []byte
+        PTBytes
+
+        // PTString is used to represent all strings (text).
+        //
+        // PTString is a Projection-query type and may be projected to PTBytes or
+        // PTBlobKey.
+        PTString
+
+        // PTFloat is always a float64.
+        //
+        // This is a Projection-query type.
         PTFloat
+
+        // PTGeoPoint is a Projection-query type.
         PTGeoPoint
+
+        // PTKey represents a Key object.
+        //
+        // PTKey is a Projection-query type.
         PTKey
+
+        // PTBlobKey represents a blobstore.Key
         PTBlobKey
 
         // NOTE: THIS MUST BE LAST VALUE FOR THE init() ASSERTION BELOW TO WORK.
         PTUnknown
 )
 
 func init() {
         if PTUnknown > 0x7e {
                 panic(
                         "PTUnknown (and therefore PropertyType) exceeds 0x7e. This conflicts " +
                                 "with serialize.WriteProperty's use of the high bit to indicate " +
                                 "NoIndex and/or \"impl/memory\".increment's ability to guarantee " +
                                 "incrementability.")
         }
 }
 
 func (t PropertyType) String() string {
         switch t {
         case PTNull:
                 return "PTNull"
         case PTInt:
                 return "PTInt"
         case PTTime:
                 return "PTTime"
-        case PTBoolFalse:
-                return "PTBoolFalse"
-        case PTBoolTrue:
-                return "PTBoolTrue"
+        case PTBool:
+                return "PTBool"
         case PTBytes:
                 return "PTBytes"
         case PTString:
                 return "PTString"
         case PTFloat:
                 return "PTFloat"
         case PTGeoPoint:
                 return "PTGeoPoint"
         case PTKey:
                 return "PTKey"
         case PTBlobKey:
                 return "PTBlobKey"
         default:
                 return fmt.Sprintf("PTUnknown(%02x)", byte(t))
         }
 }
 
 // PropertyTypeOf returns the PT* type of the given Property-compatible
 // value v. If checkValid is true, this method will also ensure that time.Time
 // and GeoPoint have valid values.
 func PropertyTypeOf(v interface{}, checkValid bool) (PropertyType, error) {
         switch x := v.(type) {
         case nil:
                 return PTNull, nil
         case int64:
                 return PTInt, nil
         case float64:
                 return PTFloat, nil
         case bool:
-                if x {
-                        return PTBoolTrue, nil
-                }
-                return PTBoolFalse, nil
-        case []byte, ByteString:
+                return PTBool, nil
+        case []byte:
                 return PTBytes, nil
         case blobstore.Key:
                 return PTBlobKey, nil
         case string:
                 return PTString, nil
         case Key:
                 // TODO(riannucci): Check key for validity in its own namespace?
                 return PTKey, nil
         case time.Time:
                 err := error(nil)
@@ skipping 40 matching lines @@
                 o = v.Int()
         case reflect.Bool:
                 o = v.Bool()
         case reflect.String:
                 if t != typeOfBSKey {
                         o = v.String()
                 }
         case reflect.Float32, reflect.Float64:
                 o = v.Float()
         case reflect.Slice:
-                if t != typeOfByteString && t.Elem().Kind() == reflect.Uint8 {
+                if t.Elem().Kind() == reflect.Uint8 {
                         o = v.Bytes()
                 }
         case reflect.Struct:
                 if t == typeOfTime {
                         // time in a Property can only hold microseconds
                         o = v.Interface().(time.Time).Round(time.Microsecond)
                 }
         }
         return o
 }
 
 // Value returns the current value held by this property. It's guaranteed to
 // be a valid value type (i.e. `p.SetValue(p.Value(), true)` will never return
 // an error).
 func (p *Property) Value() interface{} { return p.value }
 
 // IndexSetting says weather or not the datastore should create indicies for
 // this value.
 func (p *Property) IndexSetting() IndexSetting { return p.indexSetting }
 
 // Type is the PT* type of the data contained in Value().
 func (p *Property) Type() PropertyType { return p.propType }
 
 // SetValue sets the Value field of a Property, and ensures that its value
 // conforms to the permissible types. That way, you're guaranteed that if you
 // have a Property, its value is valid.
 //
 // value is the property value. The valid types are:
 //      - int64
+//      - time.Time
 //      - bool
 //      - string
+//    (only the first 1500 bytes is indexable)
+//      - []byte
+//    (only the first 1500 bytes is indexable)
+//      - blobstore.Key
+//    (only the first 1500 bytes is indexable)
 //      - float64
-//      - ByteString
 //      - Key
-//      - time.Time
-//      - blobstore.Key
 //      - GeoPoint
-//      - []byte (up to 1 megabyte in length)
 // This set is smaller than the set of valid struct field types that the
 // datastore can load and save. A Property Value cannot be a slice (apart
 // from []byte); use multiple Properties instead. Also, a Value's type
 // must be explicitly on the list above; it is not sufficient for the
 // underlying type to be on that list. For example, a Value of "type
 // myInt64 int64" is invalid. Smaller-width integers and floats are also
 // invalid. Again, this is more restrictive than the set of valid struct
 // field types.
 //
 // A value may also be the nil interface value; this is equivalent to
 // Python's None but not directly representable by a Go struct. Loading
 // a nil-valued property into a struct will set that field to the zero
 // value.
 func (p *Property) SetValue(value interface{}, is IndexSetting) (err error) {
         pt := PTNull
         if value != nil {
                 value = UpconvertUnderlyingType(value)
                 if pt, err = PropertyTypeOf(value, true); err != nil {
                         return
                 }
         }
         p.propType = pt
         p.value = value
         p.indexSetting = is
-        if _, ok := value.([]byte); ok {
-                p.indexSetting = NoIndex
+        return
+}
+
+// ForIndex gets a new Property with its value and type converted as if it were
+// being stored in a datastore index. See the doc on PropertyType for more info.
+func (p Property) ForIndex() Property {
+        switch p.propType {
+        case PTNull, PTInt, PTBool, PTString, PTFloat, PTGeoPoint, PTKey:
+                return p
+
+        case PTTime:
+                v, _ := p.Project(PTInt)
+                return Property{v, p.indexSetting, PTInt}
+
+        case PTBytes, PTBlobKey:
+                v, _ := p.Project(PTString)
+                return Property{v, p.indexSetting, PTString}
         }
-        return
+        panic(fmt.Errorf("unknown PropertyType: %s", p.propType))
+}
+
+// Project can be used to project a Property retrieved from a Projection query
+// into a different datatype. For example, if you have a PTInt property, you
+// could Project(PTTime) to convert it to a time.Time. The following conversions
+// are supported:
+//   PTInt <-> PTTime

[dnj (Google), 2015/09/10 16:26:11]

Note that you can project a type to itself (identity) as well.

[iannucci, 2015/09/10 17:29:21]

Oh, right. Done.

+//   PTString <-> PTBlobKey
+//   PTString <-> PTBytes
+//   PTNull <-> Anything
+func (p *Property) Project(to PropertyType) (interface{}, error) {
+        switch {
+        case to == p.propType:
+                return p.value, nil
+
+        case to == PTInt && p.propType == PTTime:
+                t := p.value.(time.Time)
+                v := uint64(t.Unix())*1e6 + uint64(t.Nanosecond()/1e3)
+                return int64(v), nil
+
+        case to == PTTime && p.propType == PTInt:
+                v := p.value.(int64)
+                return time.Unix(int64(v/1e6), int64((v%1e6)*1e3)).UTC(), nil
+
+        case to == PTString && p.propType == PTBytes:
+                return string(p.value.([]byte)), nil
+
+        case to == PTString && p.propType == PTBlobKey:
+                return string(p.value.(blobstore.Key)), nil
+
+        case to == PTBytes && p.propType == PTString:
+                return []byte(p.value.(string)), nil
+
+        case to == PTBlobKey && p.propType == PTString:
+                return blobstore.Key(p.value.(string)), nil
+
+        case to == PTNull:
+                return nil, nil
+
+        case p.propType == PTNull:
+                switch to {
+                case PTInt:
+                        return int64(0), nil
+                case PTTime:
+                        return time.Time{}, nil
+                case PTBool:
+                        return false, nil
+                case PTBytes:
+                        return []byte(nil), nil
+                case PTString:
+                        return "", nil
+                case PTFloat:
+                        return float64(0), nil
+                case PTGeoPoint:
+                        return GeoPoint{}, nil
+                case PTKey:
+                        return nil, nil
+                case PTBlobKey:
+                        return blobstore.Key(""), nil
+                }
+                fallthrough
+        default:
+                return nil, fmt.Errorf("unable to project %s to %s", p.propType, to)
+        }
 }
 
 // MetaGetter is a subinterface of PropertyLoadSaver, but is also used to
 // abstract the meta argument for RawInterface.GetMulti.
 type MetaGetter interface {
         // GetMeta will get information about the field which has the struct tag in
         // the form of `gae:"$<key>[,<default>]?"`.
         //
         // Supported metadata types are:
         //   int64  - may have default (ascii encoded base-10)
@@ skipping 163 matching lines @@
         dflt = UpconvertUnderlyingType(dflt)
         cur, err := gm(key)
         if err != nil {
                 return dflt
         }
         if dflt != nil && reflect.TypeOf(cur) != reflect.TypeOf(dflt) {
                 return dflt
         }
         return cur
 }