Add ability to estimate the size of a PropertyMap

service/datastore/properties.go

 // Copyright 2015 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 package datastore
 
 import (
         "encoding/base64"
         "errors"
         "fmt"
@@ skipping 570 matching lines @@
         panic(fmt.Errorf("bad type: %s", p.propType))
 }
 
 // PropertySlice is a slice of Properties. It implements sort.Interface.
 type PropertySlice []Property
 
 func (s PropertySlice) Len() int           { return len(s) }
 func (s PropertySlice) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
 func (s PropertySlice) Less(i, j int) bool { return s[i].Less(&s[j]) }
 
+// EstimateSize estimates the amount of space that this Property would consume
+// if it were committed as part of an entity in the real production datastore.
+//
+// It uses https://cloud.google.com/appengine/articles/storage_breakdown?csw=1
+// as a guide for these values.
+func (p *Property) EstimateSize(withNamespace bool) int {
+        switch p.Type() {
+        case PTNull:
+                return 1
+        case PTBool:
+                return 1 + 4
+        case PTInt, PTTime, PTFloat:
+                return 1 + 8
+        case PTGeoPoint:
+                return 1 + (8 * 2)
+        case PTString:
+                return 1 + len(p.value.(string))
+        case PTBlobKey:
+                return 1 + len(p.value.(blobstore.Key))
+        case PTBytes:
+                return 1 + len(p.value.([]byte))
+        case PTKey:
+                return 1 + p.value.(*Key).EstimateSize(withNamespace)
+        }
+        panic(fmt.Errorf("Unknown property type: %s", p.Type().String()))
+}
+
 // MetaGetter is a subinterface of PropertyLoadSaver, but is also used to
 // abstract the meta argument for RawInterface.GetMulti.
 type MetaGetter interface {
         // GetMeta will get information about the field which has the struct tag in
         // the form of `gae:"$<key>[,<default>]?"`.
         //
         // Supported metadata types are:
         //   int64  - may have default (ascii encoded base-10)
         //   string - may have default
         //   Toggle - MUST have default ("true" or "false")
@@ skipping 159 matching lines @@
         }
         pm["$"+key] = []Property{prop}
         return nil
 }
 
 // Problem implements PropertyLoadSaver.Problem. It ALWAYS returns nil.
 func (pm PropertyMap) Problem() error {
         return nil
 }
 
+// EstimateSize estimates the size that it would take to encode this PropertyMap
+// in the production Appengine datastore. The calculation excludes metadata
+// fields in the map.
+//
+// It uses https://cloud.google.com/appengine/articles/storage_breakdown?csw=1
+// as a guide for sizes.
+func (pm PropertyMap) EstimateSize(withNamespace bool) int {

[dnj (Google), 2015/09/23 20:03:19]

Accumulating int, maybe should use int64?

[dnj (Google), 2015/09/23 20:03:19]

Why is "withNamespace" not always "true"?

[iannucci, 2015/09/23 20:28:41]

good question. I looked at the doc and it seems like they're always encoded with
the namespace in the actual entity, but are duplicately encoded in the row key
but with a special prefix delta scheme (of unspecified nature).

+        ret := 0
+        for k, vals := range pm {
+                if !isMetaKey(k) {
+                        ret += len(k)
+                        for i := range vals {
+                                ret += vals[i].EstimateSize(withNamespace)
+                        }
+                }
+        }
+        return ret
+}
+
 func isMetaKey(k string) bool {
         // empty counts as a metakey since it's not a valid data key, but it's
         // not really a valid metakey either.
         return k == "" || k[0] == '$'
 }
 
 // GetMetaDefaultImpl is the implementation of PropertyLoadSaver.GetMetaDefault.
 //
 // It takes the normal GetMeta function, the key and the default, and returns
 // the value according to PropertyLoadSaver.GetMetaDefault.
 func GetMetaDefaultImpl(gm func(string) (interface{}, error), key string, dflt interface{}) interface{} {
         dflt = UpconvertUnderlyingType(dflt)
         cur, err := gm(key)
         if err != nil {
                 return dflt
         }
         if dflt != nil && reflect.TypeOf(cur) != reflect.TypeOf(dflt) {
                 return dflt
         }
         return cur
 }