| OLD | NEW |
|---|
| (Empty) |
| 1 // Copyright 2015 The Chromium Authors. All rights reserved. | |
| 2 // Use of this source code is governed by a BSD-style license that can be | |
| 3 // found in the LICENSE file. | |
| 4 | |
| 5 // HEAVILY adapted from github.com/golang/appengine/datastore | |
| 6 | |
| 7 package helper | |
| 8 | |
| 9 import ( | |
| 10 "errors" | |
| 11 "fmt" | |
| 12 "reflect" | |
| 13 "strconv" | |
| 14 "strings" | |
| 15 "sync" | |
| 16 "time" | |
| 17 "unicode" | |
| 18 | |
| 19 "github.com/luci/gae" | |
| 20 ) | |
| 21 | |
| 22 // Entities with more than this many indexed properties will not be saved. | |
| 23 const maxIndexedProperties = 20000 | |
| 24 | |
| 25 var ( | |
| 26 typeOfDSKey = reflect.TypeOf((*gae.DSKey)(nil)).Elem() | |
| 27 typeOfDSPropertyConverter = reflect.TypeOf((*gae.DSPropertyConverter)(ni
l)).Elem() | |
| 28 typeOfGeoPoint = reflect.TypeOf(gae.DSGeoPoint{}) | |
| 29 typeOfTime = reflect.TypeOf(time.Time{}) | |
| 30 typeOfString = reflect.TypeOf("") | |
| 31 typeOfInt64 = reflect.TypeOf(int64(0)) | |
| 32 typeOfBool = reflect.TypeOf(true) | |
| 33 | |
| 34 valueOfnilDSKey = reflect.Zero(typeOfDSKey) | |
| 35 ) | |
| 36 | |
| 37 type structTag struct { | |
| 38 name string | |
| 39 idxSetting gae.IndexSetting | |
| 40 isSlice bool | |
| 41 substructCodec *structCodec | |
| 42 convert bool | |
| 43 metaVal interface{} | |
| 44 canSet bool | |
| 45 } | |
| 46 | |
| 47 type structCodec struct { | |
| 48 byMeta map[string]int | |
| 49 byName map[string]int | |
| 50 byIndex []structTag | |
| 51 hasSlice bool | |
| 52 problem error | |
| 53 } | |
| 54 | |
| 55 type structPLS struct { | |
| 56 o reflect.Value | |
| 57 c *structCodec | |
| 58 } | |
| 59 | |
| 60 var _ gae.DSPropertyLoadSaver = (*structPLS)(nil) | |
| 61 | |
| 62 // typeMismatchReason returns a string explaining why the property p could not | |
| 63 // be stored in an entity field of type v.Type(). | |
| 64 func typeMismatchReason(val interface{}, v reflect.Value) string { | |
| 65 entityType := reflect.TypeOf(val) | |
| 66 return fmt.Sprintf("type mismatch: %s versus %v", entityType, v.Type()) | |
| 67 } | |
| 68 | |
| 69 func (p *structPLS) Load(propMap gae.DSPropertyMap) error { | |
| 70 if err := p.Problem(); err != nil { | |
| 71 return err | |
| 72 } | |
| 73 | |
| 74 convFailures := gae.MultiError(nil) | |
| 75 | |
| 76 t := reflect.Type(nil) | |
| 77 for name, props := range propMap { | |
| 78 multiple := len(props) > 1 | |
| 79 for i, prop := range props { | |
| 80 if reason := loadInner(p.c, p.o, i, name, prop, multiple
); reason != "" { | |
| 81 if t == nil { | |
| 82 t = p.o.Type() | |
| 83 } | |
| 84 convFailures = append(convFailures, &gae.ErrDSFi
eldMismatch{ | |
| 85 StructType: t, | |
| 86 FieldName: name, | |
| 87 Reason: reason, | |
| 88 }) | |
| 89 } | |
| 90 } | |
| 91 } | |
| 92 | |
| 93 if len(convFailures) > 0 { | |
| 94 return convFailures | |
| 95 } | |
| 96 | |
| 97 return nil | |
| 98 } | |
| 99 | |
| 100 func loadInner(codec *structCodec, structValue reflect.Value, index int, name st
ring, p gae.DSProperty, requireSlice bool) string { | |
| 101 var v reflect.Value | |
| 102 // Traverse a struct's struct-typed fields. | |
| 103 for { | |
| 104 fieldIndex, ok := codec.byName[name] | |
| 105 if !ok { | |
| 106 return "no such struct field" | |
| 107 } | |
| 108 v = structValue.Field(fieldIndex) | |
| 109 | |
| 110 st := codec.byIndex[fieldIndex] | |
| 111 if st.substructCodec == nil { | |
| 112 break | |
| 113 } | |
| 114 | |
| 115 if v.Kind() == reflect.Slice { | |
| 116 for v.Len() <= index { | |
| 117 v.Set(reflect.Append(v, reflect.New(v.Type().Ele
m()).Elem())) | |
| 118 } | |
| 119 structValue = v.Index(index) | |
| 120 requireSlice = false | |
| 121 } else { | |
| 122 structValue = v | |
| 123 } | |
| 124 // Strip the "I." from "I.X". | |
| 125 name = name[len(st.name):] | |
| 126 codec = st.substructCodec | |
| 127 } | |
| 128 | |
| 129 doConversion := func(v reflect.Value) (string, bool) { | |
| 130 a := v.Addr() | |
| 131 if conv, ok := a.Interface().(gae.DSPropertyConverter); ok { | |
| 132 err := conv.FromDSProperty(p) | |
| 133 if err != nil { | |
| 134 return err.Error(), true | |
| 135 } | |
| 136 return "", true | |
| 137 } | |
| 138 return "", false | |
| 139 } | |
| 140 | |
| 141 if ret, ok := doConversion(v); ok { | |
| 142 return ret | |
| 143 } | |
| 144 | |
| 145 var slice reflect.Value | |
| 146 if v.Kind() == reflect.Slice && v.Type().Elem().Kind() != reflect.Uint8
{ | |
| 147 slice = v | |
| 148 v = reflect.New(v.Type().Elem()).Elem() | |
| 149 } else if requireSlice { | |
| 150 return "multiple-valued property requires a slice field type" | |
| 151 } | |
| 152 | |
| 153 pVal := p.Value() | |
| 154 | |
| 155 if ret, ok := doConversion(v); ok { | |
| 156 if ret != "" { | |
| 157 return ret | |
| 158 } | |
| 159 } else { | |
| 160 knd := v.Kind() | |
| 161 if v.Type().Implements(typeOfDSKey) { | |
| 162 knd = reflect.Interface | |
| 163 } | |
| 164 switch knd { | |
| 165 case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, re
flect.Int64: | |
| 166 x, ok := pVal.(int64) | |
| 167 if !ok && pVal != nil { | |
| 168 return typeMismatchReason(pVal, v) | |
| 169 } | |
| 170 if v.OverflowInt(x) { | |
| 171 return fmt.Sprintf("value %v overflows struct fi
eld of type %v", x, v.Type()) | |
| 172 } | |
| 173 v.SetInt(x) | |
| 174 case reflect.Bool: | |
| 175 x, ok := pVal.(bool) | |
| 176 if !ok && pVal != nil { | |
| 177 return typeMismatchReason(pVal, v) | |
| 178 } | |
| 179 v.SetBool(x) | |
| 180 case reflect.String: | |
| 181 switch x := pVal.(type) { | |
| 182 case gae.BSKey: | |
| 183 v.SetString(string(x)) | |
| 184 case string: | |
| 185 v.SetString(x) | |
| 186 default: | |
| 187 if pVal != nil { | |
| 188 return typeMismatchReason(pVal, v) | |
| 189 } | |
| 190 } | |
| 191 case reflect.Float32, reflect.Float64: | |
| 192 x, ok := pVal.(float64) | |
| 193 if !ok && pVal != nil { | |
| 194 return typeMismatchReason(pVal, v) | |
| 195 } | |
| 196 if v.OverflowFloat(x) { | |
| 197 return fmt.Sprintf("value %v overflows struct fi
eld of type %v", x, v.Type()) | |
| 198 } | |
| 199 v.SetFloat(x) | |
| 200 case reflect.Interface: | |
| 201 x, ok := pVal.(gae.DSKey) | |
| 202 if !ok && pVal != nil { | |
| 203 return typeMismatchReason(pVal, v) | |
| 204 } | |
| 205 if x != nil { | |
| 206 v.Set(reflect.ValueOf(x)) | |
| 207 } | |
| 208 case reflect.Struct: | |
| 209 switch v.Type() { | |
| 210 case typeOfTime: | |
| 211 x, ok := pVal.(time.Time) | |
| 212 if !ok && pVal != nil { | |
| 213 return typeMismatchReason(pVal, v) | |
| 214 } | |
| 215 v.Set(reflect.ValueOf(x)) | |
| 216 case typeOfGeoPoint: | |
| 217 x, ok := pVal.(gae.DSGeoPoint) | |
| 218 if !ok && pVal != nil { | |
| 219 return typeMismatchReason(pVal, v) | |
| 220 } | |
| 221 v.Set(reflect.ValueOf(x)) | |
| 222 default: | |
| 223 panic(fmt.Errorf("helper: impossible: %s", typeM
ismatchReason(pVal, v))) | |
| 224 } | |
| 225 case reflect.Slice: | |
| 226 switch x := pVal.(type) { | |
| 227 case []byte: | |
| 228 v.SetBytes(x) | |
| 229 case gae.DSByteString: | |
| 230 v.SetBytes([]byte(x)) | |
| 231 default: | |
| 232 panic(fmt.Errorf("helper: impossible: %s", typeM
ismatchReason(pVal, v))) | |
| 233 } | |
| 234 default: | |
| 235 panic(fmt.Errorf("helper: impossible: %s", typeMismatchR
eason(pVal, v))) | |
| 236 } | |
| 237 } | |
| 238 if slice.IsValid() { | |
| 239 slice.Set(reflect.Append(slice, v)) | |
| 240 } | |
| 241 return "" | |
| 242 } | |
| 243 | |
| 244 func (p *structPLS) Save(withMeta bool) (gae.DSPropertyMap, error) { | |
| 245 size := len(p.c.byName) | |
| 246 if withMeta { | |
| 247 size += len(p.c.byMeta) | |
| 248 } | |
| 249 ret := make(gae.DSPropertyMap, size) | |
| 250 if _, err := p.save(ret, "", gae.ShouldIndex); err != nil { | |
| 251 return nil, err | |
| 252 } | |
| 253 if withMeta { | |
| 254 for k := range p.c.byMeta { | |
| 255 val, err := p.GetMeta(k) | |
| 256 if err != nil { | |
| 257 return nil, err // TODO(riannucci): should these
be ignored? | |
| 258 } | |
| 259 p := gae.DSProperty{} | |
| 260 if err = p.SetValue(val, gae.NoIndex); err != nil { | |
| 261 return nil, err | |
| 262 } | |
| 263 ret["$"+k] = []gae.DSProperty{p} | |
| 264 } | |
| 265 } | |
| 266 return ret, nil | |
| 267 } | |
| 268 | |
| 269 func (p *structPLS) save(propMap gae.DSPropertyMap, prefix string, is gae.IndexS
etting) (idxCount int, err error) { | |
| 270 if err = p.Problem(); err != nil { | |
| 271 return | |
| 272 } | |
| 273 | |
| 274 saveProp := func(name string, si gae.IndexSetting, v reflect.Value, st *
structTag) (err error) { | |
| 275 if st.substructCodec != nil { | |
| 276 count, err := (&structPLS{v, st.substructCodec}).save(pr
opMap, name, si) | |
| 277 if err == nil { | |
| 278 idxCount += count | |
| 279 if idxCount > maxIndexedProperties { | |
| 280 err = errors.New("gae: too many indexed
properties") | |
| 281 } | |
| 282 } | |
| 283 return err | |
| 284 } | |
| 285 | |
| 286 prop := gae.DSProperty{} | |
| 287 if st.convert { | |
| 288 prop, err = v.Addr().Interface().(gae.DSPropertyConverte
r).ToDSProperty() | |
| 289 } else { | |
| 290 err = prop.SetValue(v.Interface(), si) | |
| 291 } | |
| 292 if err != nil { | |
| 293 return err | |
| 294 } | |
| 295 propMap[name] = append(propMap[name], prop) | |
| 296 if prop.IndexSetting() == gae.ShouldIndex { | |
| 297 idxCount++ | |
| 298 if idxCount > maxIndexedProperties { | |
| 299 return errors.New("gae: too many indexed propert
ies") | |
| 300 } | |
| 301 } | |
| 302 return nil | |
| 303 } | |
| 304 | |
| 305 for i, st := range p.c.byIndex { | |
| 306 if st.name == "-" { | |
| 307 continue | |
| 308 } | |
| 309 name := st.name | |
| 310 if prefix != "" { | |
| 311 name = prefix + name | |
| 312 } | |
| 313 v := p.o.Field(i) | |
| 314 is1 := is | |
| 315 if st.idxSetting == gae.NoIndex { | |
| 316 is1 = gae.NoIndex | |
| 317 } | |
| 318 if st.isSlice { | |
| 319 for j := 0; j < v.Len(); j++ { | |
| 320 if err = saveProp(name, is1, v.Index(j), &st); e
rr != nil { | |
| 321 return | |
| 322 } | |
| 323 } | |
| 324 } else { | |
| 325 if err = saveProp(name, is1, v, &st); err != nil { | |
| 326 return | |
| 327 } | |
| 328 } | |
| 329 } | |
| 330 return | |
| 331 } | |
| 332 | |
| 333 func (p *structPLS) GetMeta(key string) (interface{}, error) { | |
| 334 if err := p.Problem(); err != nil { | |
| 335 return nil, err | |
| 336 } | |
| 337 idx, ok := p.c.byMeta[key] | |
| 338 if !ok { | |
| 339 return nil, gae.ErrDSMetaFieldUnset | |
| 340 } | |
| 341 st := p.c.byIndex[idx] | |
| 342 val := st.metaVal | |
| 343 f := p.o.Field(idx) | |
| 344 if st.canSet { | |
| 345 if !reflect.DeepEqual(reflect.Zero(f.Type()).Interface(), f.Inte
rface()) { | |
| 346 val = f.Interface() | |
| 347 } | |
| 348 } | |
| 349 return val, nil | |
| 350 } | |
| 351 | |
| 352 func (p *structPLS) SetMeta(key string, val interface{}) (err error) { | |
| 353 if err = p.Problem(); err != nil { | |
| 354 return | |
| 355 } | |
| 356 idx, ok := p.c.byMeta[key] | |
| 357 if !ok { | |
| 358 return gae.ErrDSMetaFieldUnset | |
| 359 } | |
| 360 if !p.c.byIndex[idx].canSet { | |
| 361 return fmt.Errorf("gae/helper: cannot set meta %q: unexported fi
eld", key) | |
| 362 } | |
| 363 p.o.Field(idx).Set(reflect.ValueOf(val)) | |
| 364 return nil | |
| 365 } | |
| 366 | |
| 367 func (p *structPLS) Problem() error { return p.c.problem } | |
| 368 | |
| 369 var ( | |
| 370 // The RWMutex is chosen intentionally, as the majority of access to the | |
| 371 // structCodecs map will be in parallel and will be to read an existing
codec. | |
| 372 // There's no reason to serialize goroutines on every | |
| 373 // gae.RawDatastore.{Get,Put}{,Multi} call. | |
| 374 structCodecsMutex sync.RWMutex | |
| 375 structCodecs = map[reflect.Type]*structCodec{} | |
| 376 ) | |
| 377 | |
| 378 // validPropertyName returns whether name consists of one or more valid Go | |
| 379 // identifiers joined by ".". | |
| 380 func validPropertyName(name string) bool { | |
| 381 if name == "" { | |
| 382 return false | |
| 383 } | |
| 384 for _, s := range strings.Split(name, ".") { | |
| 385 if s == "" { | |
| 386 return false | |
| 387 } | |
| 388 first := true | |
| 389 for _, c := range s { | |
| 390 if first { | |
| 391 first = false | |
| 392 if c != '_' && !unicode.IsLetter(c) { | |
| 393 return false | |
| 394 } | |
| 395 } else { | |
| 396 if c != '_' && !unicode.IsLetter(c) && !unicode.
IsDigit(c) { | |
| 397 return false | |
| 398 } | |
| 399 } | |
| 400 } | |
| 401 } | |
| 402 return true | |
| 403 } | |
| 404 | |
| 405 var ( | |
| 406 errRecursiveStruct = fmt.Errorf("(internal): struct type is recursively
defined") | |
| 407 ) | |
| 408 | |
| 409 func getStructCodecLocked(t reflect.Type) (c *structCodec) { | |
| 410 if c, ok := structCodecs[t]; ok { | |
| 411 return c | |
| 412 } | |
| 413 | |
| 414 me := func(fmtStr string, args ...interface{}) error { | |
| 415 return fmt.Errorf(fmtStr, args...) | |
| 416 } | |
| 417 | |
| 418 c = &structCodec{ | |
| 419 byIndex: make([]structTag, t.NumField()), | |
| 420 byName: make(map[string]int, t.NumField()), | |
| 421 byMeta: make(map[string]int, t.NumField()), | |
| 422 problem: errRecursiveStruct, // we'll clear this later if it's n
ot recursive | |
| 423 } | |
| 424 defer func() { | |
| 425 // If the codec has a problem, free up the indexes | |
| 426 if c.problem != nil { | |
| 427 c.byIndex = nil | |
| 428 c.byName = nil | |
| 429 c.byMeta = nil | |
| 430 } | |
| 431 }() | |
| 432 structCodecs[t] = c | |
| 433 | |
| 434 for i := range c.byIndex { | |
| 435 st := &c.byIndex[i] | |
| 436 f := t.Field(i) | |
| 437 name := f.Tag.Get("gae") | |
| 438 opts := "" | |
| 439 if i := strings.Index(name, ","); i != -1 { | |
| 440 name, opts = name[:i], name[i+1:] | |
| 441 } | |
| 442 st.canSet = f.PkgPath == "" // blank == exported | |
| 443 switch { | |
| 444 case name == "": | |
| 445 if !f.Anonymous { | |
| 446 name = f.Name | |
| 447 } | |
| 448 case name[0] == '$': | |
| 449 name = name[1:] | |
| 450 if _, ok := c.byMeta[name]; ok { | |
| 451 c.problem = me("meta field %q set multiple times
", "$"+name) | |
| 452 return | |
| 453 } | |
| 454 c.byMeta[name] = i | |
| 455 mv, err := convertMeta(opts, f.Type) | |
| 456 if err != nil { | |
| 457 c.problem = me("meta field %q has bad type: %s",
"$"+name, err) | |
| 458 return | |
| 459 } | |
| 460 st.metaVal = mv | |
| 461 fallthrough | |
| 462 case name == "-": | |
| 463 st.name = "-" | |
| 464 continue | |
| 465 default: | |
| 466 if !validPropertyName(name) { | |
| 467 c.problem = me("struct tag has invalid property
name: %q", name) | |
| 468 return | |
| 469 } | |
| 470 } | |
| 471 if !st.canSet { | |
| 472 st.name = "-" | |
| 473 continue | |
| 474 } | |
| 475 | |
| 476 substructType := reflect.Type(nil) | |
| 477 ft := f.Type | |
| 478 if reflect.PtrTo(ft).Implements(typeOfDSPropertyConverter) { | |
| 479 st.convert = true | |
| 480 } else { | |
| 481 switch f.Type.Kind() { | |
| 482 case reflect.Struct: | |
| 483 if ft != typeOfTime && ft != typeOfGeoPoint { | |
| 484 substructType = ft | |
| 485 } | |
| 486 case reflect.Slice: | |
| 487 if reflect.PtrTo(ft.Elem()).Implements(typeOfDSP
ropertyConverter) { | |
| 488 st.convert = true | |
| 489 } else if ft.Elem().Kind() == reflect.Struct { | |
| 490 substructType = ft.Elem() | |
| 491 } | |
| 492 st.isSlice = ft.Elem().Kind() != reflect.Uint8 | |
| 493 c.hasSlice = c.hasSlice || st.isSlice | |
| 494 case reflect.Interface: | |
| 495 if ft != typeOfDSKey { | |
| 496 c.problem = me("field %q has non-concret
e interface type %s", | |
| 497 f.Name, f.Type) | |
| 498 return | |
| 499 } | |
| 500 } | |
| 501 } | |
| 502 | |
| 503 if substructType != nil { | |
| 504 sub := getStructCodecLocked(substructType) | |
| 505 if sub.problem != nil { | |
| 506 if sub.problem == errRecursiveStruct { | |
| 507 c.problem = me("field %q is recursively
defined", f.Name) | |
| 508 } else { | |
| 509 c.problem = me("field %q has problem: %s
", f.Name, sub.problem) | |
| 510 } | |
| 511 return | |
| 512 } | |
| 513 st.substructCodec = sub | |
| 514 if st.isSlice && sub.hasSlice { | |
| 515 c.problem = me( | |
| 516 "flattening nested structs leads to a sl
ice of slices: field %q", | |
| 517 f.Name) | |
| 518 return | |
| 519 } | |
| 520 c.hasSlice = c.hasSlice || sub.hasSlice | |
| 521 if name != "" { | |
| 522 name += "." | |
| 523 } | |
| 524 for relName := range sub.byName { | |
| 525 absName := name + relName | |
| 526 if _, ok := c.byName[absName]; ok { | |
| 527 c.problem = me("struct tag has repeated
property name: %q", absName) | |
| 528 return | |
| 529 } | |
| 530 c.byName[absName] = i | |
| 531 } | |
| 532 } else { | |
| 533 if !st.convert { // check the underlying static type of
the field | |
| 534 t := ft | |
| 535 if st.isSlice { | |
| 536 t = t.Elem() | |
| 537 } | |
| 538 v := reflect.New(t).Elem().Interface() | |
| 539 v, _ = gae.DSUpconvertUnderlyingType(v, t) | |
| 540 if _, err := gae.DSPropertyTypeOf(v, false); err
!= nil { | |
| 541 c.problem = me("field %q has invalid typ
e: %s", name, ft) | |
| 542 return | |
| 543 } | |
| 544 } | |
| 545 | |
| 546 if _, ok := c.byName[name]; ok { | |
| 547 c.problem = me("struct tag has repeated property
name: %q", name) | |
| 548 return | |
| 549 } | |
| 550 c.byName[name] = i | |
| 551 } | |
| 552 st.name = name | |
| 553 if opts == "noindex" { | |
| 554 st.idxSetting = gae.NoIndex | |
| 555 } | |
| 556 } | |
| 557 if c.problem == errRecursiveStruct { | |
| 558 c.problem = nil | |
| 559 } | |
| 560 return | |
| 561 } | |
| 562 | |
| 563 func convertMeta(val string, t reflect.Type) (interface{}, error) { | |
| 564 switch t { | |
| 565 case typeOfString: | |
| 566 return val, nil | |
| 567 case typeOfInt64: | |
| 568 if val == "" { | |
| 569 return int64(0), nil | |
| 570 } | |
| 571 return strconv.ParseInt(val, 10, 64) | |
| 572 } | |
| 573 return nil, fmt.Errorf("helper: meta field with bad type/value %s/%s", t
, val) | |
| 574 } | |
| OLD | NEW |
|---|
Chromium Code Reviews
