Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(631)

Side by Side Diff: service/datastore/dumper/dumper.go

Issue 2037433002: Add datastore dumper tool. (Closed) Base URL: https://chromium.googlesource.com/external/github.com/luci/gae@allow_unsigned
Patch Set: scope Created 4 years, 6 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
« no previous file with comments | « no previous file | service/datastore/properties.go » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
(Empty)
1 // Copyright 2016 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 // Package dumper implements a very VERY dumb datastore-dumping debugging aid.
6 // You shouldn't plan on having this work with the production datastore with any
7 // appreciable amount of data.
8 //
9 // This will take an arbitrary query (or even a query for every entity in the
10 // entire datastore), and print every entity to some output stream.
11 package dumper
12
13 import (
14 "fmt"
15 "io"
16 "os"
17 "sort"
18 "strings"
19
20 "github.com/luci/gae/service/datastore"
21 "golang.org/x/net/context"
22 )
23
24 // Key is a key into a PropFilterMap
25 type Key struct {
26 Kind string
27 PropName string
28 }
29
30 // A PropFilterMap maps from Kind+PropertyName tuples to a formatting function. You
31 // may use this to specially format particular properties.
32 type PropFilterMap map[Key]func(datastore.Property) string
33
34 // KindFilterMap maps from a Kind to a formatting function. You may use this to
35 // specially format particular Kinds. If this function returns an empty string,
36 // the default formatting function (including any PropFilterMap entries) will be
37 // used.
38 type KindFilterMap map[string]func(datastore.PropertyMap) string
39
40 // Config is a configured dumper.
41 type Config struct {
42 // OutStream is the output stream to use. If this is nil, os.Stdout will be
43 // used.
44 OutStream io.Writer
45
46 // WithSpecial, if true, includes entities which have kinds that begin a nd
47 // end with "__". By default, these entities are skipped.
48 WithSpecial bool
49
50 // PropFilters is an optional property filter map for controlling the
51 // rendering of certain Kind/Property values.
52 PropFilters PropFilterMap
53
54 // KindFilters is an optional kind filter for controlling the rendering of
55 // certain Kind values.
56 KindFilters KindFilterMap
57 }
58
59 // Query will dump everything matching the provided query.
60 //
61 // If the provided query is nil, a kindless query without any filters will be
62 // used.
63 func (cfg Config) Query(c context.Context, q *datastore.Query) (n int, err error ) {
64 ds := datastore.Get(c)
65
66 if q == nil {
67 q = datastore.NewQuery("")
68 }
69
70 out := cfg.OutStream
71 if out == nil {
72 out = os.Stdout
73 }
74
75 fmtVal := func(kind, name string, prop datastore.Property) string {
76 if fn := cfg.PropFilters[Key{kind, name}]; fn != nil {
77 return fn(prop)
78 }
79 return prop.String()
80 }
81
82 prnt := func(format string, args ...interface{}) bool {
dnj (Google) 2016/06/02 03:55:58 I don't think "bool on error, check bool" is any b
83 var amt int
84 amt, err = fmt.Fprintf(out, format, args)
85 n += amt
86 return err != nil
87 }
88
89 prop := func(kind, name string, vals []datastore.Property) bool {
90 if len(vals) <= 1 {
91 return prnt(" %q: [%s]\n", name, fmtVal(kind, name, val s[0]))
92 }
93 if !prnt(" %q: [\n %s", name, fmtVal(kind, name, vals[0])) {
94 return false
95 }
96 for _, v := range vals[1:] {
97 if !prnt(",\n %s\n", fmtVal(kind, name, v)) {
98 return false
99 }
100 }
101 return prnt("\n]\n")
102 }
103
104 runErr := ds.Run(q, func(pm datastore.PropertyMap) error {
105 key := datastore.GetMetaDefault(pm, "key", nil).(*datastore.Key)
106 if !cfg.WithSpecial && strings.HasPrefix(key.Kind(), "__") && st rings.HasSuffix(key.Kind(), "__") {
107 return nil
108 }
109 if !prnt("\n%s:\n", key) {
110 return datastore.Stop
dnj (Google) 2016/06/02 03:55:58 Why do we stop on print failure here? Seems like a
iannucci 2016/06/02 22:59:45 the only errors which may occur are IO errors, whi
dnj 2016/06/06 14:01:38 Acknowledged.
111 }
112 pm, _ = pm.Save(false)
113
114 // See if we have a KindFilter for this
115 if flt, ok := cfg.KindFilters[key.Kind()]; ok {
116 if kindOut := flt(pm); kindOut != "" {
117 if !prnt("%s", kindOut) {
118 return datastore.Stop
119 }
120 return nil
121 }
122 }
123
124 keys := make([]string, 0, len(pm))
125 for k := range pm {
126 keys = append(keys, k)
127 }
128 sort.Strings(keys)
129 for _, k := range keys {
130 prop(key.Kind(), k, pm[k])
dnj (Google) 2016/06/02 03:55:58 If one prop fails, then another succeeds, we mask
131 }
132 return nil
133 })
134 if err == nil {
135 err = runErr
dnj (Google) 2016/06/02 03:55:58 If you return inner error through Run, you won't n
136 }
137 return
138 }
139
140 // Query dumps the provided query to stdout without special entities and with
141 // default rendering.
142 func Query(c context.Context, q *datastore.Query) {
143 Config{}.Query(c, q)
144 }
145
146 // All dumps all entities to stdout without special entities and with default
147 // rendering.
148 func All(c context.Context) {
149 Config{}.Query(c, nil)
150 }
OLDNEW
« no previous file with comments | « no previous file | service/datastore/properties.go » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698