| OLD | NEW |
|---|
| (Empty) |
| 1 // Copyright 2015 The LUCI Authors. | |
| 2 // | |
| 3 // Licensed under the Apache License, Version 2.0 (the "License"); | |
| 4 // you may not use this file except in compliance with the License. | |
| 5 // You may obtain a copy of the License at | |
| 6 // | |
| 7 // http://www.apache.org/licenses/LICENSE-2.0 | |
| 8 // | |
| 9 // Unless required by applicable law or agreed to in writing, software | |
| 10 // distributed under the License is distributed on an "AS IS" BASIS, | |
| 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| 12 // See the License for the specific language governing permissions and | |
| 13 // limitations under the License. | |
| 14 | |
| 15 package hierarchy | |
| 16 | |
| 17 import ( | |
| 18 "crypto/sha256" | |
| 19 "encoding/base64" | |
| 20 "encoding/hex" | |
| 21 | |
| 22 log "github.com/luci/luci-go/common/logging" | |
| 23 "github.com/luci/luci-go/grpc/grpcutil" | |
| 24 "github.com/luci/luci-go/logdog/appengine/coordinator" | |
| 25 "github.com/luci/luci-go/logdog/common/types" | |
| 26 "github.com/luci/luci-go/luci_config/common/cfgtypes" | |
| 27 | |
| 28 ds "github.com/luci/gae/service/datastore" | |
| 29 | |
| 30 "golang.org/x/net/context" | |
| 31 "google.golang.org/grpc/codes" | |
| 32 ) | |
| 33 | |
| 34 // componentEntity is a hierarchial component that stores a specific log path | |
| 35 // component. A given stream path is broken into components, each of which is | |
| 36 // represented by a single componentEntity. | |
| 37 // | |
| 38 // A componentEntity is keyed based on its path component name and the hash of | |
| 39 // its parent component's path. This ensures that the set of child components | |
| 40 // can be obtained for any parent component path. | |
| 41 // | |
| 42 // The child component's key is chosen to be sortable with the following | |
| 43 // preferences: | |
| 44 // - Stream components sort before path components. | |
| 45 // - Stream components sort alphanumerically. Elements that are entirely | |
| 46 // numeric will be constructed to sort numerically. | |
| 47 // | |
| 48 // Storing the component's name in its key ensures that a keys query will pull | |
| 49 // the set of elements in the correct order, requiring no non-default indexes. | |
| 50 // | |
| 51 // Writing path components is inherently idempotent, since each one is defined | |
| 52 // solely by its identity. Consequently, transactions are not necessary when | |
| 53 // writing path components. | |
| 54 // | |
| 55 // All componentEntity share a common implicit ancestor, "/". | |
| 56 // | |
| 57 // This entity is created at stream and prefix registration, and is entirely | |
| 58 // centered around being queried for log stream "directory" listings. For | |
| 59 // example: | |
| 60 // | |
| 61 // foo/bar/+/baz/qux | |
| 62 // | |
| 63 // This stream would add the following name components to the datastore, keyed | |
| 64 // (parent-key, id) as: | |
| 65 // ("/", "foo") | |
| 66 // ("/foo", "bar") | |
| 67 // ("/foo/bar", "+") | |
| 68 // ("/foo/bar/+", "baz") | |
| 69 // ("/foo/bar/+/baz", "qux") | |
| 70 // | |
| 71 // A generated property, "s", is stored to indicate whether this entry is a | |
| 72 // stream or path component. | |
| 73 type componentEntity struct { | |
| 74 // _kind is the entity's kind. This is intentionally short to mitigate i
ndex | |
| 75 // bloat since this will be repeated in a lot of keys. | |
| 76 _kind string `gae:"$kind,_StreamNameComponent"` | |
| 77 | |
| 78 // ID is the name component of this specific stream. | |
| 79 ID componentID `gae:"$id"` | |
| 80 } | |
| 81 | |
| 82 var _ ds.PropertyLoadSaver = (*componentEntity)(nil) | |
| 83 | |
| 84 func (c *componentEntity) Load(pmap ds.PropertyMap) error { | |
| 85 // Delete custom elements (added in Save). | |
| 86 delete(pmap, "s") | |
| 87 delete(pmap, "p") | |
| 88 | |
| 89 return ds.GetPLS(c).Load(pmap) | |
| 90 } | |
| 91 | |
| 92 func (c *componentEntity) Save(withMeta bool) (ds.PropertyMap, error) { | |
| 93 pmap, err := ds.GetPLS(c).Save(withMeta) | |
| 94 if err != nil { | |
| 95 return nil, err | |
| 96 } | |
| 97 pmap["s"] = ds.MkProperty(c.ID.stream) | |
| 98 pmap["p"] = ds.MkProperty(c.ID.parent) | |
| 99 return pmap, nil | |
| 100 } | |
| 101 | |
| 102 // streamPath returns the StreamPath of this component, given its parent path. | |
| 103 // | |
| 104 // This is only valid if the component is a stream component. | |
| 105 func (c *componentEntity) streamPath(parent types.StreamPath) types.StreamPath { | |
| 106 return parent.Append(c.ID.name) | |
| 107 } | |
| 108 | |
| 109 func componentEntityParent(parent types.StreamPath) string { | |
| 110 hash := sha256.Sum256([]byte(parent)) | |
| 111 return hex.EncodeToString(hash[:]) | |
| 112 } | |
| 113 | |
| 114 func mkComponentEntity(parent types.StreamPath, name string, stream bool) *compo
nentEntity { | |
| 115 return &componentEntity{ | |
| 116 ID: componentID{ | |
| 117 parent: componentEntityParent(parent), | |
| 118 name: name, | |
| 119 stream: stream, | |
| 120 }, | |
| 121 } | |
| 122 } | |
| 123 | |
| 124 // Request is a listing request to execute. | |
| 125 // | |
| 126 // It describes a hierarchy listing request. For example, given the following | |
| 127 // streams in project "qux": | |
| 128 // qux/foo/+/bar | |
| 129 // qux/foo/+/bar/baz | |
| 130 // qux/foo/bar/+/baz | |
| 131 // | |
| 132 // The following queries would return values ("$" denotes streams vs. paths): | |
| 133 // Project="", Path="": ["qux"] | |
| 134 // Project="qux", Path="": ["foo"] | |
| 135 // Project="qux", Path="foo": ["+", "bar"] | |
| 136 // Project="qux", Path="foo/+": ["bar$", "bar"] | |
| 137 // Project="qux", Path="foo/bar": ["+"] | |
| 138 // Project="qux", Path="foo/bar/+": ["baz$"] | |
| 139 // | |
| 140 // If Limit is >0, it will be used to constrain the results. Otherwise, all | |
| 141 // results will be returned. | |
| 142 // | |
| 143 // If Next is not empty, it is a datastore cursor for a continued query. If | |
| 144 // supplied, it must use the same parameters as the previous queries in the | |
| 145 // sequence. | |
| 146 type Request struct { | |
| 147 // Project is the project to list. If empty, Request will perform a | |
| 148 // project-level listing. | |
| 149 Project string | |
| 150 // PathBase is the base path within Project to list. | |
| 151 PathBase string | |
| 152 // StreamOnly, if true, only returns stream path components. | |
| 153 StreamOnly bool | |
| 154 | |
| 155 // Limit, if >0, is the maximum number of results to return. If more res
ults | |
| 156 // are available, the returned List will have its Next field set to a cu
rsor | |
| 157 // that can be used to issue iterative requests. | |
| 158 Limit int | |
| 159 // Next, if not empty, is the start cursor for this stream. | |
| 160 Next string | |
| 161 // Skip, if >0, skips past the first Skip query results. | |
| 162 Skip int | |
| 163 } | |
| 164 | |
| 165 // ListComponent is a single component element in the stream path hierarchy. | |
| 166 // | |
| 167 // This can represent a path component "path" in (path/component/+/stream) | |
| 168 // or a stream component ("stream"). | |
| 169 type ListComponent struct { | |
| 170 // Name is the name of this hierarchy element. | |
| 171 Name string | |
| 172 // Stream, if true, indicates that this is a stream component. | |
| 173 Stream bool | |
| 174 } | |
| 175 | |
| 176 // List is a branch of the stream path tree. | |
| 177 // | |
| 178 // It may represent either the top-level project hierarchy, or the sub-project | |
| 179 // stream space hierarchy, depending on the query base. | |
| 180 type List struct { | |
| 181 // Project is the listed project name. If empty, the list refers to the | |
| 182 // project namespace. | |
| 183 Project cfgtypes.ProjectName | |
| 184 // PathBase is the stream path base. | |
| 185 PathBase types.StreamPath | |
| 186 // Comp is the set of elements in this hierarchy result. | |
| 187 Comp []*ListComponent | |
| 188 | |
| 189 // Next, if not empty, is the iterative query cursor. | |
| 190 Next string | |
| 191 } | |
| 192 | |
| 193 // Path returns the StreamPath of the supplied Component. | |
| 194 func (l *List) Path(c *ListComponent) types.StreamPath { | |
| 195 return l.PathBase.Append(c.Name) | |
| 196 } | |
| 197 | |
| 198 // Get performs a hierarchy query based on parameters in the supplied Request | |
| 199 // and returns the resulting List. | |
| 200 // | |
| 201 // This method will set the namespace based on the request after asserting user | |
| 202 // membership. | |
| 203 // | |
| 204 // The supplied Context should not be bound to a namespace (i.e., default | |
| 205 // namespace). | |
| 206 // | |
| 207 // If a failure is encountered, a wrapped gRPC error will be returned. | |
| 208 func Get(c context.Context, r Request) (*List, error) { | |
| 209 // If our project is empty, this is a project-level query. | |
| 210 if r.Project == "" { | |
| 211 return getProjects(c, &r) | |
| 212 } | |
| 213 | |
| 214 // Build our List result. | |
| 215 // | |
| 216 // We will validate our Project and PathBase types immediately afterward
s. | |
| 217 l := List{ | |
| 218 Project: cfgtypes.ProjectName(r.Project), | |
| 219 PathBase: types.StreamPath(r.PathBase), | |
| 220 } | |
| 221 | |
| 222 // Validate our PathBase component. | |
| 223 if err := l.PathBase.ValidatePartial(); err != nil { | |
| 224 return nil, grpcutil.Errf(codes.InvalidArgument, "invalid stream
path base %q: %v", l.PathBase, err) | |
| 225 } | |
| 226 | |
| 227 // Enter the supplied Project namespace. This will assert the the user h
as | |
| 228 // access to the project. | |
| 229 if err := coordinator.WithProjectNamespace(&c, l.Project, coordinator.Na
mespaceAccessREAD); err != nil { | |
| 230 return nil, err | |
| 231 } | |
| 232 | |
| 233 // Determine our ancestor component. | |
| 234 q := ds.NewQuery("_StreamNameComponent") | |
| 235 q = q.Eq("p", componentEntityParent(l.PathBase)) | |
| 236 if r.StreamOnly { | |
| 237 q = q.Eq("s", true) | |
| 238 } | |
| 239 if r.Next != "" { | |
| 240 k, err := keyForCursor(c, r.Next) | |
| 241 if err != nil { | |
| 242 return nil, grpcutil.Errf(codes.InvalidArgument, "invali
d cursor: %s", err) | |
| 243 } | |
| 244 q = q.Gt("__key__", k) | |
| 245 } | |
| 246 if r.Skip > 0 { | |
| 247 q = q.Offset(int32(r.Skip)) | |
| 248 } | |
| 249 | |
| 250 limit := r.Limit | |
| 251 if limit > 0 { | |
| 252 q = q.Limit(int32(limit)) | |
| 253 } | |
| 254 | |
| 255 err := ds.Run(c, q, func(e *componentEntity) error { | |
| 256 l.Comp = append(l.Comp, &ListComponent{ | |
| 257 Name: e.ID.name, | |
| 258 Stream: e.ID.stream, | |
| 259 }) | |
| 260 | |
| 261 if limit > 0 && len(l.Comp) >= limit { | |
| 262 l.Next = cursorForKey(c, e) | |
| 263 return ds.Stop | |
| 264 } | |
| 265 return nil | |
| 266 }) | |
| 267 if err != nil { | |
| 268 log.WithError(err).Errorf(c, "Failed to execute hierarhcy query.
") | |
| 269 return nil, grpcutil.Internal | |
| 270 } | |
| 271 return &l, nil | |
| 272 } | |
| 273 | |
| 274 // keyForCursor returns the component key for the supplied cursor. | |
| 275 // | |
| 276 // If the cursor string is not valid, an error will be returned. | |
| 277 func keyForCursor(c context.Context, curs string) (*ds.Key, error) { | |
| 278 d, err := base64.URLEncoding.DecodeString(curs) | |
| 279 if err != nil { | |
| 280 return nil, err | |
| 281 } | |
| 282 | |
| 283 return ds.NewKey(c, "_StreamNameComponent", string(d), 0, nil), nil | |
| 284 } | |
| 285 | |
| 286 // cursorForKey returns a cursor for the supplied componentID. This cursor will | |
| 287 // start new queries at the component immediately following this ID. | |
| 288 func cursorForKey(c context.Context, e *componentEntity) string { | |
| 289 key := ds.KeyForObj(c, e) | |
| 290 return base64.URLEncoding.EncodeToString([]byte(key.StringID())) | |
| 291 } | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 2991253004:
[logdog] Remove list functionality. (Closed)
