Milo flex raw log viewer endpoint

milo/appengine/common/middleware.go

 // Copyright 2015 The LUCI Authors. All rights reserved.
 // Use of this source code is governed under the Apache License, Version 2.0
 // that can be found in the LICENSE file.
 
 package common
 
 import (
         "net/http"
         "strings"
 
+        "cloud.google.com/go/compute/metadata"
+        "cloud.google.com/go/datastore"
+
         "golang.org/x/net/context"
+        "golang.org/x/oauth2"
+        "golang.org/x/oauth2/google"
 
+        "github.com/luci/gae/impl/cloud"
+        "github.com/luci/gae/impl/memory"
         "github.com/luci/gae/service/info"
         "github.com/luci/luci-go/appengine/gaeauth/server"
+        "github.com/luci/luci-go/appengine/gaeauth/server/gaesigner"
         "github.com/luci/luci-go/appengine/gaemiddleware"
+        "github.com/luci/luci-go/appengine/gaesecrets"
         "github.com/luci/luci-go/common/clock"
+        "github.com/luci/luci-go/common/logging/gologger"
         "github.com/luci/luci-go/server/analytics"
         "github.com/luci/luci-go/server/auth"
+        "github.com/luci/luci-go/server/auth/authdb"
         "github.com/luci/luci-go/server/auth/identity"
         "github.com/luci/luci-go/server/router"
         "github.com/luci/luci-go/server/templates"
 )
 
 // GetTemplateBundles is used to render HTML templates. It provides base args
 // passed to all templates.
 func GetTemplateBundle() *templates.Bundle {
         return &templates.Bundle{
                 Loader:          templates.FileSystemLoader("templates"),
@@ skipping 18 matching lines @@
                                 "LogoutURL":   logoutURL,
                                 "CurrentTime": clock.Now(c),
                                 "Analytics":   analytics.Snippet(c),
                                 "RequestID":   info.RequestID(c),
                         }, nil
                 },
                 FuncMap: funcMap,
         }
 }
 
+// authMethods returns the set of authentication methods used for Milo.
+func authMethods() auth.Authenticator {
+        return auth.Authenticator{
+                &server.OAuth2Method{Scopes: []string{server.EmailScope}},

[Vadim Sh., 2017/04/18 00:33:24]

fyi: this doesn't work on Flex currently

[hinoka, 2017/04/21 22:12:41]

I'll wait to rebase https://codereview.chromium.org/2830443003/ in

+                server.CookieAuth,
+                &server.InboundAppIDAuthMethod{},

[Vadim Sh., 2017/04/18 00:33:24]

nit: drop this, we don't actually use it, and it won't work on Flex (because it
uses magic GAE headers)

[hinoka, 2017/04/21 22:12:41]

Same as above

+        }
+}
+
+// Flex returns the basic middleware for use on appengine flex.  Flex does not
+// allow the use of appengine APIs.
+func FlexBase() router.MiddlewareChain {
+        // Get the name of this project from the metadata server, since we're on GCE.
+        project, err := metadata.Get("project/project-id")
+        if err != nil {
+                panic(err)
+        }
+        // Use the standard Go logger.
+        // TODO(hinoka): Use the cloud logger, somehow.

[Vadim Sh., 2017/04/18 00:33:24]

There's this:
https://github.com/luci/luci-go/blob/master/common/logging/cloudlog/logging.g...

I believe some logdog guts use it.

[hinoka, 2017/04/21 22:12:41]

Done.

+        logger := func(c *router.Context, next router.Handler) {
+                c.Context = gologger.StdConfig.Use(c.Context)
+                next(c)
+        }
+        // The base set of services uses memory implementations.
+        memory := func(c *router.Context, next router.Handler) {
+                c.Context = memory.UseWithAppID(c.Context, project)

[Vadim Sh., 2017/04/18 00:33:24]

hm... This looks dangerous. Most services there are mocks. Dangerous, because
instead of failing, the code may do slightly wrong thing. For example, memcache
is assumed to be globally consistent. With impl/memory implementation, it is
consistent only within the scope of one process. It may break code that makes
assumptions about global nature of memcache. For example, we use memcache-based
global locks in some places.

A less dangerous approach would be to add only service that really work, and
carefully refactor code that depends on other services (like memcache) to not
depend on them (somehow).

[hinoka, 2017/04/21 22:12:41]

Addressing in https://codereview.chromium.org/2829903002 (Need to roll that in
first before landing this)

+                next(c)
+        }
+        // Use the cloud datastore client.
+        ds := func(c *router.Context, next router.Handler) {
+                client, err := datastore.NewClient(c.Context, project)
+                if err != nil {
+                        panic(err)
+                }
+                c.Context = cloud.UseDatastore(c.Context, client)
+                next(c)
+        }
+        // Secret store, used for decryping OID tokens, which is needed for cookieauth.
+        secrets := func(c *router.Context, next router.Handler) {
+                c.Context = gaesecrets.Use(c.Context, nil)
+                next(c)
+        }
+        // Swap out the default auth configs with one that uses a in-memory cache, since
+        // we can't use memcache.
+        authConfig := func(c *router.Context, next router.Handler) {
+                mustToken := func(c context.Context, scopes []string) (*oauth2.Token, error) {
+                        // Ignore context and scope since its not needed.
+                        return google.ComputeTokenSource("").Token()
+                }
+                config := auth.Config{
+                        DBProvider:          authdb.NewDBCache(server.GetAuthDB),

[Vadim Sh., 2017/04/18 00:33:24]

'config' variable should be global. It is stateful, in particular DBProvider and
Cache fields point to caches.

Currently each invocation of FlexBase() will generate new copy of all these
caches, this is not good.

[hinoka, 2017/04/21 22:12:41]

Moved to init()

+                        Signer:              gaesigner.Signer{},

[Vadim Sh., 2017/04/18 00:33:24]

fyi: this will not work, but milo isn't signing anything, so it's fine.

[hinoka, 2017/04/21 22:12:41]

Acknowledged.

+                        AccessTokenProvider: mustToken,
+                        AnonymousTransport: func(c context.Context) http.RoundTripper {
+                                return http.DefaultTransport
+                        },
+                        Cache:     auth.MemoryCache(10),

[Vadim Sh., 2017/04/18 00:33:24]

nit: 1000 or something
we have gigs of ram on Flex IIUC :)

[hinoka, 2017/04/21 22:12:41]

Done.

+                        IsDevMode: false,
+                }
+                c.Context = auth.SetConfig(c.Context, config)
+                next(c)
+        }
+        // Now chain it all together!
+        return router.NewMiddlewareChain(
+                logger, memory, ds, secrets, auth.Use(authMethods()), authConfig, auth.Authenticate)

[Vadim Sh., 2017/04/18 00:33:24]

I think authConfig should be before auth.Use(...)

[hinoka, 2017/04/21 22:12:41]

Done.

+}
+
 // Base returns the basic LUCI appengine middlewares.
 func Base() router.MiddlewareChain {
-        methods := auth.Authenticator{
-                &server.OAuth2Method{Scopes: []string{server.EmailScope}},
-                server.CookieAuth,
-                &server.InboundAppIDAuthMethod{},
-        }
-        m := gaemiddleware.BaseProd().Extend(auth.Use(methods), auth.Authenticate)
+        m := gaemiddleware.BaseProd().Extend(auth.Use(authMethods()), auth.Authenticate)
         m = m.Extend(withRequestMiddleware)
         m = m.Extend(templates.WithTemplates(GetTemplateBundle()))
         return m
 }
 
 // The context key, so that we can embed the http.Request object into
 // the context.
 var requestKey = "http.request"
 
 // WithRequest returns a context with the http.Request object
 // in it.
 func WithRequest(c context.Context, r *http.Request) context.Context {
         return context.WithValue(c, &requestKey, r)
 }
 
 // withRequestMiddleware is a middleware that installs a request into the context.
 // This is used for various things in the default template.
 func withRequestMiddleware(c *router.Context, next router.Handler) {
         c.Context = WithRequest(c.Context, c.Request)
         next(c)
 }
 
 func getRequest(c context.Context) *http.Request {
         if req, ok := c.Value(&requestKey).(*http.Request); ok {
                 return req
         }
         panic("No http.request found in context")
 }