A client/server helper wrapper for endpoints in Go.

go/src/infra/libs/epclient/epclient.go

+// Copyright 2015 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// Package epclient provides a simple implementation of a Google Cloud Endpoints
+// client (for services implemented in Go).
+//
+// Unlike a traditional endpoitns client, the interfaces and datatypes are
+// statically built with the application, using the "infra/gae/libs/endpoints"
+// library.
+//
+// Example usage:
+//
+//   import "github.com/GoogleCloudPlatform/go-endpoints/endpoints"
+//   import infra_endpoints "infra/gae/libs/endpoints"

[Vadim Sh., 2015/06/03 18:13:39]

I'm sort of against intermixing GAE and non-GAE libraries at this point, since
we didn't solve +appengine tag problem yet. Are you using infra/libs/epclient
from non-GAE environment? If not, move it to infra/gaes/libs.

+//   import "some/service"
+//
+//   // service.Info = *endpoints.ServiceInfo{...}
+//   // service.MethodInfo = *infra_endpoints.MethodInfoMap
+//
+//   // service.
+//
+//   func
+//
+package epclient
+
+import (
+        "bytes"
+        "encoding/json"
+        "fmt"
+        "golang.org/x/net/context"
+        "io"
+        "io/ioutil"
+        "net"
+        "net/http"
+        "reflect"
+        "regexp"
+        "strings"
+        "sync"
+        "text/template"
+        "time"
+
+        "github.com/GoogleCloudPlatform/go-endpoints/endpoints"
+        "github.com/luci/luci-go/common/logging"

[Vadim Sh., 2015/06/03 18:13:39]

nit: seems weird to group luci-go import for go-endpoints, but keep it separate
from another luci-go import...

+
+        "github.com/luci/luci-go/common/errors"
+)
+
+var backoffSlot = 50 * time.Millisecond
+
+// Backoff calculates an amount of time to back off, depending on the number
+// of failures which have occured so far. It is non-randomized to aid in
+// determinism during tests.
+func Backoff(failures uint) time.Duration {
+        const failcap = 10
+
+        if failures > failcap {
+                failures = failcap
+        }
+        return backoffSlot * time.Duration(1<<failures)
+}
+
+// Client is an endpoints-client which provides a very simple in/out RPC
+// interface for a given endpoints server.
+//
+// Mapping is done statically using the infra/gae/libs/endpoints.MethodInfoMap
+// of a given service. This means that you need to recompile the client any time
+// you change the interface of your service (but honestly, you probably needed
+// to do that anyway). Serialization and deserialization follow the
+// encoding/json rules. The service implementation should expose the correct
+// input and output types (along with any json field tags) necessary to interact
+// with the service.
+type Client interface {
+        // DoWithRetries will try the given method up to `upto` times before returning
+        // an error. An `upto` value of 0 will not do the API call at all.
+        //
+        // in and out must be the approprite concrete Go types which correspond to the
+        // method.
+        //
+        // methodName is the service method name, not the REST name for the api (e.g.
+        // it would be the Method portion of `Service.Method`, if you were hitting
+        // the spi interface).
+        //
+        // DoWithRetries will calculate a time to back off between attempts using
+        // the Backoff method.
+        DoWithRetries(serviceName, methodName string, in interface{}, out interface{}, upto uint) error

[Vadim Sh., 2015/06/03 18:13:39]

consider using https://github.com/luci/luci-go/tree/master/client/internal/retry

(it will be made available here https://codereview.chromium.org/1159563002/)

+
+        ForService(serviceName string) (ServiceClient, error)

[Vadim Sh., 2015/06/03 18:13:39]

document

+}
+
+// ServiceClient is like a Client, but restricted to a single service under a
+// given endpoints server.
+type ServiceClient interface {
+        DoWithRetries(methodName string, in interface{}, out interface{}, upto uint) error
+}
+
+var mark = errors.MakeMarkFn("endpoints")
+
+type client struct {
+        sync.Mutex
+
+        url string
+
+        ctx         context.Context
+        serverDef   *endpoints.Server
+        templateMap map[string]map[string]*templateEntry
+
+        // for testing. Returns equivalent of http.Do and a closer function which
+        // should be defer'd to close the underlying client.
+        mkHTTPDoer func() (
+                doer func(*http.Request) (*http.Response, error),
+                closer func())
+}
+
+var _ Client = (*client)(nil)

[Vadim Sh., 2015/06/03 18:13:39]

why? NewClient does this type check already.

+
+// NewClient returns a new Client object bound to hit `url`, which should be
+// the url to the service (e.g. "https://foo.appspot.com"). It will expect
+// to find the REST-style APIs at "<url>/_ah/api/<service>/<version>/<api>".
+func NewClient(ctx context.Context, url string, s *endpoints.Server) Client {
+        // TODO(riannucci): provide an 'spi' mode.
+        return &client{
+                url:         url,
+                ctx:         ctx,
+                serverDef:   s,
+                templateMap: map[string]map[string]*templateEntry{},
+                mkHTTPDoer:  mkHTTPDoer,
+        }
+}
+
+var getFieldsRE = regexp.MustCompile("{{\\.([^}]*)}}")
+
+type templateEntry struct {
+        *template.Template
+        used map[string]struct{}
+        si   *endpoints.ServiceInfo
+        mi   *endpoints.MethodInfo
+
+        reqType reflect.Type
+        rspType reflect.Type
+}
+
+func (c *client) getTemplateMap(serviceName string) map[string]*templateEntry {
+        c.Lock()
+        if m, ok := c.templateMap[serviceName]; ok {
+                c.Unlock()
+                return m
+        }
+        c.Unlock()
+
+        templateConvert := strings.NewReplacer("{", "{{.", "}", "}}")
+        service := c.serverDef.ServiceByName(serviceName)
+        if service == nil {
+                c.Lock()
+                defer c.Unlock()
+                c.templateMap[serviceName] = nil
+                return nil
+        }
+
+        methods := service.Methods()
+        ret := make(map[string]*templateEntry, len(methods))
+
+        // due to a bug (feature?) in go-endpoints, method.Info().Name is always
+        // lowercase, even when MethodByName expects the uppercase name.
+        // So let's reflect and grab those keys :)
+        methodNames := reflect.ValueOf(service).Elem().FieldByName("methods").MapKeys()
+        for _, nameV := range methodNames {
+                name := nameV.String()
+                method := service.MethodByName(name)
+                info := method.Info()
+                path := templateConvert.Replace(info.Path)
+
+                usedFields := map[string]struct{}{}
+                for _, submatches := range getFieldsRE.FindAllStringSubmatch(path, -1) {
+                        usedFields[submatches[1]] = struct{}{}
+                }
+
+                ret[name] = &templateEntry{
+                        template.Must(template.New(name).Parse(path)),
+                        usedFields,
+                        service.Info(),
+                        info,
+                        reflect.PtrTo(method.ReqType),
+                        reflect.PtrTo(method.RespType),
+                }
+        }
+
+        c.Lock()
+        defer c.Unlock()
+        c.templateMap[serviceName] = ret // who cares if we overwrite it?
+        return ret
+}
+
+func (c *client) getMethodInfo(serviceName, methodName string) (*templateEntry, error) {
+        m := c.getTemplateMap(serviceName)
+        if m == nil {
+                return nil, mark(fmt.Errorf("getMethodInfo: no such service %s", serviceName))
+        }
+
+        methodTmpl := m[methodName]
+        if methodTmpl == nil {
+                return nil, mark(fmt.Errorf("getMethodInfo: no such method %s.%s", serviceName, methodName))
+        }
+
+        return methodTmpl, nil
+}
+
+func renderURL(base string, t *templateEntry, requestData interface{}) (string, error) {
+        buf := &bytes.Buffer{}
+        if err := t.Execute(buf, requestData); err != nil {
+                return "", err
+        }
+
+        ret := fmt.Sprintf("%s/_ah/api/%s/%s/%s", base,
+                t.si.Name, t.si.Version, buf.String())
+
+        addedQuestion := false
+
+        if t.mi.HTTPMethod == "GET" && requestData != nil {
+                val := reflect.ValueOf(requestData)
+                stype := reflect.TypeOf(requestData)
+                if stype.Kind() == reflect.Ptr {
+                        stype = stype.Elem()
+                        val = val.Elem()
+                }
+                for i := 0; i < stype.NumField(); i++ {
+                        f := stype.Field(i)
+                        if _, skip := t.used[f.Name]; !skip {
+                                connect := "?"
+                                if addedQuestion {
+                                        connect = "&"
+                                }
+                                addedQuestion = true
+                                marshalled, err := json.Marshal(val.Field(i).Interface())
+                                if err != nil {
+                                        return "", err
+                                }
+                                ret = fmt.Sprintf("%s%s%s=%s", ret, connect, f.Name, string(marshalled))
+                        }
+                }
+        }
+
+        return ret, nil
+}
+
+func (c *client) DoWithRetries(serviceName, methodName string, in interface{}, out interface{}, upto uint) (err error) {
+        for i := uint(0); i < upto; i++ {
+                err = c.do(serviceName, methodName, in, out)
+                if err != nil {
+                        sleepIval := Backoff(i + 1)
+                        ctx := logging.SetField(c.ctx, "retriesLeft", upto-i)
+                        ctx = logging.SetField(ctx, "sleepFor", sleepIval)
+                        if err != nil {
+                                ctx = logging.SetField(ctx, "prevErr", err)
+                        }
+                        logging.Get(ctx).Warningf("retrying")
+                        select {
+                        case <-ctx.Done():
+                                return
+                        case <-time.After(sleepIval):
+                        }
+                } else {
+                        break
+                }
+        }
+        return
+}
+
+func mkHTTPDoer() (func(*http.Request) (*http.Response, error), func()) {

[Vadim Sh., 2015/06/03 18:13:39]

consider using https://github.com/luci/luci-go/tree/master/client/internal/lhttp

also make it possible to pass custom http.Client from outside. Your client
doesn't do authentication, for example.

+        // TODO(riannucci): Figure out why the default client isn't good enough. Was
+        // seeing random hangups when running against appspot.com that the default
+        // transport wasn't smart enough to deal with (e.g. would get Do calls which
+        // died with 'Connection Closed' errors.
+        transp := &http.Transport{
+                Dial: (&net.Dialer{
+                        Timeout:   30 * time.Second,
+                        KeepAlive: 30 * time.Second,
+                }).Dial,
+                TLSHandshakeTimeout: 10 * time.Second,
+                DisableKeepAlives:   true,
+        }
+        ret := &http.Client{Transport: transp}
+        return ret.Do, transp.CloseIdleConnections
+}
+
+func (c *client) do(serviceName, methodName string, in interface{}, out interface{}) error {
+        do, closer := c.mkHTTPDoer()
+        defer closer()
+
+        tentry, err := c.getMethodInfo(serviceName, methodName)
+        if err != nil {
+                return err
+        }
+
+        if reflect.TypeOf(in) != tentry.reqType {
+                return fmt.Errorf("do(%s) type mismatch for input. Got %T but expected %s",
+                        methodName, in, tentry.reqType)
+        }
+        // out must be a pointer to something.
+        if reflect.TypeOf(out) != tentry.rspType {
+                return fmt.Errorf("do(%s) type mismatch for output. Got %T but expected %s",
+                        methodName, out, tentry.rspType)
+        }
+
+        var inBuf io.ReadWriter
+        if tentry.mi.HTTPMethod != "GET" {
+                inBuf = &bytes.Buffer{}
+                enc := json.NewEncoder(inBuf)
+                err := enc.Encode(in)
+                if err != nil {
+                        return err
+                }
+        }
+
+        url, err := renderURL(c.url, tentry, in)
+        if err != nil {
+                return err
+        }
+
+        req, err := http.NewRequest(tentry.mi.HTTPMethod, url, inBuf)
+        if err != nil {
+                return err
+        }
+        if inBuf != nil {
+                req.Header.Add("content-type", "application/json")
+        }
+
+        rsp, err := do(req)
+        if err != nil {
+                return err
+        }
+        defer rsp.Body.Close()
+
+        data, err := ioutil.ReadAll(rsp.Body)
+        if err != nil {
+                return mark(fmt.Errorf("do(%s) failed (reading body): %s", methodName, err))
+        }
+
+        if rsp.StatusCode/100 == 2 {

[Vadim Sh., 2015/06/03 18:13:39]

wat? Is rsp.StatusCode >= 200 && rsp.StatusCode < 300 so hard to type?

+                if out != nil {
+                        err = json.Unmarshal(data, out)
+                }
+        } else {
+                msg := &map[string]map[string]interface{}{}
+                err = json.Unmarshal(data, msg)
+                if err != nil {
+                        return mark(fmt.Errorf("do(%s) failed (decoding json %q): %s", methodName, string(data), err))
+                }
+
+                return mark(fmt.Errorf("do(%s) failed: %s", methodName, (*msg)["error"]["message"]))
+        }
+        return nil
+}
+
+func (c *client) ForService(serviceName string) (ServiceClient, error) {
+        if c.serverDef.ServiceByName(serviceName) == nil {
+                return nil, mark(fmt.Errorf("getMethodInfo: no such service %s", serviceName))
+        }
+        return &serviceClient{c, serviceName}, nil
+}
+
+type serviceClient struct {
+        c       Client
+        service string
+}
+
+var _ ServiceClient = (*serviceClient)(nil)
+
+func (s *serviceClient) DoWithRetries(methodName string, in interface{}, out interface{}, upto uint) error {
+        return s.c.DoWithRetries(s.service, methodName, in, out, upto)
+}