[milo] initial call to get git history

milo/git/history.go

Index: milo/git/history.go
diff --git a/milo/git/history.go b/milo/git/history.go
new file mode 100644
index 0000000000000000000000000000000000000000..fb32ed132c57433ca9e12fd7d0122c1911695607
--- /dev/null
+++ b/milo/git/history.go
@@ -0,0 +1,167 @@
+// Copyright 2017 The LUCI Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package git
+
+import (
+	"bytes"
+	"compress/gzip"
+	"encoding/hex"
+	"fmt"
+	"io/ioutil"
+	"regexp"
+
+	"golang.org/x/net/context"
+
+	"github.com/golang/protobuf/proto"
+
+	"github.com/luci/gae/service/memcache"
+
+	"github.com/luci/luci-go/common/api/gitiles"
+	"github.com/luci/luci-go/common/errors"
+	"github.com/luci/luci-go/common/logging"
+	"github.com/luci/luci-go/common/proto/google"
+
+	milo "github.com/luci/luci-go/milo/api/proto"
+)
+
+var gitHash = regexp.MustCompile("[0-9a-fA-F]{40}")
+
+// Resolve resolves a commitish to a git commit hash.
+//
+// This operation will assumed to be either fully local (in the case that
+// commitish is already a git-hash-looking-thing), or local to the datastore (in
+// the case that commitish is a fully-qualified-ref, ref tables are populated by
+// a backend cron).
+//
+// If commitish is some other pattern (e.g. "HEAD~"), this will return the
+// commitish as-is.
+//
+// `resolved` will be true if `commit` is, in fact, a git hash; if it's false,
+// then higher layers should be careful about using it as part of a cache key.
+func Resolve(c context.Context, url, commitish string) (commit string, resolved bool, err error) {
+	if gitHash.MatchString(commitish) {
+		return commitish, true, nil
+	}
+
+	// TODO(iannucci): actually do lookup and cache it. Maybe have a backend cron
+	// which refreshes the entire refs space?
+	return commitish, false, nil
+}
+
+func protoCache(c context.Context, enabled bool, key string, out proto.Message, get func() error) error {

[Ryan Tseng, 2017/07/17 20:25:51]

comments plz

Also this is fairly specific and internal function for this module, can we make
it return (*milo.ConsoleGitInfo, err) instead?  It makes the flow easier to
read.

[iannucci, 2017/07/17 21:54:39]

Done.
No, I plan to use this for other proto caches as well.

+	if !enabled {
+		return get()
+	}
+
+	cacheEntry := memcache.NewItem(c, key+"|gz")
+	// try reading from cache
+	ok := func() bool {
+		switch err := memcache.Get(c, cacheEntry); err {
+		case nil:
+			r, err := gzip.NewReader(bytes.NewReader(cacheEntry.Value()))
+			if err != nil {
+				logging.WithError(err).Warningf(c, "making ungzip reader for memcache entry")
+				return false
+			}
+
+			data, err := ioutil.ReadAll(r)
+			if err != nil {
+				logging.WithError(err).Warningf(c, "ungziping memcache entry")
+				return false
+			}
+
+			if err := proto.Unmarshal(data, out); err != nil {
+				logging.WithError(err).Warningf(c, "unmarshalling cache entry")
+				return false
+			}
+
+			return true
+		case memcache.ErrCacheMiss:
+		default:
+			logging.WithError(err).Warningf(c, "memcache lookup")
+		}
+		return false
+	}()
+	if ok {
+		return nil
+	}
+
+	if err := get(); err != nil {
+		return err
+	}
+
+	data, err := proto.Marshal(out)
+	if err != nil {
+		logging.WithError(err).Warningf(c, "marshaling proto")
+		return nil
+	}
+
+	var buf bytes.Buffer
+	wr := gzip.NewWriter(&buf)
+	_, _ = wr.Write(data) // err is buffered on the writer till Close

[Ryan Tseng, 2017/07/17 20:25:51]

might as well skip assignment

[iannucci, 2017/07/17 21:54:39]

Done.

+	if err := wr.Close(); err != nil {
+		logging.WithError(err).Warningf(c, "gziping proto")

[Ryan Tseng, 2017/07/17 20:25:51]

nit: gzipping

[iannucci, 2017/07/17 21:54:39]

Done.

+		return nil
+	}
+
+	cacheEntry.SetValue(buf.Bytes())
+	if err := memcache.Set(c, cacheEntry); err != nil {
+		logging.WithError(err).Warningf(c, "memcache set")
+	}
+
+	return nil
+}
+
+// GetHistory makes a (cached) call to gitiles to obtain the ConsoleGitInfo for
+// the given url, commitish and limit.
+func GetHistory(c context.Context, url, commitish string, limit int) (*milo.ConsoleGitInfo, error) {
+	commitish, useCache, err := Resolve(c, url, commitish)
+	if err != nil {
+		return nil, errors.Annotate(err, "resolving %q", commitish).Err()
+	}
+
+	ret := &milo.ConsoleGitInfo{}
+	cacheKey := fmt.Sprintf("GetHistory|%s|%s|%d", url, commitish, limit)
+	err = protoCache(c, useCache, cacheKey, ret, func() error {
+		rawEntries, err := gitiles.Log(c, url, commitish, limit)
+		if err != nil {
+			return errors.Annotate(err, "GetHistory").Err()
+		}
+
+		ret.Commits = make([]*milo.ConsoleGitInfo_Commit, len(rawEntries))
+
+		for i, e := range rawEntries {
+			commit := &milo.ConsoleGitInfo_Commit{}
+			if commit.Id, err = hex.DecodeString(e.Commit); err != nil {
+				return errors.Annotate(err, "commit is not hex (%q)", e.Commit).Err()
+			}
+
+			commit.AuthorName = e.Author.Name
+			commit.AuthorEmail = e.Author.Email
+
+			ts, err := e.Committer.GetTime()
+			if err != nil {
+				return errors.Annotate(err, "commit time unparsible (%q)", e.Committer.Time).Err()
+			}
+
+			commit.CommitTime = google.NewTimestamp(ts)
+			commit.Msg = e.Message
+
+			ret.Commits[i] = commit
+		}
+		return nil
+	})
+	return ret, err
+}