Drover's back, baby!

go/src/infra/libs/git/get_object.go

Index: go/src/infra/libs/git/get_object.go
diff --git a/go/src/infra/libs/git/get_object.go b/go/src/infra/libs/git/get_object.go
new file mode 100644
index 0000000000000000000000000000000000000000..eb31b8950264c51ece3e9b1c355d1bae20e329a8
--- /dev/null
+++ b/go/src/infra/libs/git/get_object.go
@@ -0,0 +1,231 @@
+package git
+
+import "bufio"
+import "fmt"
+import "os/exec"
+import "strconv"
+import "strings"
+import "sync"
+import "sync/atomic"
+import "unsafe"
+
+import "infra/libs/infra_util"
+
+// GetObject will asynchronously fetch |objectish| from the Repo, and
+// return a channel for the result. If the object is missing, GetObject will
+// push nil to the channel.
+func (r *Repo) GetObject(objectish string) InternableObject {
+	r.ensureCatFileServer(&r.catFile, false)
+
+	rchan := make(chan *catFileReply, 1)
+	(*r.catFile) <- catFileRequest{
+		objectish: objectish,
+		reply:     rchan,
+	}
+
+	if rsp := <-rchan; rsp == nil {
+		return nil
+	} else {
+		switch rsp.typ {
+		case "blob":
+			return BlobFromRawWithID(rsp.id, rsp.data)
+		case "tree":
+			if t, err := TreeFromRawWithID(rsp.id, rsp.data); err != nil {
+				panic(err)
+			} else {
+				return t
+			}
+		case "commit":
+			if c, err := CommitFromRawWithID(rsp.id, rsp.data); err != nil {
+				panic(err)
+			} else {
+				return c
+			}
+		default:
+			panic(fmt.Errorf("unsupported object type: %s", rsp.typ))
+		}
+	}
+}
+
+func (r *Repo) GetObjectID(id ObjectID) InternableObject {
+	return r.GetObject(id.String())
+}
+
+// HasObject will return true iff the Repo contains the objectish
+func (r *Repo) HasObject(objectish string) bool {
+	r.ensureCatFileServer(&r.catFileCheck, true)
+	rchan := make(chan *catFileReply, 1)
+	(*r.catFileCheck) <- catFileRequest{
+		objectish: objectish,
+		reply:     rchan,
+	}
+	return (<-rchan) != nil
+}
+
+func (r *Repo) HasObjectID(id ObjectID) bool {
+	return r.HasObject(id.String())
+}
+
+type blobOpt bool
+
+const (
+	WithBlobs blobOpt = true
+	NoBlobs           = false
+)
+
+type fullTree bool
+
+const (
+	FullTree  fullTree = true

[M-A Ruel, 2014/10/18 00:47:05]

move constants and simple types at the top

[iannucci, 2014/10/20 21:11:57]

Done.

+	MissingOK          = false
+)
+
+// GetFullTree gets a recursively-enumerated Tree.
+//
+// blobOpt:
+//	 WithBlobs - Load blobs from Repo
+//	 NoBlobs - Blobs will be EmptyObject
+//
+// fullTree:
+//   FullTree - All entries in tree must load
+//   MissingOK - Missing entries will remain EmptyObject (or an !Complete() Tree)
+func (r *Repo) GetFullTree(treeish string, b blobOpt, f fullTree) *Tree {
+	base, ok := r.GetObject(treeish).(*Tree)
+	if !ok {
+		if f == FullTree {
+			panic(fmt.Errorf("could not load object %s", treeish))
+		} else {
+			return nil
+		}
+	}
+	grp := sync.WaitGroup{}
+	for p, c := range base.children {
+		p := p

[M-A Ruel, 2014/10/18 00:47:05]

Oops?

[iannucci, 2014/10/20 21:11:57]

Nope, to avoid aliasing p in the func()

[M-A Ruel, 2014/10/21 00:55:53]

It's because you need to call with p as a parameter;

go func(p foo, c bar) {
   ...
}(p, c)

I prefer this to explicit aliasing. Better is likely to create a named function
outside this scope. It's less risky.

+		c := c
+		grp.Add(1)
+		go func() {
+			defer grp.Done()
+			switch c.Mode.Type() {
+			case "tree":
+				subtree := r.GetFullTreeID(c.Object.ID(), b, f)
+				if subtree == nil {
+					if f == FullTree {
+						panic(fmt.Errorf("could not load tree %s", c.Object.ID()))
+					}
+				} else {
+					base.children[p] = &Child{subtree, c.Mode}
+				}
+			case "blob":
+				if b == WithBlobs {
+					rslt := r.GetObjectID(c.Object.ID())
+					if rslt == nil && f == FullTree {
+						panic(fmt.Errorf("could not load object %s", c.Object.ID()))
+					}
+					base.children[p] = &Child{rslt, c.Mode}
+				}
+			}
+		}()
+	}
+	grp.Wait()
+	return base
+}
+
+func (r *Repo) GetFullTreeID(tree ObjectID, b blobOpt, f fullTree) *Tree {
+	return r.GetFullTree(tree.String(), b, f)
+}
+
+/// Private
+
+type catFileReply struct {
+	id   ObjectID
+	typ  string
+	size int
+	data []byte
+}
+
+type catFileRequest struct {
+	objectish string
+	reply     chan<- *catFileReply
+}
+
+func (r *Repo) ensureCatFileServer(ch **chan<- catFileRequest, checkOnly bool) {

[M-A Ruel, 2014/10/18 00:47:05]

Double pointer is unnecessary and you shouldn't normally need unsafe at all.
Please refactor.

[iannucci, 2014/10/20 21:11:57]

Come again? How do I safely start the service lazily then?

+	if *ch == nil {
+		c := make(chan catFileRequest, 16)
+		swapped := atomic.CompareAndSwapPointer(
+			(*unsafe.Pointer)(unsafe.Pointer(ch)),
+			nil,
+			unsafe.Pointer(&c),
+		)
+		if swapped {
+			go r.catFileServer(c, checkOnly)
+		}
+	}
+}
+
+func (r *Repo) catFileServer(rchan chan catFileRequest, checkOnly bool) {
+	defer func() {
+		atomic.StorePointer((*unsafe.Pointer)(unsafe.Pointer(&r.catFile)), nil)
+		close(rchan)
+
+		if err := recover(); err != nil {
+			fmt.Println("recovering panick'd catFileServer", err)
+		}
+	}()
+
+	arg := "--batch"
+	if checkOnly {
+		arg = "--batch-check"
+	}
+	catFile := exec.Command("git", "cat-file", arg)
+	catFile.Dir = r.Path
+	in, err := catFile.StdinPipe()
+	if err != nil {
+		panic(err)

[M-A Ruel, 2014/10/18 00:47:05]

Send error back in channel

[iannucci, 2014/10/20 21:11:57]

Nowhere to send it to, this condition should never happen. There's no reasonable
recovery here.

+	}
+	defer in.Close()
+	outRaw, err := catFile.StdoutPipe()
+	if err != nil {
+		panic(err)
+	}
+	defer outRaw.Close()
+	out := bufio.NewReader(outRaw)
+
+	if err = catFile.Start(); err != nil {
+		panic(err)
+	}
+
+	nom := infra_util.Nom(out)
+	yoink := infra_util.Yoink(out)
+
+	for req := range rchan {
+		if strings.ContainsAny(req.objectish, "\n") {
+			panic("catFile request may not contain a newline")
+		}
+
+		in.Write([]byte(req.objectish + "\n"))
+		rsp := nom('\n')
+		if strings.HasSuffix(rsp, " missing") {
+			req.reply <- nil
+			continue
+		}
+
+		parts := strings.Split(rsp, " ")
+		objID, typ, sizeStr := parts[0], parts[1], parts[2]
+		size, err := strconv.ParseUint(sizeStr, 10, 64)
+		if err != nil {
+			panic(err)
+		}
+
+		data := []byte{}
+		if !checkOnly {
+			data = yoink(int(size))
+			out.ReadByte() // drop extra newline
+		}
+		req.reply <- &catFileReply{
+			id:   MakeObjectID(objID),
+			typ:  typ,
+			data: data,
+			size: int(size),
+		}
+	}
+}