Drover's back, baby!

go/src/infra/libs/git/get_object.go

+package git
+
+import "bufio"
+import "fmt"
+import "os/exec"
+import "strconv"
+import "strings"
+import "sync"
+import "sync/atomic"
+import "unsafe"
+
+import "infra/libs/infra_util"
+
+// GetObject will asynchronously fetch |objectish| from the Repo, and
+// return a channel for the result. If the object is missing, GetObject will
+// push nil to the channel.
+func (r *Repo) GetObject(objectish string) InternableObject {
+        r.ensureCatFileServer(&r.catFile, false)
+
+        rchan := make(chan *catFileReply, 1)
+        (*r.catFile) <- catFileRequest{
+                objectish: objectish,
+                reply:     rchan,
+        }
+
+        if rsp := <-rchan; rsp == nil {
+                return nil
+        } else {
+                switch rsp.typ {
+                case "blob":
+                        return BlobFromRawWithID(rsp.id, rsp.data)
+                case "tree":
+                        if t, err := TreeFromRawWithID(rsp.id, rsp.data); err != nil {
+                                panic(err)
+                        } else {
+                                return t
+                        }
+                case "commit":
+                        if c, err := CommitFromRawWithID(rsp.id, rsp.data); err != nil {
+                                panic(err)
+                        } else {
+                                return c
+                        }
+                default:
+                        panic(fmt.Errorf("unsupported object type: %s", rsp.typ))
+                }
+        }
+}
+
+func (r *Repo) GetObjectID(id ObjectID) InternableObject {
+        return r.GetObject(id.String())
+}
+
+// HasObject will return true iff the Repo contains the objectish
+func (r *Repo) HasObject(objectish string) bool {
+        r.ensureCatFileServer(&r.catFileCheck, true)
+        rchan := make(chan *catFileReply, 1)
+        (*r.catFileCheck) <- catFileRequest{
+                objectish: objectish,
+                reply:     rchan,
+        }
+        return (<-rchan) != nil
+}
+
+func (r *Repo) HasObjectID(id ObjectID) bool {
+        return r.HasObject(id.String())
+}
+
+type blobOpt bool
+
+const (
+        WithBlobs blobOpt = true
+        NoBlobs           = false
+)
+
+type fullTree bool
+
+const (
+        FullTree  fullTree = true

[M-A Ruel, 2014/10/18 00:47:05]

move constants and simple types at the top

[iannucci, 2014/10/20 21:11:57]

Done.

+        MissingOK          = false
+)
+
+// GetFullTree gets a recursively-enumerated Tree.
+//
+// blobOpt:
+//       WithBlobs - Load blobs from Repo
+//       NoBlobs - Blobs will be EmptyObject
+//
+// fullTree:
+//   FullTree - All entries in tree must load
+//   MissingOK - Missing entries will remain EmptyObject (or an !Complete() Tree)
+func (r *Repo) GetFullTree(treeish string, b blobOpt, f fullTree) *Tree {
+        base, ok := r.GetObject(treeish).(*Tree)
+        if !ok {
+                if f == FullTree {
+                        panic(fmt.Errorf("could not load object %s", treeish))
+                } else {
+                        return nil
+                }
+        }
+        grp := sync.WaitGroup{}
+        for p, c := range base.children {
+                p := p

[M-A Ruel, 2014/10/18 00:47:05]

Oops?

[iannucci, 2014/10/20 21:11:57]

Nope, to avoid aliasing p in the func()

[M-A Ruel, 2014/10/21 00:55:53]

It's because you need to call with p as a parameter;

go func(p foo, c bar) {
   ...
}(p, c)

I prefer this to explicit aliasing. Better is likely to create a named function
outside this scope. It's less risky.

+                c := c
+                grp.Add(1)
+                go func() {
+                        defer grp.Done()
+                        switch c.Mode.Type() {
+                        case "tree":
+                                subtree := r.GetFullTreeID(c.Object.ID(), b, f)
+                                if subtree == nil {
+                                        if f == FullTree {
+                                                panic(fmt.Errorf("could not load tree %s", c.Object.ID()))
+                                        }
+                                } else {
+                                        base.children[p] = &Child{subtree, c.Mode}
+                                }
+                        case "blob":
+                                if b == WithBlobs {
+                                        rslt := r.GetObjectID(c.Object.ID())
+                                        if rslt == nil && f == FullTree {
+                                                panic(fmt.Errorf("could not load object %s", c.Object.ID()))
+                                        }
+                                        base.children[p] = &Child{rslt, c.Mode}
+                                }
+                        }
+                }()
+        }
+        grp.Wait()
+        return base
+}
+
+func (r *Repo) GetFullTreeID(tree ObjectID, b blobOpt, f fullTree) *Tree {
+        return r.GetFullTree(tree.String(), b, f)
+}
+
+/// Private
+
+type catFileReply struct {
+        id   ObjectID
+        typ  string
+        size int
+        data []byte
+}
+
+type catFileRequest struct {
+        objectish string
+        reply     chan<- *catFileReply
+}
+
+func (r *Repo) ensureCatFileServer(ch **chan<- catFileRequest, checkOnly bool) {

[M-A Ruel, 2014/10/18 00:47:05]

Double pointer is unnecessary and you shouldn't normally need unsafe at all.
Please refactor.

[iannucci, 2014/10/20 21:11:57]

Come again? How do I safely start the service lazily then?

+        if *ch == nil {
+                c := make(chan catFileRequest, 16)
+                swapped := atomic.CompareAndSwapPointer(
+                        (*unsafe.Pointer)(unsafe.Pointer(ch)),
+                        nil,
+                        unsafe.Pointer(&c),
+                )
+                if swapped {
+                        go r.catFileServer(c, checkOnly)
+                }
+        }
+}
+
+func (r *Repo) catFileServer(rchan chan catFileRequest, checkOnly bool) {
+        defer func() {
+                atomic.StorePointer((*unsafe.Pointer)(unsafe.Pointer(&r.catFile)), nil)
+                close(rchan)
+
+                if err := recover(); err != nil {
+                        fmt.Println("recovering panick'd catFileServer", err)
+                }
+        }()
+
+        arg := "--batch"
+        if checkOnly {
+                arg = "--batch-check"
+        }
+        catFile := exec.Command("git", "cat-file", arg)
+        catFile.Dir = r.Path
+        in, err := catFile.StdinPipe()
+        if err != nil {
+                panic(err)

[M-A Ruel, 2014/10/18 00:47:05]

Send error back in channel

[iannucci, 2014/10/20 21:11:57]

Nowhere to send it to, this condition should never happen. There's no reasonable
recovery here.

+        }
+        defer in.Close()
+        outRaw, err := catFile.StdoutPipe()
+        if err != nil {
+                panic(err)
+        }
+        defer outRaw.Close()
+        out := bufio.NewReader(outRaw)
+
+        if err = catFile.Start(); err != nil {
+                panic(err)
+        }
+
+        nom := infra_util.Nom(out)
+        yoink := infra_util.Yoink(out)
+
+        for req := range rchan {
+                if strings.ContainsAny(req.objectish, "\n") {
+                        panic("catFile request may not contain a newline")
+                }
+
+                in.Write([]byte(req.objectish + "\n"))
+                rsp := nom('\n')
+                if strings.HasSuffix(rsp, " missing") {
+                        req.reply <- nil
+                        continue
+                }
+
+                parts := strings.Split(rsp, " ")
+                objID, typ, sizeStr := parts[0], parts[1], parts[2]
+                size, err := strconv.ParseUint(sizeStr, 10, 64)
+                if err != nil {
+                        panic(err)
+                }
+
+                data := []byte{}
+                if !checkOnly {
+                        data = yoink(int(size))
+                        out.ReadByte() // drop extra newline
+                }
+                req.reply <- &catFileReply{
+                        id:   MakeObjectID(objID),
+                        typ:  typ,
+                        data: data,
+                        size: int(size),
+                }
+        }
+}