WIP: Archive command which is much faster

common/archive/ar/reader.go

Index: common/archive/ar/reader.go
diff --git a/common/archive/ar/reader.go b/common/archive/ar/reader.go
new file mode 100644
index 0000000000000000000000000000000000000000..c884e1d7a9958cbbbef1c4a2ab01bfd32d4ab96a
--- /dev/null
+++ b/common/archive/ar/reader.go
@@ -0,0 +1,274 @@
+// Copyright 2016 The LUCI Authors. All rights reserved.
+// Use of this source code is governed under the Apache License, Version 2.0
+// that can be found in the LICENSE file.
+
+// Read an ar file with BSD formatted file names.
+
+package ar
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"time"
+)
+
+type arFileInfo struct {
+	// os.FileInfo parts
+	name    string
+	size    int64
+	mode    uint32
+	modtime uint64
+	// Extra parts
+	uid int
+	gid int
+}
+
+// os.FileInfo interface
+func (fi *arFileInfo) Name() string       { return fi.name }
+func (fi *arFileInfo) Size() int64        { return fi.size }
+func (fi *arFileInfo) Mode() os.FileMode  { return os.FileMode(fi.mode) }
+func (fi *arFileInfo) ModTime() time.Time { return time.Unix(int64(fi.modtime), 0) }
+func (fi *arFileInfo) IsDir() bool        { return fi.Mode().IsDir() }
+func (fi *arFileInfo) Sys() interface{}   { return fi }
+
+// Extra
+func (fi *arFileInfo) UserId() int  { return fi.uid }
+func (fi *arFileInfo) GroupId() int { return fi.gid }
+
+var (
+	ErrHeader = errors.New("archive/ar: invalid ar header")
+)
+
+type ReaderStage uint
+
+const (
+	READ_HEADER ReaderStage = iota
+	READ_BODY
+	READ_CLOSED
+)
+
+type Reader struct {
+	stage         ReaderStage
+	r             io.Reader
+	bytesrequired int64
+	needspadding  bool
+}
+
+func NewReader(r io.Reader) (*Reader, error) {
+	reader := Reader{r: r, bytesrequired: 0, needspadding: false}
+	err := reader.checkBytes("header", []byte("!<arch>\n"))
+	if err != nil {
+		return nil, err
+	} else {
+		return &reader, nil
+	}
+}
+
+func (ar *Reader) checkBytes(name string, str []byte) error {
+	buffer := make([]byte, len(str))
+
+	count, err := io.ReadFull(ar.r, buffer)
+	if err != nil {
+		return err
+	}
+
+	if count != len(buffer) {
+		return errors.New(fmt.Sprintf("%s: Not enough data read (only %d, needed %d)", name, count, len(buffer)))
+	}
+
+	if bytes.Equal(str, buffer) {
+		return nil
+	} else {
+		return errors.New(fmt.Sprintf("%s: error in bytes (wanted: %v got: %v)", name, buffer, str))
+	}
+}
+
+func (ar *Reader) Close() error {
+	switch ar.stage {
+	case READ_HEADER:
+		// Good
+	case READ_BODY:
+		return errors.New("Usage error, reading a file.")
+	case READ_CLOSED:
+		return errors.New("Usage error, archive already closed.")
+	default:
+		panic(fmt.Sprintf("Unknown reader mode: %d", ar.stage))
+	}
+	//ar.r.Close()
+	ar.stage = READ_CLOSED
+	return nil
+}
+
+func (ar *Reader) readBytes(numbytes int64) error {
+	if numbytes > ar.bytesrequired {
+		return errors.New(fmt.Sprintf("To much data read! Needed %d, got %d", ar.bytesrequired, numbytes))
+	}
+
+	ar.bytesrequired -= numbytes
+	if ar.bytesrequired != 0 {
+		return nil
+	}
+
+	// Padding to 16bit boundary
+	if ar.needspadding {
+		err := ar.checkBytes("padding", []byte("\n"))
+		if err != nil {
+			return err
+		}
+		ar.needspadding = false
+	}
+	ar.stage = READ_HEADER
+	return nil
+}
+
+// Check you can write bytes to the ar at this moment.
+func (ar *Reader) checkRead() error {
+	switch ar.stage {
+	case READ_HEADER:
+		return errors.New("Usage error, need to read header first.")
+		// Good
+	case READ_BODY:
+		return nil
+	case READ_CLOSED:
+		return errors.New("Usage error, archive closed.")
+	default:
+		panic(fmt.Sprintf("Unknown reader mode: %d", ar.stage))
+	}
+}
+
+// Check we have finished writing bytes
+func (ar *Reader) checkFinished() error {
+	if ar.bytesrequired != 0 {
+		return errors.New(fmt.Sprintf("Didn't read enough bytes %d still needed!", ar.bytesrequired))
+	}
+	return nil
+}
+
+func (ar *Reader) readPartial(name string, data []byte) error {
+	err := ar.checkRead()
+	if err != nil {
+		return err
+	}
+
+	datalen := int64(len(data))
+	if datalen > ar.bytesrequired {
+		return errors.New(fmt.Sprintf("To much data! Wanted %d, but had %d", ar.bytesrequired, datalen))
+	}
+
+	count, err := io.ReadFull(ar.r, data)
+	ar.readBytes(int64(count))
+	return nil
+}
+
+func (ar *Reader) readHeaderBytes(name string, bytes int, formatstr string) (int64, error) {
+	data := make([]byte, bytes)
+	_, err := io.ReadFull(ar.r, data)
+	if err != nil {
+		return -1, err
+	}
+
+	var output int64 = 0
+	_, err = fmt.Sscanf(string(data), formatstr, &output)
+	if err != nil {
+		return -1, err
+	}
+
+	if output <= 0 {
+		return -1, errors.New(fmt.Sprintf("%s: bad value %d", name, output))
+	}
+	return output, nil
+}
+
+func (ar *Reader) readHeader() (*arFileInfo, error) {
+	switch ar.stage {
+	case READ_HEADER:
+		// Good
+	case READ_BODY:
+		return nil, errors.New("Usage error, already writing a file.")
+	case READ_CLOSED:
+		return nil, errors.New("Usage error, archive closed.")
+	default:
+		panic(fmt.Sprintf("Unknown writer mode: %d", ar.stage))
+	}
+
+	var fi arFileInfo
+
+	// File name length prefixed with '#1/' (BSD variant), 16 bytes
+	namelen, err := ar.readHeaderBytes("filename length", 16, "#1/%13d")
+	if err != nil {
+		return nil, err
+	}
+
+	// Modtime, 12 bytes
+	modtime, err := ar.readHeaderBytes("modtime", 12, "%12d")
+	if err != nil {
+		return nil, err
+	}
+	fi.modtime = uint64(modtime)
+
+	// Owner ID, 6 bytes
+	ownerid, err := ar.readHeaderBytes("ownerid", 6, "%6d")
+	if err != nil {
+		return nil, err
+	}
+	fi.uid = int(ownerid)
+
+	// Group ID, 6 bytes
+	groupid, err := ar.readHeaderBytes("groupid", 6, "%6d")
+	if err != nil {
+		return nil, err
+	}
+	fi.gid = int(groupid)
+
+	// File mode, 8 bytes
+	filemod, err := ar.readHeaderBytes("groupid", 8, "%8o")
+	if err != nil {
+		return nil, err
+	}
+	fi.mode = uint32(filemod)
+
+	// File size, 10 bytes
+	size, err := ar.readHeaderBytes("datasize", 10, "%10d")
+	if err != nil {
+		return nil, err
+	}
+	fi.size = size - namelen
+
+	ar.stage = READ_BODY
+	ar.bytesrequired = size
+	ar.needspadding = (ar.bytesrequired%2 != 0)
+
+	// File magic, 2 bytes
+	err = ar.checkBytes("filemagic", []byte("\x60\n"))
+	if err != nil {
+		return nil, err
+	}
+
+	// Filename - BSD variant
+	filename := make([]byte, namelen)
+	err = ar.readPartial("filename", filename)
+	if err != nil {
+		return nil, err
+	}
+	fi.name = string(filename)
+
+	return &fi, nil
+}
+
+func (ar *Reader) Read(b []byte) (n int, err error) {
+	err = ar.readPartial("data", b)
+	if err != nil {
+		return -1, err
+	}
+	err = ar.checkFinished()
+	if err != nil {
+		return -1, err
+	}
+	return len(b), nil
+}
+func (ar *Reader) Next() (*arFileInfo, error) {
+	return ar.readHeader()
+}