luci-go: Tools for working with BSD style ar archives.

common/archive/ar/reader.go

Index: common/archive/ar/reader.go
diff --git a/common/archive/ar/reader.go b/common/archive/ar/reader.go
new file mode 100644
index 0000000000000000000000000000000000000000..c53fb6b2b3cb4975a9dac3d237432472052028d5
--- /dev/null
+++ b/common/archive/ar/reader.go
@@ -0,0 +1,268 @@
+// Copyright 2016 The LUCI Authors. All rights reserved.
+// Use of this source code is governed under the Apache License, Version 2.0
+// that can be found in the LICENSE file.
+
+// Read an ar file with BSD formatted file names.
+
+package ar
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"io"
+	"log"
+	"os"
+	"time"
+)
+
+type ArFileInfo interface {
+	os.FileInfo
+	UserId() int
+	GroupId() int
+}
+
+type arFileInfoData struct {
+	// os.FileInfo parts
+	name    string
+	size    int64
+	mode    uint32
+	modtime uint64
+	// Extra parts
+	uid int
+	gid int
+}
+
+// os.FileInfo interface
+func (fi *arFileInfoData) Name() string       { return fi.name }
+func (fi *arFileInfoData) Size() int64        { return fi.size }
+func (fi *arFileInfoData) Mode() os.FileMode  { return os.FileMode(fi.mode) }
+func (fi *arFileInfoData) ModTime() time.Time { return time.Unix(int64(fi.modtime), 0) }
+func (fi *arFileInfoData) IsDir() bool        { return fi.Mode().IsDir() }
+func (fi *arFileInfoData) Sys() interface{}   { return fi }
+
+// Extra
+func (fi *arFileInfoData) UserId() int  { return fi.uid }
+func (fi *arFileInfoData) GroupId() int { return fi.gid }
+
+type readerStage uint
+
+const (
+	READ_HEADER readerStage = iota

[M-A Ruel, 2016/06/10 17:51:10]

don't export the const either

[mithro, 2016/06/14 10:57:51]

Done and now obsolete.

+	READ_BODY
+	READ_CLOSED

[M-A Ruel, 2016/06/10 17:51:11]

Not needed, can detect with r == nil

[mithro, 2016/06/14 10:57:51]

Done and now obsolete.

+)
+
+type Reader struct {
+	stage         readerStage
+	r             io.Reader
+	bytesrequired int64

[M-A Ruel, 2016/06/10 17:51:10]

streamSizeRemaining would probably be more understandable

[mithro, 2016/06/14 10:57:52]

Done and now obsolete.

+	needspadding  bool
+}
+
+func NewReader(r io.Reader) (*Reader, error) {
+	reader := Reader{r: r, stage: READ_HEADER}
+	err := reader.checkBytes("header", []byte("!<arch>\n"))
+	if err != nil {

[M-A Ruel, 2016/06/10 17:51:10]

merge with line 65

[mithro, 2016/06/14 10:57:51]

Done.

+		return nil, err
+	}
+	return &reader, nil
+}
+
+func (ar *Reader) checkBytes(name string, str []byte) error {
+	buffer := make([]byte, len(str))
+
+	_, err := io.ReadFull(ar.r, buffer)

[M-A Ruel, 2016/06/10 17:51:10]

merge with line 76

[mithro, 2016/06/14 10:57:51]

Done.

+	if err != nil {
+		return fmt.Errorf("%s: error in reading bytes (%v)", name, err)
+	}
+
+	if bytes.Equal(str, buffer) {
+		return nil
+	} else {

[M-A Ruel, 2016/06/10 17:51:10]

no need for else, I'd recommend to to reverse the condition for consistency with
the line above

[mithro, 2016/06/14 10:57:51]

Done.

+		return fmt.Errorf("%s: error in bytes (wanted: %v got: %v)", name, buffer, str)
+	}
+}
+
+func (ar *Reader) Close() error {
+	switch ar.stage {
+	case READ_HEADER:
+		// Good
+	case READ_BODY:
+		return errors.New("usage error, reading a file.")

[M-A Ruel, 2016/06/10 17:51:10]

Why is it a problem?

[mithro, 2016/06/14 10:57:51]

I guess in the reading case it isn't.

+	case READ_CLOSED:
+		return errors.New("usage error, archive already closed.")

[M-A Ruel, 2016/06/10 17:51:11]

I don't think it's worth reporting to the user.

[mithro, 2016/06/14 10:57:52]

This is how other things in Go seem to behave. They return an error when closing
something which is already closed.

+	default:
+		log.Fatalf("unknown reader mode: %d", ar.stage)
+	}
+	ar.stage = READ_CLOSED

[M-A Ruel, 2016/06/10 17:51:10]

When ar.r is nil, it's closed.
So stage becomes a bool

[mithro, 2016/06/14 10:57:52]

Done and now obsolete.

+	ar.r = nil
+	return nil
+}
+
+func (ar *Reader) completeReadBytes(numbytes int64) error {

[M-A Ruel, 2016/06/10 17:51:10]

So this function is actually readPadding(), right?

[mithro, 2016/06/14 10:57:52]

Now obsolete.

+	if numbytes > ar.bytesrequired {
+		return fmt.Errorf("to much data read, needed %d, got %d", ar.bytesrequired, numbytes)
+	}
+
+	ar.bytesrequired -= numbytes
+	if ar.bytesrequired != 0 {

[M-A Ruel, 2016/06/10 17:51:10]

when can this happen?

[mithro, 2016/06/14 10:57:52]

When a partial read is used.

+		return nil
+	}
+
+	// Padding to 16bit boundary
+	if ar.needspadding {
+		err := ar.checkBytes("padding", []byte("\n"))

[M-A Ruel, 2016/06/10 17:51:11]

merge with next line
[]byte{'\n'}

[mithro, 2016/06/14 10:57:52]

Done.

+		if err != nil {
+			return err
+		}
+		ar.needspadding = false
+	}
+	ar.stage = READ_HEADER
+	return nil
+}
+
+// Check we have finished writing bytes

[M-A Ruel, 2016/06/10 17:51:10]

writing?

[mithro, 2016/06/14 10:57:52]

Done.

+func (ar *Reader) checkFinished() error {
+	if ar.bytesrequired != 0 {
+		return fmt.Errorf("didn't read enough %d bytes still needed", ar.bytesrequired)
+	}
+	return nil
+}
+
+func (ar *Reader) readPartial(name string, data []byte) error {
+	// Check you can read bytes from the ar at this moment.
+	switch ar.stage {
+	case READ_HEADER:
+		return errors.New("usage error, need to read header first")

[M-A Ruel, 2016/06/10 17:51:10]

Keep in mind it's an internal function, it should just work.

[mithro, 2016/06/14 10:57:51]

Now obsolete.

+	case READ_BODY:
+		// Good
+	case READ_CLOSED:
+		return errors.New("usage error, archive closed")
+	default:
+		log.Fatalf("unknown reader mode: %d", ar.stage)
+	}
+
+	datalen := int64(len(data))

[M-A Ruel, 2016/06/10 17:51:10]

inline datalen

[mithro, 2016/06/14 10:57:51]

Done.

+	if datalen > ar.bytesrequired {
+		return fmt.Errorf("to much data, wanted %d, but had %d", ar.bytesrequired, datalen)
+	}
+
+	count, err := io.ReadFull(ar.r, data)
+	if err != nil {
+		return err
+	}
+	ar.completeReadBytes(int64(count))
+	return nil
+}
+
+func (ar *Reader) readHeaderBytes(name string, bytes int, formatstr string) (int64, error) {

[M-A Ruel, 2016/06/10 17:51:10]

readHeaderInt64() ?

[mithro, 2016/06/14 10:57:51]

Done.

+	data := make([]byte, bytes)
+	_, err := io.ReadFull(ar.r, data)
+	if err != nil {
+		return -1, err
+	}
+
+	var output int64 = 0

[M-A Ruel, 2016/06/10 17:51:10]

=0 is not needed

[mithro, 2016/06/14 10:57:52]

Done.

+	_, err = fmt.Sscanf(string(data), formatstr, &output)

[M-A Ruel, 2016/06/10 17:51:10]

merge with next line

[mithro, 2016/06/14 10:57:51]

Done.

+	if err != nil {
+		return -1, err
+	}
+
+	if output <= 0 {
+		return -1, fmt.Errorf("%s: bad value %d", name, output)
+	}
+	return output, nil
+}
+
+func (ar *Reader) readHeader() (*arFileInfoData, error) {

[M-A Ruel, 2016/06/10 17:51:10]

Inline into Next()

[mithro, 2016/06/14 10:57:51]

Done.

+	switch ar.stage {
+	case READ_HEADER:
+		// Good
+	case READ_BODY:
+		return nil, errors.New("usage error, already writing a file")
+	case READ_CLOSED:
+		return nil, errors.New("usage error, archive closed")
+	default:
+		log.Fatalf("unknown reader mode: %d", ar.stage)
+	}
+
+	var fi arFileInfoData
+
+	// File name length prefixed with '#1/' (BSD variant), 16 bytes
+	namelen, err := ar.readHeaderBytes("filename length", 16, "#1/%13d")
+	if err != nil {
+		return nil, err
+	}
+
+	// Modtime, 12 bytes
+	modtime, err := ar.readHeaderBytes("modtime", 12, "%12d")
+	if err != nil {
+		return nil, err
+	}
+	fi.modtime = uint64(modtime)
+
+	// Owner ID, 6 bytes
+	ownerid, err := ar.readHeaderBytes("ownerid", 6, "%6d")
+	if err != nil {
+		return nil, err
+	}
+	fi.uid = int(ownerid)
+
+	// Group ID, 6 bytes
+	groupid, err := ar.readHeaderBytes("groupid", 6, "%6d")
+	if err != nil {
+		return nil, err
+	}
+	fi.gid = int(groupid)
+
+	// File mode, 8 bytes
+	filemod, err := ar.readHeaderBytes("filemod", 8, "%8o")
+	if err != nil {
+		return nil, err
+	}
+	fi.mode = uint32(filemod)
+
+	// File size, 10 bytes
+	size, err := ar.readHeaderBytes("datasize", 10, "%10d")
+	if err != nil {
+		return nil, err
+	}
+	fi.size = size - namelen
+
+	// File magic, 2 bytes
+	err = ar.checkBytes("filemagic", []byte("\x60\n"))

[M-A Ruel, 2016/06/10 17:51:11]

create a byte array directly
merge with next line

[mithro, 2016/06/14 10:57:51]

Done.

+	if err != nil {
+		return nil, err
+	}
+
+	ar.stage = READ_BODY
+	ar.bytesrequired = size
+	ar.needspadding = (ar.bytesrequired%2 != 0)
+
+	// Filename - BSD variant
+	filename := make([]byte, namelen)
+	err = ar.readPartial("filename", filename)

[M-A Ruel, 2016/06/10 17:51:10]

merge

[mithro, 2016/06/14 10:57:52]

Done.

+	if err != nil {
+		return nil, err
+	}
+	fi.name = string(filename)

[M-A Ruel, 2016/06/10 17:51:10]

encoding is assumed to be utf8?

[mithro, 2016/06/14 10:57:52]

The "spec" seems to indicate it should be ASCII but everything seems to work
fine with utf-8.

+
+	return &fi, nil
+}
+
+func (ar *Reader) Read(b []byte) (n int, err error) {
+	err = ar.readPartial("data", b)

[M-A Ruel, 2016/06/10 17:51:11]

I'd prefer to not use named return values
merge

[mithro, 2016/06/14 10:57:52]

I actually didn't even know that was possible! Fixed.

+	if err != nil {
+		return -1, err
+	}
+	err = ar.checkFinished()
+	if err != nil {
+		return -1, err
+	}
+	return len(b), nil
+}
+
+func (ar *Reader) Next() (ArFileInfo, error) {
+	return ar.readHeader()
+}