luci-go: Tools for working with BSD style ar archives.

common/archive/ar/reader.go

+// Copyright 2016 The LUCI Authors. All rights reserved.
+// Use of this source code is governed under the Apache License, Version 2.0
+// that can be found in the LICENSE file.
+
+// Read an ar file with BSD formatted file names.
+
+package ar
+
+import (
+        "bytes"
+        "errors"
+        "fmt"
+        "io"
+        "log"
+        "os"
+        "time"
+)
+
+type ArFileInfo interface {
+        os.FileInfo
+        UserId() int
+        GroupId() int
+}
+
+type arFileInfoData struct {
+        // os.FileInfo parts
+        name    string
+        size    int64
+        mode    uint32
+        modtime uint64
+        // Extra parts
+        uid int
+        gid int
+}
+
+// os.FileInfo interface
+func (fi *arFileInfoData) Name() string       { return fi.name }
+func (fi *arFileInfoData) Size() int64        { return fi.size }
+func (fi *arFileInfoData) Mode() os.FileMode  { return os.FileMode(fi.mode) }
+func (fi *arFileInfoData) ModTime() time.Time { return time.Unix(int64(fi.modtime), 0) }
+func (fi *arFileInfoData) IsDir() bool        { return fi.Mode().IsDir() }
+func (fi *arFileInfoData) Sys() interface{}   { return fi }
+
+// Extra
+func (fi *arFileInfoData) UserId() int  { return fi.uid }
+func (fi *arFileInfoData) GroupId() int { return fi.gid }
+
+type readerStage uint
+
+const (
+        READ_HEADER readerStage = iota

[M-A Ruel, 2016/06/10 17:51:10]

don't export the const either

[mithro, 2016/06/14 10:57:51]

Done and now obsolete.

+        READ_BODY
+        READ_CLOSED

[M-A Ruel, 2016/06/10 17:51:11]

Not needed, can detect with r == nil

[mithro, 2016/06/14 10:57:51]

Done and now obsolete.

+)
+
+type Reader struct {
+        stage         readerStage
+        r             io.Reader
+        bytesrequired int64

[M-A Ruel, 2016/06/10 17:51:10]

streamSizeRemaining would probably be more understandable

[mithro, 2016/06/14 10:57:52]

Done and now obsolete.

+        needspadding  bool
+}
+
+func NewReader(r io.Reader) (*Reader, error) {
+        reader := Reader{r: r, stage: READ_HEADER}
+        err := reader.checkBytes("header", []byte("!<arch>\n"))
+        if err != nil {

[M-A Ruel, 2016/06/10 17:51:10]

merge with line 65

[mithro, 2016/06/14 10:57:51]

Done.

+                return nil, err
+        }
+        return &reader, nil
+}
+
+func (ar *Reader) checkBytes(name string, str []byte) error {
+        buffer := make([]byte, len(str))
+
+        _, err := io.ReadFull(ar.r, buffer)

[M-A Ruel, 2016/06/10 17:51:10]

merge with line 76

[mithro, 2016/06/14 10:57:51]

Done.

+        if err != nil {
+                return fmt.Errorf("%s: error in reading bytes (%v)", name, err)
+        }
+
+        if bytes.Equal(str, buffer) {
+                return nil
+        } else {

[M-A Ruel, 2016/06/10 17:51:10]

no need for else, I'd recommend to to reverse the condition for consistency with
the line above

[mithro, 2016/06/14 10:57:51]

Done.

+                return fmt.Errorf("%s: error in bytes (wanted: %v got: %v)", name, buffer, str)
+        }
+}
+
+func (ar *Reader) Close() error {
+        switch ar.stage {
+        case READ_HEADER:
+                // Good
+        case READ_BODY:
+                return errors.New("usage error, reading a file.")

[M-A Ruel, 2016/06/10 17:51:10]

Why is it a problem?

[mithro, 2016/06/14 10:57:51]

I guess in the reading case it isn't.

+        case READ_CLOSED:
+                return errors.New("usage error, archive already closed.")

[M-A Ruel, 2016/06/10 17:51:11]

I don't think it's worth reporting to the user.

[mithro, 2016/06/14 10:57:52]

This is how other things in Go seem to behave. They return an error when closing
something which is already closed.

+        default:
+                log.Fatalf("unknown reader mode: %d", ar.stage)
+        }
+        ar.stage = READ_CLOSED

[M-A Ruel, 2016/06/10 17:51:10]

When ar.r is nil, it's closed.
So stage becomes a bool

[mithro, 2016/06/14 10:57:52]

Done and now obsolete.

+        ar.r = nil
+        return nil
+}
+
+func (ar *Reader) completeReadBytes(numbytes int64) error {

[M-A Ruel, 2016/06/10 17:51:10]

So this function is actually readPadding(), right?

[mithro, 2016/06/14 10:57:52]

Now obsolete.

+        if numbytes > ar.bytesrequired {
+                return fmt.Errorf("to much data read, needed %d, got %d", ar.bytesrequired, numbytes)
+        }
+
+        ar.bytesrequired -= numbytes
+        if ar.bytesrequired != 0 {

[M-A Ruel, 2016/06/10 17:51:10]

when can this happen?

[mithro, 2016/06/14 10:57:52]

When a partial read is used.

+                return nil
+        }
+
+        // Padding to 16bit boundary
+        if ar.needspadding {
+                err := ar.checkBytes("padding", []byte("\n"))

[M-A Ruel, 2016/06/10 17:51:11]

merge with next line
[]byte{'\n'}

[mithro, 2016/06/14 10:57:52]

Done.

+                if err != nil {
+                        return err
+                }
+                ar.needspadding = false
+        }
+        ar.stage = READ_HEADER
+        return nil
+}
+
+// Check we have finished writing bytes

[M-A Ruel, 2016/06/10 17:51:10]

writing?

[mithro, 2016/06/14 10:57:52]

Done.

+func (ar *Reader) checkFinished() error {
+        if ar.bytesrequired != 0 {
+                return fmt.Errorf("didn't read enough %d bytes still needed", ar.bytesrequired)
+        }
+        return nil
+}
+
+func (ar *Reader) readPartial(name string, data []byte) error {
+        // Check you can read bytes from the ar at this moment.
+        switch ar.stage {
+        case READ_HEADER:
+                return errors.New("usage error, need to read header first")

[M-A Ruel, 2016/06/10 17:51:10]

Keep in mind it's an internal function, it should just work.

[mithro, 2016/06/14 10:57:51]

Now obsolete.

+        case READ_BODY:
+                // Good
+        case READ_CLOSED:
+                return errors.New("usage error, archive closed")
+        default:
+                log.Fatalf("unknown reader mode: %d", ar.stage)
+        }
+
+        datalen := int64(len(data))

[M-A Ruel, 2016/06/10 17:51:10]

inline datalen

[mithro, 2016/06/14 10:57:51]

Done.

+        if datalen > ar.bytesrequired {
+                return fmt.Errorf("to much data, wanted %d, but had %d", ar.bytesrequired, datalen)
+        }
+
+        count, err := io.ReadFull(ar.r, data)
+        if err != nil {
+                return err
+        }
+        ar.completeReadBytes(int64(count))
+        return nil
+}
+
+func (ar *Reader) readHeaderBytes(name string, bytes int, formatstr string) (int64, error) {

[M-A Ruel, 2016/06/10 17:51:10]

readHeaderInt64() ?

[mithro, 2016/06/14 10:57:51]

Done.

+        data := make([]byte, bytes)
+        _, err := io.ReadFull(ar.r, data)
+        if err != nil {
+                return -1, err
+        }
+
+        var output int64 = 0

[M-A Ruel, 2016/06/10 17:51:10]

=0 is not needed

[mithro, 2016/06/14 10:57:52]

Done.

+        _, err = fmt.Sscanf(string(data), formatstr, &output)

[M-A Ruel, 2016/06/10 17:51:10]

merge with next line

[mithro, 2016/06/14 10:57:51]

Done.

+        if err != nil {
+                return -1, err
+        }
+
+        if output <= 0 {
+                return -1, fmt.Errorf("%s: bad value %d", name, output)
+        }
+        return output, nil
+}
+
+func (ar *Reader) readHeader() (*arFileInfoData, error) {

[M-A Ruel, 2016/06/10 17:51:10]

Inline into Next()

[mithro, 2016/06/14 10:57:51]

Done.

+        switch ar.stage {
+        case READ_HEADER:
+                // Good
+        case READ_BODY:
+                return nil, errors.New("usage error, already writing a file")
+        case READ_CLOSED:
+                return nil, errors.New("usage error, archive closed")
+        default:
+                log.Fatalf("unknown reader mode: %d", ar.stage)
+        }
+
+        var fi arFileInfoData
+
+        // File name length prefixed with '#1/' (BSD variant), 16 bytes
+        namelen, err := ar.readHeaderBytes("filename length", 16, "#1/%13d")
+        if err != nil {
+                return nil, err
+        }
+
+        // Modtime, 12 bytes
+        modtime, err := ar.readHeaderBytes("modtime", 12, "%12d")
+        if err != nil {
+                return nil, err
+        }
+        fi.modtime = uint64(modtime)
+
+        // Owner ID, 6 bytes
+        ownerid, err := ar.readHeaderBytes("ownerid", 6, "%6d")
+        if err != nil {
+                return nil, err
+        }
+        fi.uid = int(ownerid)
+
+        // Group ID, 6 bytes
+        groupid, err := ar.readHeaderBytes("groupid", 6, "%6d")
+        if err != nil {
+                return nil, err
+        }
+        fi.gid = int(groupid)
+
+        // File mode, 8 bytes
+        filemod, err := ar.readHeaderBytes("filemod", 8, "%8o")
+        if err != nil {
+                return nil, err
+        }
+        fi.mode = uint32(filemod)
+
+        // File size, 10 bytes
+        size, err := ar.readHeaderBytes("datasize", 10, "%10d")
+        if err != nil {
+                return nil, err
+        }
+        fi.size = size - namelen
+
+        // File magic, 2 bytes
+        err = ar.checkBytes("filemagic", []byte("\x60\n"))

[M-A Ruel, 2016/06/10 17:51:11]

create a byte array directly
merge with next line

[mithro, 2016/06/14 10:57:51]

Done.

+        if err != nil {
+                return nil, err
+        }
+
+        ar.stage = READ_BODY
+        ar.bytesrequired = size
+        ar.needspadding = (ar.bytesrequired%2 != 0)
+
+        // Filename - BSD variant
+        filename := make([]byte, namelen)
+        err = ar.readPartial("filename", filename)

[M-A Ruel, 2016/06/10 17:51:10]

merge

[mithro, 2016/06/14 10:57:52]

Done.

+        if err != nil {
+                return nil, err
+        }
+        fi.name = string(filename)

[M-A Ruel, 2016/06/10 17:51:10]

encoding is assumed to be utf8?

[mithro, 2016/06/14 10:57:52]

The "spec" seems to indicate it should be ASCII but everything seems to work
fine with utf-8.

+
+        return &fi, nil
+}
+
+func (ar *Reader) Read(b []byte) (n int, err error) {
+        err = ar.readPartial("data", b)

[M-A Ruel, 2016/06/10 17:51:11]

I'd prefer to not use named return values
merge

[mithro, 2016/06/14 10:57:52]

I actually didn't even know that was possible! Fixed.

+        if err != nil {
+                return -1, err
+        }
+        err = ar.checkFinished()
+        if err != nil {
+                return -1, err
+        }
+        return len(b), nil
+}
+
+func (ar *Reader) Next() (ArFileInfo, error) {
+        return ar.readHeader()
+}