client/isolate: expand deps into individual files

client/cmd/isolate/exp_archive.go

Index: client/cmd/isolate/exp_archive.go
diff --git a/client/cmd/isolate/exp_archive.go b/client/cmd/isolate/exp_archive.go
index 53ba6737042927f8746f12c035721cb55d007803..54d0751529e109b1d81c4192cc51180e20bde36c 100644
--- a/client/cmd/isolate/exp_archive.go
+++ b/client/cmd/isolate/exp_archive.go
@@ -13,11 +13,19 @@ import (
 	"os"
 	"path/filepath"
 
+	humanize "github.com/dustin/go-humanize"
 	"github.com/luci/luci-go/client/isolate"
 	"github.com/luci/luci-go/common/isolated"
 	"github.com/maruel/subcommands"
 )
 
+const (
+	// archiveThreshold is the size (in bytes) used to determine whether to add
+	// files to a tar archive before uploading. Files smaller than this size will
+	// be combined into archives before being uploaded to the server.
+	archiveThreshold = 100e3 // 100kB
+)
+
 var cmdExpArchive = &subcommands.Command{
 	UsageLine: "exparchive <options>",
 	ShortDesc: "EXPERIMENTAL parses a .isolate file to create a .isolated file, and uploads it and all referenced files to an isolate server",
@@ -37,6 +45,14 @@ type expArchiveRun struct {
 	isolateFlags      isolateFlags
 }
 
+// Item represents a file or symlink referenced by an isolate file.
+type Item struct {
+	Path    string
+	RelPath string
+	Size    int64
+	Mode    os.FileMode
+}
+
 // main contains the core logic for experimental archive.
 func (c *expArchiveRun) main() error {
 	archiveOpts := &c.isolateFlags.ArchiveOptions
@@ -47,8 +63,53 @@ func (c *expArchiveRun) main() error {
 	}
 	log.Printf("Isolate referenced %d deps", len(deps))
 
-	// TODO(djd): actually do something with the isolated.
-	_ = rootDir
+	// Walk each of the deps, partioning the results into symlinks and files categorised by size.
+	var links, archiveFiles, indivFiles []*Item
+	var archiveSize, indivSize int64 // Cumulative size of archived/individual files.
+	for _, dep := range deps {
+		// Try to walk dep. If dep is a file (or symlink), the inner function is called exactly once.
+		err := filepath.Walk(filepath.Clean(dep), func(path string, info os.FileInfo, err error) error {
+			if err != nil {
+				return err
+			}
+			if info.IsDir() {
+				return nil
+			}
+
+			relPath, err := filepath.Rel(rootDir, path)
+			if err != nil {
+				return err
+			}
+
+			item := &Item{
+				Path:    path,
+				RelPath: relPath,
+				Mode:    info.Mode(),
+				Size:    info.Size(),
+			}
+
+			switch {
+			case item.Mode&os.ModeSymlink == os.ModeSymlink:
+				links = append(links, item)
+			case item.Size < archiveThreshold:
+				archiveFiles = append(archiveFiles, item)
+				archiveSize += item.Size
+			default:
+				indivFiles = append(indivFiles, item)
+				indivSize += item.Size
+			}
+			return nil
+		})
+		if err != nil {
+			return err
+		}
+	}
+
+	log.Printf("Isolate expanded to %d files (total size %s) and %d symlinks", len(archiveFiles)+len(indivFiles), humanize.Bytes(uint64(archiveSize+indivSize)), len(links))
+	log.Printf("\t%d files (%s) to be isolated individually", len(indivFiles), humanize.Bytes(uint64(indivSize)))
+	log.Printf("\t%d files (%s) to be isolated in archives", len(archiveFiles), humanize.Bytes(uint64(archiveSize)))
+
+	// TODO(djd): actually do something with the each of links, archiveFiles and indivFiles.
 
 	// Marshal the isolated file into JSON.
 	isolJSON, err := json.Marshal(isol)