client/isolate: expand deps into individual files

client/cmd/isolate/exp_archive.go

 // Copyright 2015 The LUCI Authors. All rights reserved.
 // Use of this source code is governed under the Apache License, Version 2.0
 // that can be found in the LICENSE file.
 
 package main
 
 import (
         "encoding/json"
         "errors"
         "fmt"
         "io/ioutil"
         "log"
         "os"
         "path/filepath"
 
+        humanize "github.com/dustin/go-humanize"
         "github.com/luci/luci-go/client/isolate"
         "github.com/luci/luci-go/common/isolated"
         "github.com/maruel/subcommands"
 )
 
+const (
+        // archiveThreshold is the size (in bytes) used to determine whether to add

[mcgreevy, 2016/11/22 23:23:47]

I've added some nitpicky comments about the wording of this comment, but I
actually think you'd be better off just doing:

func shouldArchive(item Item) bool {
  return item.Size < 100 * 1024
}

I think that this function will be simpler to document.  Then, on line 81, do:

var links, archiveFiles, individualFiles []*Item

etc.

[djd-OOO-Apr2017, 2016/11/23 00:36:34]

In future stages of this change we have more thresholds that need to be
documented (like such as the total size to which we let archives grow). I'd like
to have a const() stanza at the start of the file which lets us clearly document
what these magic thresholds are (and easily switch to a flag-controlled
threshold instead).

So if I introduced a func, I'd still like to have the const. At that point a
func/method doesn't seem to hold its weight. wdyt?

[mcgreevy, 2016/11/23 00:44:13]

OK, let's stick with the consts, no func, and see how we go (you could still
change the name changes on line 81 though).

[djd-OOO-Apr2017, 2016/11/23 00:46:29]

(I had done – see ps2)

+        // files to an tar archive before uploading. Files smaller than this size will

[mcgreevy, 2016/11/22 23:23:47]

s/an tar/a tar/

[djd-OOO-Apr2017, 2016/11/23 00:36:34]

Done.

+        // be combined into archives before

[mcgreevy, 2016/11/22 23:23:46]

s/before uploaded directly to/before being uploaded to/

[djd-OOO-Apr2017, 2016/11/23 00:36:34]

Done.

+        // uploaded directly to the server.
+        archiveThreshold = 100e3 // 100kB

[mcgreevy, 2016/11/22 23:23:47]

I'd expect this to be 100 * 1024 ...

[djd-OOO-Apr2017, 2016/11/23 00:36:34]

Without invoking religious wars (https://en.wikipedia.org/wiki/Kibibyte), I
thought 100e3 was easier to read (in the context of also having a maximum
archive size 10e6). I don't think it matter much either way. Happy to be swayed.

[mcgreevy, 2016/11/23 00:44:13]

I don't care that much.

+)
+
 var cmdExpArchive = &subcommands.Command{
         UsageLine: "exparchive <options>",
         ShortDesc: "EXPERIMENTAL creates a .isolated file and uploads the tree to an isolate server.",
         LongDesc:  "All the files listed in the .isolated file are put in the isolate server cache. Small files are combined together in a tar archive before uploading.",
         CommandRun: func() subcommands.CommandRun {
                 c := expArchiveRun{}
                 c.commonServerFlags.Init()
                 c.isolateFlags.Init(&c.Flags)
                 return &c
         },
@@ skipping 14 matching lines @@
         }
         if err := c.isolateFlags.Parse(cwd, RequireIsolatedFile); err != nil {
                 return err
         }
         if len(args) != 0 {
                 return errors.New("position arguments not expected")
         }
         return nil
 }
 
+// Item represents a file or symlink referenced by an isolate file.
+type Item struct {
+        Path    string
+        RelPath string
+        Size    int64
+        Mode    os.FileMode
+}
+
 func (c *expArchiveRun) main(a subcommands.Application, args []string) error {
         // Parse the incoming isolate file.
         deps, rootDir, isol, err := isolate.ProcessIsolate(&c.ArchiveOptions)
         if err != nil {
                 return fmt.Errorf("failed to process isolate: %v", err)
         }
         log.Printf("Isolate referenced %d deps", len(deps))
 
-        // TODO(djd): actually do something with the isolated.
-        _ = rootDir
+        // Walk each of the deps, partioning the results into symlinks and files categorised by size.
+        var links, smallFiles, largeFiles []*Item
+        var smallSize, largeSize int64 // Cumulative size of small/large files.
+        for _, dep := range deps {
+                // Try to walk dep. If dep is a file (or symlink), the inner function is called exactly once.
+                err := filepath.Walk(filepath.Clean(dep), func(path string, info os.FileInfo, err error) error {
+                        if err != nil {
+                                return err
+                        }
+                        if info.IsDir() {
+                                return nil
+                        }
+
+                        relPath, err := filepath.Rel(rootDir, path)
+                        if err != nil {
+                                return err
+                        }
+
+                        item := &Item{
+                                Path:    path,
+                                RelPath: relPath,
+                                Mode:    info.Mode(),
+                                Size:    info.Size(),
+                        }
+
+                        switch {
+                        case item.Mode&os.ModeSymlink == os.ModeSymlink:
+                                links = append(links, item)
+                        case item.Size < archiveThreshold:
+                                smallFiles = append(smallFiles, item)
+                                smallSize += item.Size
+                        default:
+                                largeFiles = append(largeFiles, item)
+                                largeSize += item.Size
+                        }
+                        return nil
+                })
+                if err != nil {
+                        return err
+                }
+        }
+
+        log.Printf("Isolate expanded to %d files (total size %s) and %d symlinks", len(smallFiles)+len(largeFiles), humanize.Bytes(uint64(smallSize+largeSize)), len(links))
+        log.Printf("\t%d files (%s) to be isolated individually", len(largeFiles), humanize.Bytes(uint64(largeSize)))
+        log.Printf("\t%d files (%s) to be isolated in archives", len(smallFiles), humanize.Bytes(uint64(smallSize)))
+
+        // TODO(djd): actually do something with the each of links, smallFiles and largeFiles.
 
         // Marshal the isolated file into JSON.
         isolJSON, err := json.Marshal(isol)
         if err != nil {
                 return err
         }
         // TODO(djd): actually check/upload the isolated.
 
         // Write the isolated file, and emit its digest to stdout.
         if err := ioutil.WriteFile(c.ArchiveOptions.Isolated, isolJSON, 0644); err != nil {
@@ skipping 13 matching lines @@
         if len(c.ArchiveOptions.Blacklist) != 0 {
                 fmt.Fprintf(a.GetErr(), "%s: blacklist is not supported\n", a.GetName())
                 return 1
         }
         if err := c.main(a, args); err != nil {
                 fmt.Fprintf(a.GetErr(), "%s: %s\n", a.GetName(), err)
                 return 1
         }
         return 0
 }