Reuse blacklisting code in exp_archive.

client/cmd/isolate/exp_archive.go

 // Copyright 2015 The LUCI Authors.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
 //      http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
@@ skipping 12 matching lines @@
         "os"
         "path/filepath"
         "strings"
         "time"
 
         humanize "github.com/dustin/go-humanize"
         "github.com/golang/protobuf/proto"
         "github.com/maruel/subcommands"
         "golang.org/x/net/context"
 
+        "github.com/luci/luci-go/client/internal/common"
         "github.com/luci/luci-go/client/isolate"
         "github.com/luci/luci-go/common/auth"
         logpb "github.com/luci/luci-go/common/eventlog/proto"
         "github.com/luci/luci-go/common/isolated"
         "github.com/luci/luci-go/common/isolatedclient"
 )
 
 const (
         // archiveThreshold is the size (in bytes) used to determine whether to add
         // files to a tar archive before uploading. Files smaller than this size will
@@ skipping 50 matching lines @@
         totalSize int64
 }
 
 func (ig *itemGroup) AddItem(item *Item) {
         ig.items = append(ig.items, item)
         ig.totalSize += item.Size
 }
 
 // partitioningWalker contains the state necessary to partition isolate deps by handling multiple os.WalkFunc invocations.
 type partitioningWalker struct {
-        // rootDir must be initialized before walkFn is called.
-        rootDir string
+        // fsView must be initialized before walkFn is called.
+        fsView common.FilesystemView
 
         parts partitionedDeps
 }
 
 // partitionedDeps contains a list of items to be archived, partitioned into symlinks and files categorized by size.
 type partitionedDeps struct {
         links          itemGroup
         filesToArchive itemGroup
         indivFiles     itemGroup
 }
 
 // walkFn implements filepath.WalkFunc, for use traversing a directory hierarchy to be isolated.
 // It accumulates files in pw.parts, partitioned into symlinks and files categorized by size.
 func (pw *partitioningWalker) walkFn(path string, info os.FileInfo, err error) error {
         if err != nil {
                 return err
         }
+
+        relPath, err := pw.fsView.RelativePath(path)
+        if err != nil {
+                return err
+        }
+
+        if relPath == "" { // empty string indicates skip.
+                return returnSkip(info)
+        }
+
         if info.IsDir() {
                 return nil
         }
 
-        relPath, err := filepath.Rel(pw.rootDir, path)
-        if err != nil {
-                return err
-        }
-
         item := &Item{
                 Path:    path,
                 RelPath: relPath,
                 Mode:    info.Mode(),
                 Size:    info.Size(),
         }
 
         switch {
         case item.Mode&os.ModeSymlink == os.ModeSymlink:
                 pw.parts.links.AddItem(item)
         case item.Size < archiveThreshold:
                 pw.parts.filesToArchive.AddItem(item)
         default:
                 pw.parts.indivFiles.AddItem(item)
         }
         return nil
 }
 
+// returnSkip returns the return value expected from a filepath.WalkFunc in the case where no more processing of file should occur.

[mithro, 2017/07/21 04:58:51]

nit: This sentence is a bit clunky.

[mcgreevy, 2017/07/21 05:40:38]

I've reworded; it's quite hard to word simply, though. Any suggestions are
welcome.

I've also moved this into a common package, since it was duplicated in
directory.go.

+func returnSkip(file os.FileInfo) error {
+        if file.IsDir() {
+                // Must not return io.SkipDir for file, filepath.walk() handles this badly.

[mithro, 2017/07/21 04:58:51]

Could you add a reference to info about this? Maybe a bug or godoc link?

[mcgreevy, 2017/07/21 05:40:38]

Done.

+                return filepath.SkipDir
+        }
+        return nil
+}
+
 // partitionDeps walks each of the deps, partioning the results into symlinks and files categorized by size.
 func partitionDeps(deps []string, rootDir string) (partitionedDeps, error) {
-        walker := partitioningWalker{rootDir: rootDir}
+        // TODO(mcgreevy): initialize FilesystemView with blacklist.
+        fsView, err := common.NewFilesystemView(rootDir, nil)
+        if err != nil {
+                return partitionedDeps{}, err
+        }
+
+        walker := partitioningWalker{fsView: fsView}
         for _, dep := range deps {
                 // Try to walk dep. If dep is a file (or symlink), the inner function is called exactly once.
                 if err := filepath.Walk(filepath.Clean(dep), walker.walkFn); err != nil {
                         return partitionedDeps{}, err
                 }
         }
         return walker.parts, nil
 }
 
 // main contains the core logic for experimental archive.
@@ skipping 217 matching lines @@
 }
 
 func hashFile(path string) (isolated.HexDigest, error) {
         f, err := os.Open(path)
         if err != nil {
                 return "", err
         }
         defer f.Close()
         return isolated.Hash(f)
 }