cipd: Extract high level file system operations into the separate interface.

go/src/infra/tools/cipd/local/deployer.go

 // Copyright 2014 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 package local
 
 import (
         "crypto/sha1"
         "encoding/base64"
         "fmt"
         "io/ioutil"
         "os"
         "path/filepath"
         "sort"
         "strings"
         "sync"
-        "time"
 
         "github.com/luci/luci-go/common/logging"
 
         "infra/tools/cipd/common"
 )
 
 // TODO(vadimsh): Make it work on Windows.
 
 // TODO(vadimsh): How to handle path conflicts between two packages? Currently
 // the last one installed wins.
@@ skipping 47 matching lines @@
                 err = fmt.Errorf("Site root path is not provided")
         } else {
                 root, err = filepath.Abs(filepath.Clean(root))
         }
         if err != nil {
                 return errDeployer{err}
         }
         if logger == nil {
                 logger = logging.Null()
         }
-        return &deployerImpl{root, logger}
+        return &deployerImpl{NewFileSystem(root, logger), logger}
 }
 
 ////////////////////////////////////////////////////////////////////////////////
 // Implementation that returns error on all requests.
 
 type errDeployer struct{ err error }
 
 func (d errDeployer) DeployInstance(PackageInstance) (common.Pin, error)   { return common.Pin{}, d.err }
 func (d errDeployer) CheckDeployed(packageName string) (common.Pin, error) { return common.Pin{}, d.err }
 func (d errDeployer) FindDeployed() (out []common.Pin, err error)          { return nil, d.err }
 func (d errDeployer) RemoveDeployed(packageName string) error              { return d.err }
 func (d errDeployer) TempFile(prefix string) (*os.File, error)             { return nil, d.err }
 
 ////////////////////////////////////////////////////////////////////////////////
 // Real deployer implementation.
 
 // packagesDir is a subdirectory of site root to extract packages to.
 const packagesDir = siteServiceDir + "/pkgs"
 
 // currentSymlink is a name of a symlink that points to latest deployed version.
 const currentSymlink = "_current"
 
 // deployerImpl implements Deployer interface.
 type deployerImpl struct {
-        root   string
+        fs     FileSystem
         logger logging.Logger
 }
 
 func (d *deployerImpl) DeployInstance(inst PackageInstance) (common.Pin, error) {
         pin := inst.Pin()
-        d.logger.Infof("Deploying %s into %s", pin, d.root)
+        d.logger.Infof("Deploying %s into %s", pin, d.fs.Root())
 
         // Be paranoid.
         if err := common.ValidatePin(pin); err != nil {
                 return common.Pin{}, err
         }
-        if err := d.ensureRootExists(); err != nil {
+        if _, err := d.fs.EnsureDirectory(d.fs.Root()); err != nil {
                 return common.Pin{}, err
         }
 
         // Remember currently deployed version (to remove it later). Do not freak out
         // if it's not there (prevID is "" in that case).
         oldFiles := makeStringSet()
         prevID := d.findDeployedInstance(pin.PackageName, oldFiles)
 
         // Extract new version to a final destination.
         newFiles := makeStringSet()
         destPath, err := d.deployInstance(inst, newFiles)
         if err != nil {
                 return common.Pin{}, err
         }
 
         // Switch '_current' symlink to point to a new package instance. It is a
         // point of no return. The function must not fail going forward.
         mainSymlinkPath := d.packagePath(pin.PackageName, currentSymlink)
-        err = ensureSymlink(mainSymlinkPath, pin.InstanceID)
+        err = d.fs.EnsureSymlink(mainSymlinkPath, pin.InstanceID)
         if err != nil {
-                ensureDirectoryGone(destPath, d.logger)
+                d.fs.EnsureDirectoryGone(destPath)
                 return common.Pin{}, err
         }
 
         // Asynchronously remove previous version (best effort).
         wg := sync.WaitGroup{}
         defer wg.Wait()
         if prevID != "" && prevID != pin.InstanceID {
                 wg.Add(1)
                 go func() {
                         defer wg.Done()
-                        ensureDirectoryGone(d.packagePath(pin.PackageName, prevID), d.logger)
+                        d.fs.EnsureDirectoryGone(d.packagePath(pin.PackageName, prevID))
                 }()
         }
 
         d.logger.Infof("Adjusting symlinks for %s", pin.PackageName)
 
         // Make symlinks in the site directory for all new files. Reference a package
         // root via '_current' symlink (instead of direct destPath), to make
         // subsequent updates 'more atomic' (since they'll need to switch only
         // '_current' symlink to update _all_ files in the site root at once).
         d.linkFilesToRoot(mainSymlinkPath, newFiles)
 
         // Delete symlinks to files no longer needed i.e. set(old) - set(new).
         for relPath := range oldFiles.diff(newFiles) {
-                ensureFileGone(filepath.Join(d.root, relPath), d.logger)
+                d.fs.EnsureFileGone(filepath.Join(d.fs.Root(), relPath))
         }
 
         // Verify it's all right, read the manifest.
         newPin, err := d.CheckDeployed(pin.PackageName)
         if err == nil && newPin.InstanceID != pin.InstanceID {
                 err = fmt.Errorf("Other instance (%s) was deployed concurrently", newPin.InstanceID)
         }
         if err == nil {
                 d.logger.Infof("Successfully deployed %s", pin)
         } else {
                 d.logger.Errorf("Failed to deploy %s: %s", pin, err)
         }
         return newPin, err
 }
 
 func (d *deployerImpl) CheckDeployed(pkg string) (common.Pin, error) {
         pin, err := readPackageState(d.packagePath(pkg))
         if err == nil && pin.PackageName != pkg {
                 err = fmt.Errorf("Package path and package name in the manifest do not match")
         }
         return pin, err
 }
 
 func (d *deployerImpl) FindDeployed() (out []common.Pin, err error) {
         // Directories with packages are direct children of .cipd/pkgs/.
-        pkgs := filepath.Join(d.root, filepath.FromSlash(packagesDir))
+        pkgs := filepath.Join(d.fs.Root(), filepath.FromSlash(packagesDir))
         infos, err := ioutil.ReadDir(pkgs)
         if err != nil {
                 if os.IsNotExist(err) {
                         err = nil
                         return
                 }
                 return
         }
 
         // Read the package name from the package manifest. Skip broken stuff.
@@ skipping 17 matching lines @@
         // Sort by package name.
         sort.Strings(keys)
         out = make([]common.Pin, len(found))
         for i, k := range keys {
                 out[i] = found[k]
         }
         return
 }
 
 func (d *deployerImpl) RemoveDeployed(packageName string) error {
-        d.logger.Infof("Removing %s from %s", packageName, d.root)
+        d.logger.Infof("Removing %s from %s", packageName, d.fs.Root())
 
         // Be paranoid.
         err := common.ValidatePackageName(packageName)
         if err != nil {
                 return err
         }
 
         // Grab list of files in currently deployed package to unlink them from root.
         files := makeStringSet()
         instanceID := d.findDeployedInstance(packageName, files)
 
         // If was installed, removed symlinks pointing to the package files.
         if instanceID != "" {
                 for relPath := range files {
-                        ensureFileGone(filepath.Join(d.root, relPath), d.logger)
+                        d.fs.EnsureFileGone(filepath.Join(d.fs.Root(), relPath))
                 }
         }
 
         // Ensure all garbage is gone even if instanceID == "" was returned.
-        return ensureDirectoryGone(d.packagePath(packageName), d.logger)
+        return d.fs.EnsureDirectoryGone(d.packagePath(packageName))
 }
 
 func (d *deployerImpl) TempFile(prefix string) (*os.File, error) {
-        if err := d.ensureRootExists(); err != nil {
-                return nil, err
-        }
-        tempPath := filepath.Join(d.root, siteServiceDir, "tmp")
-        err := os.MkdirAll(tempPath, 0777)
+        dir, err := d.fs.EnsureDirectory(filepath.Join(d.fs.Root(), siteServiceDir, "tmp"))
         if err != nil {
                 return nil, err
         }
-        return ioutil.TempFile(tempPath, prefix)
+        return ioutil.TempFile(dir, prefix)
 }
 
 ////////////////////////////////////////////////////////////////////////////////
 // Utility methods.
 
-// ensureRootExists makes site root directory if it is missing.
-func (d *deployerImpl) ensureRootExists() error {
-        err := os.MkdirAll(d.root, 0777)
-        if err == nil || os.IsExist(err) {
-                return nil
-        }
-        return err
-}
-
 // findDeployedInstance returns instanceID of a currently deployed package
 // instance and finds all files in it (adding them to 'files' set). Returns ""
 // if nothing is deployed. File paths in 'files' are relative to package root.
 func (d *deployerImpl) findDeployedInstance(pkg string, files stringSet) string {
         state, err := d.CheckDeployed(pkg)
         if err != nil {
                 return ""
         }
         scanPackageDir(d.packagePath(pkg, state.InstanceID), files)
         return state.InstanceID
 }
 
 // deployInstance atomically extracts a package instance to its final
 // destination and returns a path to it. It writes a list of extracted files
 // to 'files'. File paths in 'files' are relative to package root.
 func (d *deployerImpl) deployInstance(inst PackageInstance, files stringSet) (string, error) {
         // Extract new version to a final destination. ExtractPackageInstance knows
         // how to build full paths and how to atomically extract a package. No need
         // to delete garbage if it fails.
         destPath := d.packagePath(inst.Pin().PackageName, inst.Pin().InstanceID)
-        err := ExtractInstance(inst, NewFileSystemDestination(destPath))
+        err := ExtractInstance(inst, NewFileSystemDestination(destPath, d.fs))
         if err != nil {
                 return "", err
         }
         // Enumerate files inside. Nuke it and fail if it's unreadable.
         err = scanPackageDir(d.packagePath(inst.Pin().PackageName, inst.Pin().InstanceID), files)
         if err != nil {
-                ensureDirectoryGone(destPath, d.logger)
+                d.fs.EnsureDirectoryGone(destPath)
                 return "", err
         }
         return destPath, err
 }
 
 // linkFilesToRoot makes symlinks in root that point to files in packageRoot.
 // All targets are specified by 'files' as paths relative to packageRoot. This
 // function is best effort and thus doesn't return errors.
 func (d *deployerImpl) linkFilesToRoot(packageRoot string, files stringSet) {
         for relPath := range files {
                 // E.g <root>/bin/tool.
-                symlinkAbs := filepath.Join(d.root, relPath)
+                symlinkAbs := filepath.Join(d.fs.Root(), relPath)
                 // E.g. <root>/.cipd/pkgs/name/_current/bin/tool.
                 targetAbs := filepath.Join(packageRoot, relPath)
                 // E.g. ../.cipd/pkgs/name/_current/bin/tool.
                 targetRel, err := filepath.Rel(filepath.Dir(symlinkAbs), targetAbs)
                 if err != nil {
                         d.logger.Warningf("Can't get relative path from %s to %s", filepath.Dir(symlinkAbs), targetAbs)
                         continue
                 }
-                err = ensureSymlink(symlinkAbs, targetRel)
+                err = d.fs.EnsureSymlink(symlinkAbs, targetRel)
                 if err != nil {
                         d.logger.Warningf("Failed to create symlink for %s", relPath)
                         continue
                 }
         }
 }
 
 // packagePath joins paths together to return absolute path to .cipd/pkgs sub path.
 func (d *deployerImpl) packagePath(pkg string, rest ...string) string {
-        root := filepath.Join(d.root, filepath.FromSlash(packagesDir), packageNameDigest(pkg))
+        root := filepath.Join(d.fs.Root(), filepath.FromSlash(packagesDir), packageNameDigest(pkg))
         result := filepath.Join(append([]string{root}, rest...)...)
 
         // Be paranoid and check that everything is inside .cipd directory.
         abs, err := filepath.Abs(result)
         if err != nil {
                 msg := fmt.Sprintf("Can't get absolute path of '%s'", result)
                 d.logger.Errorf("%s", msg)
                 panic(msg)
         }
         if !isSubpath(abs, root) {
@@ skipping 56 matching lines @@
         manifest, err := readManifest(r)
         if err != nil {
                 return common.Pin{}, err
         }
         return common.Pin{
                 PackageName: manifest.PackageName,
                 InstanceID:  current,
         }, nil
 }
 
-// ensureSymlink atomically creates a symlink pointing to a target. It will
-// create full directory path if necessary.
-func ensureSymlink(symlink string, target string) error {
-        // Already set?
-        existing, err := os.Readlink(symlink)
-        if err != nil && existing == target {
-                return nil
-        }
-
-        // Make sure path exists.
-        err = os.MkdirAll(filepath.Dir(symlink), 0777)
-        if err != nil {
-                return err
-        }
-
-        // Create a new symlink file, can't modify existing one.
-        temp := fmt.Sprintf("%s_%v", symlink, time.Now().UnixNano())
-        err = os.Symlink(target, temp)
-        if err != nil {
-                return err
-        }
-
-        // Atomically replace current symlink with a new one.
-        err = os.Rename(temp, symlink)
-        if err != nil {
-                os.Remove(temp)
-                return err
-        }
-
-        return nil
-}
-
 // scanPackageDir finds a set of regular files (and symlinks) in a package
 // instance directory. Adds paths relative to 'dir' to 'out'. Skips package
 // service directories (.cipdpkg and .cipd) since they contain package deployer
 // gut files, not something that needs to be deployed.
 func scanPackageDir(dir string, out stringSet) error {
         return filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
                 if err != nil {
                         return err
                 }
                 rel, err := filepath.Rel(dir, path)
                 if err != nil {
                         return err
                 }
                 if rel == packageServiceDir || rel == siteServiceDir {
                         return filepath.SkipDir
                 }
                 if info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 {
                         out.add(rel)
                 }
                 return nil
         })
 }
 
-// ensureDirectoryGone removes the directory as instantly as possible by
-// renaming it first and only then recursively deleting.
-func ensureDirectoryGone(path string, logger logging.Logger) error {
-        temp := fmt.Sprintf("%s_%v", path, time.Now().UnixNano())
-        err := os.Rename(path, temp)
-        if err != nil {
-                if !os.IsNotExist(err) {
-                        if logger != nil {
-                                logger.Warningf("Failed to rename directory %s: %v", path, err)
-                        }
-                        return err
-                }
-                return nil
-        }
-        err = os.RemoveAll(temp)
-        if err != nil {
-                if logger != nil {
-                        logger.Warningf("Failed to remove directory %s: %v", temp, err)
-                }
-                return err
-        }
-        return nil
-}
-
-// ensureFileGone removes file, logging the errors (if any).
-func ensureFileGone(path string, logger logging.Logger) error {
-        err := os.Remove(path)
-        if err != nil && !os.IsNotExist(err) {
-                if logger != nil {
-                        logger.Warningf("Failed to remove %s", path)
-                }
-                return err
-        }
-        return nil
-}
-
 ////////////////////////////////////////////////////////////////////////////////
 // Simple stringSet implementation for keeping a set of filenames.
 
 type stringSet map[string]struct{}
 
 func makeStringSet() stringSet {
         return make(stringSet)
 }
 
 // add adds an element to the string set.
 func (a stringSet) add(elem string) {
         a[elem] = struct{}{}
 }
 
 // diff returns set(a) - set(b).
 func (a stringSet) diff(b stringSet) stringSet {
         out := makeStringSet()
         for elem := range a {
                 if _, ok := b[elem]; !ok {
                         out.add(elem)
                 }
         }
         return out
 }