[cipd] Add subdir support to deployer.

cipd/client/cipd/local/deployer.go

 // Copyright 2014 The LUCI Authors. All rights reserved.
 // Use of this source code is governed under the Apache License, Version 2.0
 // that can be found in the LICENSE file.
 
 package local
 
 import (
         "fmt"
         "io/ioutil"
         "math/rand"
@@ skipping 13 matching lines @@
         "github.com/luci/luci-go/common/logging"
 )
 
 // TODO(vadimsh): How to handle path conflicts between two packages? Currently
 // the last one installed wins.
 
 // TODO(vadimsh): Use some sort of file lock to reduce a chance of corruption.
 
 // File system layout of a site directory <base> for "symlink" install method:
 // <base>/.cipd/pkgs/
-//   <package name digest>/
+//   <arbitrary index>/
+//     description.json
 //     _current -> symlink to fea3ab83440e9dfb813785e16d4101f331ed44f4
 //     fea3ab83440e9dfb813785e16d4101f331ed44f4/
 //       bin/
 //         tool
 //         ...
 //       ...
 // bin/
 //    tool -> symlink to ../.cipd/pkgs/<package name digest>/_current/bin/tool
 //    ...
 //
-// Where <package name digest> is derived from a package name. It doesn't have
-// to be reversible though, since the package name is still stored in the
-// installed package manifest and can be read from there.
+// Where <arbitrary index> is chosen to be the smallest number available for
+// this installation (installing more packages gets higher numbers, removing
+// packages and installing new ones will reuse the smallest ones).

[Vadim Sh., 2017/01/28 01:50:12]

hm... this may be a problem if some updates fail mid-way: they can leave a
garbage in the site root, symlinking to .pkgs/<index> (and if <index> is reused,
they may start point to something unexpected).

[iannucci, 2017/01/30 02:05:12]

I don't think so, actually. Every index folder is created already-containing a
description.json. This means that a given numbered folder may only be used for a
single subdir+package (regardless of which client sees that folder. If a
different client/later client attempted to re-use the folder, it would only
re-use it for the same subdir+package combination (assuming no bugs in the
packageName function).

 //
 // Some efforts are made to make sure that during the deployment a window of
 // inconsistency in the file system is as small as possible.
 //
 // For "copy" install method everything is much simpler: files are directly
 // copied to the site root directory and .cipd/pkgs/* contains only metadata,
-// such as manifest file with a list of extracted files (to know what to
-// uninstall).
+// such as description and manifest files with a list of extracted files (to
+// know what to uninstall).
 
 // Deployer knows how to unzip and place packages into site root directory.
 type Deployer interface {
         // DeployInstance installs an instance of a package into the given subdir of
         // the root.
         //
         // It unpacks the package into <base>/.cipd/pkgs/*, and rearranges
         // symlinks to point to unpacked files. It tries to make it as "atomic" as
         // possible. Returns information about the deployed instance.
         DeployInstance(ctx context.Context, subdir string, inst PackageInstance) (common.Pin, error)
@@ skipping 69 matching lines @@
 // currentTxt is a name of a text file with instance ID of latest deployed
 // version. Used on Windows.
 const currentTxt = "_current.txt"
 
 // deployerImpl implements Deployer interface.
 type deployerImpl struct {
         fs FileSystem
 }
 
 func (d *deployerImpl) DeployInstance(ctx context.Context, subdir string, inst PackageInstance) (common.Pin, error) {
-        if subdir != "" {
-                return common.Pin{}, common.ErrSubdirsNotYetSupported
+        if err := common.ValidateSubdir(subdir); err != nil {
+                return common.Pin{}, err
         }
 
         pin := inst.Pin()
-        logging.Infof(ctx, "Deploying %s into %s", pin, d.fs.Root())
+        logging.Infof(ctx, "Deploying %s into %s(/%s)", pin, d.fs.Root(), subdir)
 
         // Be paranoid.
         if err := common.ValidatePin(pin); err != nil {
                 return common.Pin{}, err
         }
-        if _, err := d.fs.EnsureDirectory(ctx, d.fs.Root()); err != nil {
+        if _, err := d.fs.EnsureDirectory(ctx, filepath.Join(d.fs.Root(), subdir)); err != nil {
                 return common.Pin{}, err
         }
 
         // Extract new version to the .cipd/pkgs/* guts. For "symlink" install mode it
         // is the final destination. For "copy" install mode it's a temp destination
         // and files will be moved to the site root later (in addToSiteRoot call).
         // ExtractPackageInstance knows how to build full paths and how to atomically
         // extract a package. No need to delete garbage if it fails.
         pkgPath, err := d.packagePath(ctx, subdir, pin.PackageName, true)
         if err != nil {
@@ skipping 15 matching lines @@
         prevManifest := Manifest{}
         if err == nil && prevInstanceID != "" {
                 prevManifest, err = d.readManifest(ctx, filepath.Join(pkgPath, prevInstanceID))
         }
         if err != nil {
                 logging.Warningf(ctx, "Previous version of the package is broken: %s", err)
                 prevManifest = Manifest{} // to make sure prevManifest.Files == nil.
         }
 
         // Install all new files to the site root.
-        err = d.addToSiteRoot(ctx, newManifest.Files, newManifest.InstallMode, pkgPath, destPath)
+        err = d.addToSiteRoot(ctx, subdir, newManifest.Files, newManifest.InstallMode, pkgPath, destPath)
         if err != nil {
                 d.fs.EnsureDirectoryGone(ctx, destPath)
                 return common.Pin{}, err
         }
 
         // Mark installed instance as a current one. After this call the package is
         // considered installed and the function must not fail. All cleanup below is
         // best effort.
         if err = d.setCurrentInstanceID(ctx, pkgPath, pin.InstanceID); err != nil {
                 d.fs.EnsureDirectoryGone(ctx, destPath)
@@ skipping 21 matching lines @@
                         toKeep := map[string]bool{}
                         for _, f := range newManifest.Files {
                                 toKeep[f.Name] = true
                         }
                         toKill := []FileInfo{}
                         for _, f := range prevManifest.Files {
                                 if !toKeep[f.Name] {
                                         toKill = append(toKill, f)
                                 }
                         }
-                        d.removeFromSiteRoot(ctx, toKill)
+                        d.removeFromSiteRoot(ctx, subdir, toKill)
                 }()
         }
 
         // Verify it's all right.
         newPin, err := d.CheckDeployed(ctx, subdir, pin.PackageName)
         if err == nil && newPin.InstanceID != pin.InstanceID {
                 err = fmt.Errorf("other instance (%s) was deployed concurrently", newPin.InstanceID)
         }
         if err == nil {
                 logging.Infof(ctx, "Successfully deployed %s", pin)
         } else {
                 logging.Errorf(ctx, "Failed to deploy %s: %s", pin, err)
         }
         return newPin, err
 }
 
 func (d *deployerImpl) CheckDeployed(ctx context.Context, subdir, pkg string) (common.Pin, error) {
-        if subdir != "" {
-                return common.Pin{}, common.ErrSubdirsNotYetSupported
+        if err := common.ValidateSubdir(subdir); err != nil {
+                return common.Pin{}, err
         }
 
         pkgPath, err := d.packagePath(ctx, subdir, pkg, false)
         if err != nil {
                 return common.Pin{}, err
         }
         if pkgPath == "" {
                 return common.Pin{}, fmt.Errorf("package %s is not installed", pkg)
         }
 
@@ skipping 14 matching lines @@
         // Directories with packages are direct children of .cipd/pkgs/.
         pkgs := filepath.Join(d.fs.Root(), filepath.FromSlash(packagesDir))
         infos, err := ioutil.ReadDir(pkgs)
         if err != nil {
                 if os.IsNotExist(err) {
                         return nil, nil
                 }
                 return nil, err
         }
 
-        found := map[string]common.Pin{}
-        keys := []string{}
+        found := common.PinMapBySubdir{}
         for _, info := range infos {
                 if !info.IsDir() {
                         continue
                 }
                 // Read the description and the 'current' link.
                 pkgPath := filepath.Join(pkgs, info.Name())
                 desc, err := d.readDescription(ctx, pkgPath)
                 if err != nil || desc == nil {
                         continue
                 }
                 currentID, err := d.getCurrentInstanceID(pkgPath)
                 if err != nil || currentID == "" {
                         continue
                 }
 
                 // Ignore duplicate entries, they can appear if someone messes with pkgs/*
                 // structure manually.
-                if _, ok := found[desc.PackageName]; !ok {
-                        keys = append(keys, desc.PackageName)
-                        found[desc.PackageName] = common.Pin{
-                                PackageName: desc.PackageName,
-                                InstanceID:  currentID,
+                if _, ok := found[desc.Subdir][desc.PackageName]; !ok {
+                        if _, ok := found[desc.Subdir]; !ok {
+                                found[desc.Subdir] = common.PinMap{}
                         }
+                        found[desc.Subdir][desc.PackageName] = currentID
                 }
         }
 
-        // Sort by package name.
-        sort.Strings(keys)
-        out := make(common.PinSlice, len(found))
-        for i, k := range keys {
-                out[i] = found[k]
-        }
-        return common.PinSliceBySubdir{"": out}, nil
+        return found.ToSlice(), nil
 }
 
 func (d *deployerImpl) RemoveDeployed(ctx context.Context, subdir, packageName string) error {
-        if subdir != "" {
-                return common.ErrSubdirsNotYetSupported
+        if err := common.ValidateSubdir(subdir); err != nil {
+                return err
         }
 
-        logging.Infof(ctx, "Removing %s from %s", packageName, d.fs.Root())
+        logging.Infof(ctx, "Removing %s from %s(/%s)", packageName, d.fs.Root(), subdir)
         if err := common.ValidatePackageName(packageName); err != nil {
                 return err
         }
         pkgPath, err := d.packagePath(ctx, subdir, packageName, false)
         if err != nil {
                 return err
         }
         if pkgPath == "" {
                 logging.Warningf(ctx, "Package %s not found", packageName)
                 return nil
         }
 
         // Read the manifest of the currently installed version.
         manifest := Manifest{}
         currentID, err := d.getCurrentInstanceID(pkgPath)
         if err == nil && currentID != "" {
                 manifest, err = d.readManifest(ctx, filepath.Join(pkgPath, currentID))
         }
 
         // Warn, but continue with removal anyway. EnsureDirectoryGone call below
         // will nuke everything (even if it's half broken).
         if err != nil {
                 logging.Warningf(ctx, "Package %s is in a broken state: %s", packageName, err)
         } else {
-                d.removeFromSiteRoot(ctx, manifest.Files)
+                d.removeFromSiteRoot(ctx, subdir, manifest.Files)
         }
         return d.fs.EnsureDirectoryGone(ctx, pkgPath)
 }
 
 func (d *deployerImpl) TempFile(ctx context.Context, prefix string) (*os.File, error) {
         dir, err := d.fs.EnsureDirectory(ctx, filepath.Join(d.fs.Root(), SiteServiceDir, "tmp"))
         if err != nil {
                 return nil, err
         }
         return ioutil.TempFile(dir, prefix)
@@ skipping 44 matching lines @@
 // packagePath returns a path to a package directory in .cipd/pkgs/.
 //
 // This will scan all directories under pkgs, looking for a description.json. If
 // an old-style package folder is encountered (e.g. has an instance folder and
 // current manifest, but doesn't have a description.json), the description.json
 // will be added.
 //
 // If no suitable path is found and allocate is true, this will create a new
 // directory with an accompanying description.json. Otherwise this returns "".
 func (d *deployerImpl) packagePath(ctx context.Context, subdir, pkg string, allocate bool) (string, error) {
+        if err := common.ValidateSubdir(subdir); err != nil {
+                return "", err
+        }
+
         if err := common.ValidatePackageName(pkg); err != nil {
                 return "", err
         }
 
         rel := filepath.Join(filepath.FromSlash(packagesDir))

[Vadim Sh., 2017/01/28 01:50:12]

er... why do we do filepath.Join here?

[iannucci, 2017/01/31 01:08:41]

dunno!

         abs, err := d.fs.RootRelToAbs(rel)
         if err != nil {
                 logging.Errorf(ctx, "Can't get absolute path of %q: %s", rel, err)
                 return "", err
         }
         files, err := ioutil.ReadDir(abs)
         if err != nil && !os.IsNotExist(err) {
                 logging.Errorf(ctx, "Can't read packages dir %q: %s", abs, err)
                 return "", err
         }
 
         seenNumbers := numSet{}
 
         for _, f := range files {
                 // keep track of all numeric children of .cipd/pkgs
                 if n, err := strconv.Atoi(f.Name()); err == nil {
                         seenNumbers.addNum(n)
                 }
 
                 fullPkgPath := filepath.Join(abs, f.Name())
                 description, err := d.readDescription(ctx, fullPkgPath)
                 if err != nil {
                         logging.Warningf(ctx, "Skipping %q: %s", fullPkgPath, err)
                         continue
                 }
-                if description.PackageName == pkg {
+                if description.PackageName == pkg && description.Subdir == subdir {
                         return fullPkgPath, nil
                 }
         }
 
         if !allocate {
                 return "", nil
         }
 
         // we didn't find one, so we have to make one
         if _, err := d.fs.EnsureDirectory(ctx, abs); err != nil {
                 logging.Errorf(ctx, "Cannot ensure packages directory: %s", err)
                 return "", err
         }
 
         tmpDir, err := d.TempDir(ctx, strings.Replace(pkg, "/", "_", -1))

[Vadim Sh., 2017/01/28 01:50:12]

using 'pkg' here can be a problem on windows. We formally do not restrict length
of package names (e.g. one can have
really/long/package/name/in/cipd/repository/and/it/should/work/windows-amd64).

[iannucci, 2017/01/31 01:08:41]

done, I have it take the last two components now.

         if err != nil {
                 logging.Errorf(ctx, "Cannot create new pkg tempdir: %s", err)
                 return "", err
         }
         defer d.fs.EnsureDirectoryGone(ctx, tmpDir)
         err = d.fs.EnsureFile(ctx, filepath.Join(tmpDir, descriptionName), func(f *os.File) error {
                 return writeDescription(&Description{Subdir: subdir, PackageName: pkg}, f)
         })
         if err != nil {
                 logging.Errorf(ctx, "Cannot create new pkg description.json: %s", err)
@@ skipping 27 matching lines @@
                         return "", err
                 }
 
                 // rename failed with ENOTEMPTY, that means that another client wrote this
                 // directory.
                 description, err := d.readDescription(ctx, pkgPath)
                 if err != nil {
                         logging.Warningf(ctx, "Skipping %q: %s", pkgPath, err)
                         continue
                 }
-                if description.PackageName == pkg {
+                if description.PackageName == pkg && description.Subdir == subdir {
                         return pkgPath, nil
                 }
         }
 
         logging.Errorf(ctx, "Unable to find valid index for package %q in %s!", pkg, abs)
         return "", err
 }
 
 // getCurrentInstanceID returns instance ID of currently installed instance
 // given a path to a package directory (.cipd/pkgs/<name>).
@@ skipping 114 matching lines @@
         if len(manifest.Files) == 0 {
                 if manifest.Files, err = scanPackageDir(ctx, instanceDir); err != nil {
                         return Manifest{}, err
                 }
         }
         return manifest, nil
 }
 
 // addToSiteRoot moves or symlinks files into the site root directory (depending
 // on passed installMode).
-func (d *deployerImpl) addToSiteRoot(ctx context.Context, files []FileInfo, installMode InstallMode, pkgDir, srcDir string) error {
+func (d *deployerImpl) addToSiteRoot(ctx context.Context, subdir string, files []FileInfo, installMode InstallMode, pkgDir, srcDir string) error {
         // On Windows only InstallModeCopy is supported.
         if runtime.GOOS == "windows" {
                 installMode = InstallModeCopy
         } else if installMode == "" {
                 installMode = InstallModeSymlink // default on non-Windows
         }
         if err := ValidateInstallMode(installMode); err != nil {
                 return err
         }
 
         for _, f := range files {
-                // E.g. bin/tool.
+                // e.g. bin/tool
                 relPath := filepath.FromSlash(f.Name)
-                destAbs, err := d.fs.RootRelToAbs(relPath)
+                // e.g. <base>/<subdir>/bin/tool
+                destAbs, err := d.fs.RootRelToAbs(filepath.Join(subdir, relPath))
                 if err != nil {
                         logging.Warningf(ctx, "Invalid relative path %q: %s", relPath, err)
                         return err
                 }
                 if installMode == InstallModeSymlink {
-                        // E.g. <base>/.cipd/pkgs/name/_current/bin/tool.
+                        // e.g. <base>/.cipd/pkgs/name/_current/bin/tool
                         targetAbs := filepath.Join(pkgDir, currentSymlink, relPath)
-                        // E.g. ../.cipd/pkgs/name/_current/bin/tool.
+                        // e.g. ../.cipd/pkgs/name/_current/bin/tool
+                        // has more `../` depending on subdir
                         targetRel, err := filepath.Rel(filepath.Dir(destAbs), targetAbs)
                         if err != nil {
                                 logging.Warningf(
                                         ctx, "Can't get relative path from %s to %s",
                                         filepath.Dir(destAbs), targetAbs)
                                 return err
                         }
                         if err = d.fs.EnsureSymlink(ctx, destAbs, targetRel); err != nil {
                                 logging.Warningf(ctx, "Failed to create symlink for %s", relPath)
                                 return err
                         }
                 } else if installMode == InstallModeCopy {
                         // E.g. <base>/.cipd/pkgs/name/<id>/bin/tool.
                         srcAbs := filepath.Join(srcDir, relPath)
                         if err := d.fs.Replace(ctx, srcAbs, destAbs); err != nil {
                                 logging.Warningf(ctx, "Failed to move %s to %s: %s", srcAbs, destAbs, err)
                                 return err
                         }
                 } else {
                         // Should not happen. ValidateInstallMode checks this.
                         return fmt.Errorf("impossible state")
                 }
         }
-        // Best effort cleanup of empty directories after all files has been moved.
+        // Best effort cleanup of empty directories after all files have been moved.
         if installMode == InstallModeCopy {
                 d.removeEmptyDirs(ctx, srcDir)
         }
         return nil
 }
 
 // removeFromSiteRoot deletes files from the site root directory.
 //
 // Best effort. Logs errors and carries on.
-func (d *deployerImpl) removeFromSiteRoot(ctx context.Context, files []FileInfo) {
+func (d *deployerImpl) removeFromSiteRoot(ctx context.Context, subdir string, files []FileInfo) {
         for _, f := range files {
-                absPath, err := d.fs.RootRelToAbs(filepath.FromSlash(f.Name))
+                absPath, err := d.fs.RootRelToAbs(filepath.Join(subdir, filepath.FromSlash(f.Name)))
+                fmt.Printf("wat: %q, %q, %q\n", subdir, files[0].Name, absPath)

[Vadim Sh., 2017/01/28 01:50:12]

wat? :)

[iannucci, 2017/01/31 01:08:41]

:D

                 if err != nil {
                         logging.Warningf(ctx, "Refusing to remove %q: %s", f.Name, err)
                         continue
                 }
                 if err := d.fs.EnsureFileGone(ctx, absPath); err != nil {
                         logging.Warningf(ctx, "Failed to remove a file from the site root: %s", err)
                 }
         }
         d.removeEmptyDirs(ctx, d.fs.Root())
 }
@@ skipping 43 matching lines @@
                                         Size:       uint64(info.Size()),
                                         Executable: (info.Mode().Perm() & 0111) != 0,
                                         Symlink:    symlink,
                                 })
                         }
                 }
                 return nil
         })
         return out, err
 }