vpython: Add VirtualEnv creation package.

vpython/venv/prune.go

+// Copyright 2017 The LUCI Authors. All rights reserved.
+// Use of this source code is governed under the Apache License, Version 2.0
+// that can be found in the LICENSE file.
+
+package venv
+
+import (
+        "io"
+        "os"
+        "strings"
+        "time"
+
+        "github.com/luci/luci-go/common/clock"
+        "github.com/luci/luci-go/common/data/rand/mathrand"
+        "github.com/luci/luci-go/common/errors"
+        "github.com/luci/luci-go/common/logging"
+
+        "github.com/danjacques/gofslock/fslock"
+        "golang.org/x/net/context"
+)
+
+// pruneReadDirSize is the number of entries to read in a directory at a time
+// when pruning.
+const pruneReadDirSize = 128
+
+// prune examines environments in cfg's BaseDir. If any are found that are older
+// than the prune threshold in "cfg", they will be safely deleted.
+func prune(c context.Context, cfg *Config, omit string) error {

[iannucci, 2017/02/23 00:54:21]

what's omit?

[dnj, 2017/02/23 20:38:49]

Done.

+        if cfg.PruneThreshold <= 0 {
+                // Pruning is disabled.
+                return nil
+        }
+        pruneThreshold := clock.Now(c).Add(-cfg.PruneThreshold)
+
+        // Get a listing of all VirtualEnv within the base directory.
+        dir, err := os.Open(cfg.BaseDir)
+        if err != nil {
+                return errors.Annotate(err).Reason("failed to open base directory: %(dir)s").
+                        D("dir", cfg.BaseDir).
+                        Err()
+        }
+        defer dir.Close()
+
+        // Run a series of independent scan/prune operations.
+        logging.Debugf(c, "Pruning entries in [%s] older than %s.", cfg.BaseDir, cfg.PruneThreshold)
+
+        var (
+                allErrs     errors.MultiError
+                totalPruned = 0
+                done        = false
+        )
+        for !done {
+                fileInfos, err := dir.Readdir(pruneReadDirSize)
+                switch err {
+                case nil:
+
+                case io.EOF:
+                        done = true
+
+                default:
+                        return errors.Annotate(err).Reason("could not read directory contents: %(dir)s").
+                                D("dir", err).
+                                Err()
+                }
+
+                // Shuffle the slice randomly. We do this in case others are also processing
+                // this directory simultaneously.
+                for i := range fileInfos {
+                        j := mathrand.Intn(c, i+1)
+                        fileInfos[i], fileInfos[j] = fileInfos[j], fileInfos[i]
+                }
+
+                for _, fi := range fileInfos {
+                        if strings.HasPrefix(fi.Name(), ".") {
+                                // Ignore hidden files.

[iannucci, 2017/02/23 00:54:20]

do we have to worry about them growing unbounded?

[dnj, 2017/02/23 20:38:49]

We shouldn't have to. If someone puts their own stuff under here, then there is
a problem; otherwise, the worst worst case is we leak lock files, which are
empty.

+                                continue
+                        }
+
+                        switch pruned, err := maybePruneFile(c, cfg, fi, pruneThreshold, omit); {
+                        case err != nil:
+                                allErrs = append(allErrs, errors.Annotate(err).
+                                        Reason("failed to prune file: %(name)s").
+                                        D("name", fi.Name()).
+                                        D("dir", cfg.BaseDir).
+                                        Err())
+
+                        case pruned:
+                                totalPruned++
+                                if cfg.PruneLimit > 0 && totalPruned >= cfg.PruneLimit {
+                                        logging.Debugf(c, "Hit prune limit of %d.", cfg.PruneLimit)
+                                        done = true
+                                }
+                        }
+                }
+        }
+
+        logging.Debugf(c, "Pruned %d environment(s) with %d error(s)", totalPruned, len(allErrs))

[iannucci, 2017/02/23 00:54:20]

I think this should be info (and look like `Pruned 10 (limit) with ....` in the
case it hit the limit)

[dnj, 2017/02/23 20:38:49]

Done.

+        if len(allErrs) > 0 {
+                return allErrs
+        }
+        return nil
+}
+
+// maybePruneFile examines the specified FileIfo within cfg.BaseDir and
+// determines if it should be pruned.
+func maybePruneFile(c context.Context, cfg *Config, fi os.FileInfo, pruneThreshold time.Time, omit string) (
+        pruned bool, err error) {
+
+        if !fi.IsDir() || fi.Name() == omit {
+                return

[iannucci, 2017/02/23 00:54:21]

log?

[dnj, 2017/02/23 20:38:49]

Done.

+        }
+
+        // Grab the lock file for this directory.
+        e := cfg.envForName(fi.Name())
+        err = fslock.With(e.lockPath, func() error {
+                // Read the complete flag file's timestamp.
+                switch st, err := os.Stat(e.completeFlagPath); {
+                case err == nil:
+                        if !st.ModTime().Before(pruneThreshold) {
+                                return nil
+                        }
+
+                        logging.Infof(c, "Env [%s] is older than the prune threshold (%v < %v); pruning...",
+                                e.name, st.ModTime(), pruneThreshold)
+
+                case os.IsNotExist(err):
+                        logging.Infof(c, "Env [%s] has no completed flag; pruning...", e.name)
+
+                default:
+                        return errors.Annotate(err).Reason("failed to stat complete flag: %(path)s").
+                                D("path", e.completeFlagPath).
+                                Err()
+                }
+
+                // Delete the environment. We currently hold its lock, so use deleteLocked.
+                if err := e.deleteLocked(c); err != nil {

[iannucci, 2017/02/23 00:54:20]

does it make sense to mv this to a 'prune' folder first, and have all prune
processes opportunistically try to delete the contents of the prune folder?

[dnj, 2017/02/23 20:38:49]

There may be a race here with moving. ATM deletion of a directory is
synchronized off of ownership of its lock file. That should be sufficient?

[iannucci, 2017/03/11 00:37:34]

I'm thinking about the case where it /partially/ prunes the file (e.g. on
windows) because there's an open file handle or something. If it moves to a
trash folder and everything tries to delete the trash folder, then we won't fail
part-way while holding the lock and end up with a half-destroyed virtualenv.

+                        return errors.Annotate(err).Reason("failed to delete Env").Err()
+                }
+                pruned = true
+                return nil
+        })
+        switch err {
+        case nil:
+                return
+
+        case fslock.ErrLockHeld:
+                // Something else currently holds the lock for this directory, so ignore it.
+                logging.Warningf(c, "Lock [%s] is currently held; skipping.", e.lockPath)
+                return
+
+        default:
+                err = errors.Annotate(err).Err()
+                return
+        }
+}