vpython: Add VirtualEnv creation package.

vpython/venv/prune.go

Index: vpython/venv/prune.go
diff --git a/vpython/venv/prune.go b/vpython/venv/prune.go
new file mode 100644
index 0000000000000000000000000000000000000000..1721e5769accc710a31f6f04096b1dd403a470f5
--- /dev/null
+++ b/vpython/venv/prune.go
@@ -0,0 +1,155 @@
+// Copyright 2017 The LUCI Authors. All rights reserved.
+// Use of this source code is governed under the Apache License, Version 2.0
+// that can be found in the LICENSE file.
+
+package venv
+
+import (
+	"io"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/luci/luci-go/common/clock"
+	"github.com/luci/luci-go/common/data/rand/mathrand"
+	"github.com/luci/luci-go/common/errors"
+	"github.com/luci/luci-go/common/logging"
+
+	"github.com/danjacques/gofslock/fslock"
+	"golang.org/x/net/context"
+)
+
+// pruneReadDirSize is the number of entries to read in a directory at a time
+// when pruning.
+const pruneReadDirSize = 128
+
+// prune examines environments in cfg's BaseDir. If any are found that are older
+// than the prune threshold in "cfg", they will be safely deleted.
+func prune(c context.Context, cfg *Config, omit string) error {

[iannucci, 2017/02/23 00:54:21]

what's omit?

[dnj, 2017/02/23 20:38:49]

Done.

+	if cfg.PruneThreshold <= 0 {
+		// Pruning is disabled.
+		return nil
+	}
+	pruneThreshold := clock.Now(c).Add(-cfg.PruneThreshold)
+
+	// Get a listing of all VirtualEnv within the base directory.
+	dir, err := os.Open(cfg.BaseDir)
+	if err != nil {
+		return errors.Annotate(err).Reason("failed to open base directory: %(dir)s").
+			D("dir", cfg.BaseDir).
+			Err()
+	}
+	defer dir.Close()
+
+	// Run a series of independent scan/prune operations.
+	logging.Debugf(c, "Pruning entries in [%s] older than %s.", cfg.BaseDir, cfg.PruneThreshold)
+
+	var (
+		allErrs     errors.MultiError
+		totalPruned = 0
+		done        = false
+	)
+	for !done {
+		fileInfos, err := dir.Readdir(pruneReadDirSize)
+		switch err {
+		case nil:
+
+		case io.EOF:
+			done = true
+
+		default:
+			return errors.Annotate(err).Reason("could not read directory contents: %(dir)s").
+				D("dir", err).
+				Err()
+		}
+
+		// Shuffle the slice randomly. We do this in case others are also processing
+		// this directory simultaneously.
+		for i := range fileInfos {
+			j := mathrand.Intn(c, i+1)
+			fileInfos[i], fileInfos[j] = fileInfos[j], fileInfos[i]
+		}
+
+		for _, fi := range fileInfos {
+			if strings.HasPrefix(fi.Name(), ".") {
+				// Ignore hidden files.

[iannucci, 2017/02/23 00:54:20]

do we have to worry about them growing unbounded?

[dnj, 2017/02/23 20:38:49]

We shouldn't have to. If someone puts their own stuff under here, then there is
a problem; otherwise, the worst worst case is we leak lock files, which are
empty.

+				continue
+			}
+
+			switch pruned, err := maybePruneFile(c, cfg, fi, pruneThreshold, omit); {
+			case err != nil:
+				allErrs = append(allErrs, errors.Annotate(err).
+					Reason("failed to prune file: %(name)s").
+					D("name", fi.Name()).
+					D("dir", cfg.BaseDir).
+					Err())
+
+			case pruned:
+				totalPruned++
+				if cfg.PruneLimit > 0 && totalPruned >= cfg.PruneLimit {
+					logging.Debugf(c, "Hit prune limit of %d.", cfg.PruneLimit)
+					done = true
+				}
+			}
+		}
+	}
+
+	logging.Debugf(c, "Pruned %d environment(s) with %d error(s)", totalPruned, len(allErrs))

[iannucci, 2017/02/23 00:54:20]

I think this should be info (and look like `Pruned 10 (limit) with ....` in the
case it hit the limit)

[dnj, 2017/02/23 20:38:49]

Done.

+	if len(allErrs) > 0 {
+		return allErrs
+	}
+	return nil
+}
+
+// maybePruneFile examines the specified FileIfo within cfg.BaseDir and
+// determines if it should be pruned.
+func maybePruneFile(c context.Context, cfg *Config, fi os.FileInfo, pruneThreshold time.Time, omit string) (
+	pruned bool, err error) {
+
+	if !fi.IsDir() || fi.Name() == omit {
+		return

[iannucci, 2017/02/23 00:54:21]

log?

[dnj, 2017/02/23 20:38:49]

Done.

+	}
+
+	// Grab the lock file for this directory.
+	e := cfg.envForName(fi.Name())
+	err = fslock.With(e.lockPath, func() error {
+		// Read the complete flag file's timestamp.
+		switch st, err := os.Stat(e.completeFlagPath); {
+		case err == nil:
+			if !st.ModTime().Before(pruneThreshold) {
+				return nil
+			}
+
+			logging.Infof(c, "Env [%s] is older than the prune threshold (%v < %v); pruning...",
+				e.name, st.ModTime(), pruneThreshold)
+
+		case os.IsNotExist(err):
+			logging.Infof(c, "Env [%s] has no completed flag; pruning...", e.name)
+
+		default:
+			return errors.Annotate(err).Reason("failed to stat complete flag: %(path)s").
+				D("path", e.completeFlagPath).
+				Err()
+		}
+
+		// Delete the environment. We currently hold its lock, so use deleteLocked.
+		if err := e.deleteLocked(c); err != nil {

[iannucci, 2017/02/23 00:54:20]

does it make sense to mv this to a 'prune' folder first, and have all prune
processes opportunistically try to delete the contents of the prune folder?

[dnj, 2017/02/23 20:38:49]

There may be a race here with moving. ATM deletion of a directory is
synchronized off of ownership of its lock file. That should be sufficient?

[iannucci, 2017/03/11 00:37:34]

I'm thinking about the case where it /partially/ prunes the file (e.g. on
windows) because there's an open file handle or something. If it moves to a
trash folder and everything tries to delete the trash folder, then we won't fail
part-way while holding the lock and end up with a half-destroyed virtualenv.

+			return errors.Annotate(err).Reason("failed to delete Env").Err()
+		}
+		pruned = true
+		return nil
+	})
+	switch err {
+	case nil:
+		return
+
+	case fslock.ErrLockHeld:
+		// Something else currently holds the lock for this directory, so ignore it.
+		logging.Warningf(c, "Lock [%s] is currently held; skipping.", e.lockPath)
+		return
+
+	default:
+		err = errors.Annotate(err).Err()
+		return
+	}
+}