vpython: Add VirtualEnv creation package.

vpython/venv/config.go

+// Copyright 2017 The LUCI Authors. All rights reserved.
+// Use of this source code is governed under the Apache License, Version 2.0
+// that can be found in the LICENSE file.
+
+package venv
+
+import (
+        "fmt"
+        "os"
+        "path/filepath"
+        "time"
+
+        "github.com/luci/luci-go/vpython/api/env"
+        "github.com/luci/luci-go/vpython/filesystem"
+        "github.com/luci/luci-go/vpython/python"
+        "github.com/luci/luci-go/vpython/spec"
+
+        "github.com/luci/luci-go/common/errors"
+        "github.com/luci/luci-go/common/logging"
+
+        "golang.org/x/net/context"
+)
+
+// PackageLoader loads package information from a specification file's Package
+// message onto the local system.
+type PackageLoader interface {
+        // Resolve processes the supplied packages, updating their fields to their
+        // resolved values. Resolved packages must fully specify the package instance
+        // that is being deployed.
+        //
+        // If needed, resolution may use the supplied root path as a persistent
+        // working directory. This path may not exist; Resolve is responsible for
+        // creating it if needed.
+        //
+        // root, if used, must be safe for concurrent use.
+        Resolve(c context.Context, root string, packages []*env.Spec_Package) error
+
+        // Ensure installs the supplied packages into root.
+        //
+        // The packages will have been previously resolved via Resolve.
+        Ensure(c context.Context, root string, packages []*env.Spec_Package) error
+}
+
+// Config is the configuration for a Env.
+//
+// A VirtualEnv is specified based on its resolved env.Spec.
+type Config struct {
+        // MaxHashLen is the maximum number of hash characters to use in VirtualEnv
+        // directory names.
+        MaxHashLen int
+
+        // BaseDir is the parent directory of all VirtualEnv.
+        BaseDir string
+
+        // Package is the VirtualEnv package to install. It must be non-nil and
+        // valid. It will be used if the environment specification doesn't supply an
+        // overriding one.
+        Package env.Spec_Package
+
+        // Python is the Python interpreter to use. If empty, one will e resolved

[iannucci, 2017/02/23 00:54:20]

be

[dnj, 2017/02/23 20:38:49]

Done.

+        // based on the Spec and the system path.

[iannucci, 2017/02/23 00:54:20]

PATH?

[dnj, 2017/02/23 20:38:49]

Done.

+        Python string
+
+        // Spec is the specification file to use to construct the VirtualEnv. If
+        // nil, or if fields are missing, they will be filled in by probing the system
+        // PATH.
+        Spec *env.Spec
+
+        // PruneThreshold, if >0, is the maximum age of a VirtualEnv before it should
+        // be pruned. If <= 0, there is no maximum age, so no pruning will be
+        // performed.
+        PruneThreshold time.Duration
+        // PruneLimit applies a limit to the number of items to prune per execution.
+        // If <= 0, no limit will be applied.
+        PruneLimit int

[iannucci, 2017/02/23 00:54:20]

unlimited seems like dangerous defaults here. wdyt about picking some reasonable
thresholds if these are unspecified? (and having -1 really be unlimited)

Also, I initially thought that PruneLimit meant 'maximum number of virtualenvs
to allow at any time'. Maybe "MaxPrunesPerSweep" or something?

[dnj, 2017/02/23 20:38:49]

Renamed to MaxPrunesPerSweep.

TBH I think unlimited is fine. This is a documented parameter, and our Infra
binary will definitely set it.

+
+        // Loader is the PackageLoader instance to use for package resolution and
+        // deployment.
+        Loader PackageLoader
+
+        // LoaderResolveRoot is the common persistent root directory to use for the
+        // package loader's package resolution. This must be safe for concurrent
+        // use.
+        //
+        // Each VirtualEnv will have its own loader destination directory (root).
+        // However, that root is homed in a directory that is named after the resolved
+        // packages. LoaderResolveRoot is used during setup to resolve those package
+        // values, and therefore can't re-use the VirtualEnv root.
+        LoaderResolveRoot string
+}
+
+// Env processes the config, validating and, where appropriate, populating
+// any components. Upon success, it returns a configured Env instance.
+//
+// The returned Env instance may or may octually exist. Setup must be called

[iannucci, 2017/02/23 00:54:20]

may not actually

[dnj, 2017/02/23 20:38:49]

Done.

+// prior to using it.
+func (cfg *Config) Env(c context.Context) (*Env, error) {
+        // We MUST have a package loader.
+        if cfg.Loader == nil {
+                return nil, errors.New("no package loader provided")
+        }
+
+        // Resolve our base directory, if one is not supplied.
+        if cfg.BaseDir == "" {
+                // Use one underneath our working directory.

[iannucci, 2017/02/23 00:54:20]

comment is wrong

[dnj, 2017/02/23 20:38:49]

Done.

+                cfg.BaseDir = filepath.Join(os.TempDir(), "vpython")

[iannucci, 2017/02/23 00:54:20]

wdyt about warning if BaseDir is longer than some threshold? e.g. warn if one of
the generated shebang lines (path to python in venv) would be greater than 127
on linux or OS X, and warn if the deepest file in the venv (just assume the
longest path in the virtualenv wheel) would have a path of >260 (or whatever it
is) for windows. 

In fact, it might be best to just exit early. Better to crash here than
experience weirdness later.

[dnj, 2017/02/23 20:38:49]

I'll add an option, "MaxGeneratedPathLen". If >0, it will enforce this based on
observed longest path.

+                logging.Debugf(c, "Using tempdir-relative environment root: %s", cfg.BaseDir)
+        }
+        if err := filesystem.AbsPath(&cfg.BaseDir); err != nil {
+                return nil, errors.Annotate(err).Reason("failed to resolve absolute path of base directory").Err()
+        }
+        if err := filesystem.MakeDirs(cfg.BaseDir); err != nil {
+                return nil, errors.Annotate(err).Reason("could not create environment root: %(root)s").
+                        D("root", cfg.BaseDir).
+                        Err()
+        }
+
+        // Determine our common loader root.
+        if cfg.LoaderResolveRoot == "" {
+                cfg.LoaderResolveRoot = filepath.Join(cfg.BaseDir, ".package_loader")
+        }
+
+        // Ensure and normalize our specification file.
+        if cfg.Spec == nil {
+                cfg.Spec = &env.Spec{}
+        }
+        if err := spec.Normalize(cfg.Spec); err != nil {
+                return nil, errors.Annotate(err).Reason("invalid specification").Err()
+        }
+
+        // Choose our VirtualEnv package.
+        if cfg.Spec.Virtualenv == nil {
+                cfg.Spec.Virtualenv = &cfg.Package
+        }
+
+        if err := cfg.resolvePackages(c); err != nil {
+                return nil, errors.Annotate(err).Reason("failed to resolve packages").Err()
+        }
+
+        if err := cfg.resolvePythonInterpreter(c); err != nil {
+                return nil, errors.Annotate(err).Reason("failed to resolve system Python interpreter").Err()
+        }
+
+        // Generate our enviornment name based on the deterministic hash of its
+        // fully-resolved specification.
+        envName := spec.Hash(cfg.Spec)
+        if cfg.MaxHashLen > 0 && len(envName) > cfg.MaxHashLen {
+                envName = envName[:cfg.MaxHashLen]
+        }
+        return cfg.envForName(envName), nil
+}
+
+// Prune performs a pruning round on the environment set described by this
+// Config.
+func (cfg *Config) Prune(c context.Context) error {
+        if err := prune(c, cfg, ""); err != nil {
+                return errors.Annotate(err).Err()
+        }
+        return nil
+}
+
+// envForExisting creates an Env for a named directory.
+func (cfg *Config) envForName(name string) *Env {

[iannucci, 2017/02/23 00:54:20]

is it worth checking collisions?

[dnj, 2017/02/23 20:38:49]

I don't think so. All operations will lock, so this should be okay even if
someone does it.

+        // Env-specific root directory: <BaseDir>/<name>
+        venvRoot := filepath.Join(cfg.BaseDir, name)
+        return &Env{
+                Config:   cfg,
+                Root:     venvRoot,
+                Python:   venvBinPath(venvRoot, "python"),
+                SpecPath: filepath.Join(venvRoot, "enviornment.pb.txt"),
+
+                name:             name,
+                lockPath:         filepath.Join(cfg.BaseDir, fmt.Sprintf(".%s.lock", name)),
+                completeFlagPath: filepath.Join(venvRoot, "complete.flag"),
+        }
+}
+
+func (cfg *Config) resolvePackages(c context.Context) error {
+        // Create a single package list. Our VirtualEnv will be index 0 (need
+        // this so we can back-port it into its VirtualEnv property).
+        packages := make([]*env.Spec_Package, 1, 1+len(cfg.Spec.Wheel))
+        packages[0] = cfg.Spec.Virtualenv
+        packages = append(packages, cfg.Spec.Wheel...)
+
+        // Resolve our packages. Because we're using pointers, the in-place
+        // updating will update the actual spec file!
+        if err := cfg.Loader.Resolve(c, cfg.LoaderResolveRoot, packages); err != nil {

[iannucci, 2017/02/23 00:54:20]

why do we need separate resolve and ensure phases? ensure (from cipd) can return
all the resolved packages.

edit: is this so we can resolve and pick a directory name first?

[dnj, 2017/02/23 20:38:49]

Yep.

+                return errors.Annotate(err).Reason("failed to resolve packages").Err()
+        }
+        return nil
+}
+
+func (cfg *Config) resolvePythonInterpreter(c context.Context) error {
+        specVers, err := python.ParseVersion(cfg.Spec.PythonVersion)
+        if err != nil {
+                return errors.Annotate(err).Reason("failed to parse Python version from: %(value)q").
+                        D("value", cfg.Spec.PythonVersion).
+                        Err()
+        }
+
+        var i *python.Interpreter
+        if cfg.Python == "" {
+                // No explicitly-specified Python path. Determine one based on the
+                // specification.
+                if i, err = python.Find(c, specVers); err != nil {
+                        return errors.Annotate(err).Reason("could not find Python for: %(vers)s").
+                                D("vers", specVers).
+                                Err()
+                }
+                cfg.Python = i.Python
+        } else {
+                i = &python.Interpreter{
+                        Python: cfg.Python,
+                }
+        }
+
+        // Confirm that the version of the interpreter matches that which is
+        // expected.
+        interpreterVers, err := i.GetVersion(c)
+        if err != nil {
+                return errors.Annotate(err).Reason("failed to determine Python version for: %(python)s").
+                        D("python", cfg.Python).
+                        Err()
+        }
+        if !specVers.IsSatisfiedBy(interpreterVers) {
+                return errors.Reason("supplied Python version (%(supplied)s) doesn't match specification (%(spec)s)").
+                        D("supplied", interpreterVers).
+                        D("spec", specVers).
+                        Err()
+        }
+        cfg.Spec.PythonVersion = interpreterVers.String()
+
+        // Resolve to absolute path.
+        if err := filesystem.AbsPath(&cfg.Python); err != nil {
+                return errors.Annotate(err).Reason("could not get absolute path for: %(python)s").
+                        D("python", cfg.Python).
+                        Err()
+        }
+        logging.Debugf(c, "Resolved system Python interpreter (%s): %s", cfg.Spec.PythonVersion, cfg.Python)
+        return nil
+}
+
+func (cfg *Config) systemInterpreter() *python.Command {
+        if cfg.Python == "" {
+                return nil
+        }
+        i := python.Interpreter{
+                Python: cfg.Python,
+        }
+        cmd := i.Command()
+        cmd.Isolated = true
+        return cmd
+}