vpython: Add environment spec package.

vpython/spec/load.go

Index: vpython/spec/load.go
diff --git a/vpython/spec/load.go b/vpython/spec/load.go
new file mode 100644
index 0000000000000000000000000000000000000000..618f856485e15fc7586e0f56189cf5f36ba0f2dd
--- /dev/null
+++ b/vpython/spec/load.go
@@ -0,0 +1,306 @@
+// Copyright 2017 The LUCI Authors. All rights reserved.
+// Use of this source code is governed under the Apache License, Version 2.0
+// that can be found in the LICENSE file.
+
+package spec
+
+import (
+	"bufio"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/luci/luci-go/vpython/api/env"
+
+	"github.com/luci/luci-go/common/errors"
+	"github.com/luci/luci-go/common/logging"
+	cproto "github.com/luci/luci-go/common/proto"
+
+	"github.com/golang/protobuf/proto"
+	"golang.org/x/net/context"
+)
+
+// Suffix is the filesystem suffix for a script's partner specification file.
+//
+// See LoadForScript for more information.
+const Suffix = ".vpython"
+
+// Load loads an environment specification file text protobuf from the supplied

[iannucci, 2017/02/21 09:09:55]

let's be careful not to overload 'environment' here, since we also use
environment variables. Maybe just 'vpython spec file'?

[dnj, 2017/02/22 07:37:19]

Done.

+// path.
+func Load(path string) (*env.Spec, error) {

[iannucci, 2017/02/21 09:09:55]

I don't suppose we can s/env/vpython?

[dnj, 2017/02/22 07:37:19]

hrm sure, done.

+	content, err := ioutil.ReadFile(path)
+	if err != nil {
+		return nil, errors.Annotate(err).Reason("failed to load file from: %(path)s").
+			D("path", path).
+			Err()
+	}
+
+	spec, err := Parse(string(content))
+	if err != nil {
+		return nil, errors.Annotate(err).Err()
+	}
+	return spec, nil
+}
+
+// Parse loads a specification message from a content string.
+func Parse(content string) (*env.Spec, error) {
+	var spec env.Spec
+	if err := cproto.UnmarshalTextML(content, &spec); err != nil {
+		return nil, errors.Annotate(err).Reason("failed to unmarshal env.Spec").Err()
+	}
+	return &spec, nil
+}
+
+// Write writes a text protobuf form of spec to path.
+func Write(spec *env.Spec, path string) error {
+	fd, err := os.Create(path)
+	if err != nil {
+		return errors.Annotate(err).Reason("failed to create output file").Err()
+	}
+
+	if err := proto.MarshalText(fd, spec); err != nil {
+		_ = fd.Close()
+		return errors.Annotate(err).Reason("failed to output text protobuf").Err()
+	}
+
+	if err := fd.Close(); err != nil {
+		return errors.Annotate(err).Reason("failed to Close file").Err()
+	}
+	return nil
+}
+
+// LoadForScript attempts to load a spec file for the specified script. If
+// nothing went wrong, a nil error will be returned. If a spec file was
+// identified, it will also be returned. Otherwise, a nil spec will be returned.
+//
+// Spec files can be specified in a variety of ways. This function will look for
+// them in the following order, and return the first one that was identified:
+//
+//	- Partner File
+//	- Inline
+//
+// Partner File
+// ============
+//
+// LoadForScript traverses the filesystem to find the environment
+// specification file that is naturally associated with the specified
+// path.
+//
+// If the path is a Python script (e.g, "/path/to/test.py"), isModule will be
+// false, and the file will be found at "/path/to/test.py.vpython".
+//
+// If the path is a Python module (isModule is true), FindEnvSpecForScript walks
+// upwards in the directory structure, looking for a file that shares a module
+// directory name and ends with ".vpython". For example, for module:
+//
+//	/path/to/foo/bar/baz/__init__.py
+//	/path/to/foo/bar/__init__.py
+//	/path/to/foo/__init__.py
+//	/path/to/foo.vpython
+//
+// LoadForScript will first look at "/path/to/foo/bar/baz", then walk upwards
+// until it either hits a directory that doesn't contain an "__init__.py" file,
+// or finds the ES path. In this case, for module "foo.bar.baz", it will
+// identify "/path/to/foo.vpython" as the ES file for that module.
+//
+// Inline
+// ======
+//
+// LoadForScript scans through the contents of the file at path and attempts to
+// load environment specification boundaries.
+//
+// If the file at path does not exist, or if the file does not contain spec
+// guards, a nil spec will be returned.
+//
+// The embedded enviornment specification is a text protobuf embedded within
+// the file. To parse it, the file is scanned line-by-line for a beginning and
+// ending guard. The content between those guards is minimally processed, then
+// interpreted as a text protobuf.
+//
+//	[VPYTHON:BEGIN]
+//	wheel {
+//	  path: ...
+//	  version: ...
+//	}
+//	[VPYTHON:END]
+//
+// To allow VPYTHON directives to be embedded in a language-compatible manner
+// (with indentation, comments, etc.), the processor will identify any common
+// characters preceding the BEGIN and END clauses. If they match, those
+// characters will be automatically stripped out of the intermediate lines. This
+// can be used to embed the directives in comments:
+//
+//	// [VPYTHON:BEGIN]
+//	// wheel {
+//	//   path: ...
+//	//   version: ...
+//	// }
+//	// [VPYTHON:END]
+//
+// In this case, the "// " characters will be removed.
+func LoadForScript(c context.Context, path string, isModule bool) (*env.Spec, error) {
+	// Partner File: Try loading the spec from an adjacent file.
+	specPath, err := findForScript(path, isModule)
+	if err != nil {
+		return nil, errors.Annotate(err).Reason("failed to scan for filesystem spec").Err()
+	}
+	if specPath != "" {
+		switch sp, err := Load(specPath); {
+		case err != nil:
+			return nil, errors.Annotate(err).Reason("failed to load specification file").
+				D("specPath", specPath).
+				Err()
+
+		case sp != nil:
+			logging.Infof(c, "Loaded environment spec from: %s", specPath)
+			return sp, nil
+		}
+	}
+
+	// Inline: Try and parse the main script for the spec file.
+	mainScript := path
+	if isModule {
+		// Module.
+		mainScript = filepath.Join(mainScript, "__main__.py")
+	}
+	switch sp, err := parseFrom(mainScript); {
+	case err != nil:
+		return nil, errors.Annotate(err).Reason("failed to parse inline spec from: %(script)s").
+			D("script", mainScript).
+			Err()
+
+	case sp != nil:
+		logging.Infof(c, "Loaded inline spec from: %s", mainScript)
+		return sp, nil
+	}
+
+	// No spec file found.
+	return nil, nil
+}
+
+func findForScript(path string, isModule bool) (string, error) {
+	// Otherwise, try and find the ES file associated with this script.

[iannucci, 2017/02/21 09:09:54]

otherwise?

[dnj, 2017/02/22 07:37:19]

Done.

+	if !isModule {
+		path += Suffix
+		if st, err := os.Stat(path); err != nil || st.IsDir() {
+			// File does not exist at this path.
+			return "", nil
+		}
+		return path, nil
+	}
+
+	// If it's a directory, scan for an ".es" file until we don't have a

[iannucci, 2017/02/21 09:09:54]

.es file?

[dnj, 2017/02/22 07:37:19]

Done.

+	// __init__.py.
+	for {
+		prev := path
+
+		// Directory must be a Python module.
+		initPath := filepath.Join(path, "__init__.py")
+		if _, err := os.Stat(initPath); err != nil {
+			if os.IsNotExist(err) {
+				// Not a Python module, so we're done our search.
+				return "", nil
+			}
+			return "", errors.Annotate(err).Reason("failed to stat for: %(path)").
+				D("path", initPath).
+				Err()
+		}
+
+		// Does a spec file exist for this path?
+		specPath := path + Suffix
+		switch _, err := os.Stat(specPath); {
+		case err == nil:
+			// Found the file.
+			return specPath, nil
+
+		case os.IsNotExist(err):
+			// Recurse to parent.
+			path = filepath.Dir(path)
+			if path == prev {
+				// Finished recursing, no ES file.
+				return "", nil
+			}
+
+		default:
+			return "", errors.Annotate(err).Reason("failed to check for spec file at: %(path)s").
+				D("path", specPath).
+				Err()
+		}
+	}
+}
+
+func parseFrom(path string) (*env.Spec, error) {
+	const (
+		beginGuard = "[VPYTHON:BEGIN]"
+		endGuard   = "[VPYTHON:END]"
+	)
+
+	fd, err := os.Open(path)
+	if err != nil {
+		return nil, errors.Annotate(err).Reason("failed to open file").Err()
+	}
+	defer fd.Close()
+
+	s := bufio.NewScanner(fd)
+	var (
+		content   []string
+		beginLine string
+		endLine   string
+		inRegion  = false
+	)
+	for s.Scan() {
+		line := strings.TrimSpace(s.Text())
+		if !inRegion {
+			inRegion = strings.HasSuffix(line, beginGuard)
+			beginLine = line
+		} else {
+			if strings.HasSuffix(line, endGuard) {
+				// Finished processing.
+				endLine = line
+				break
+			}
+			content = append(content, line)
+		}
+	}
+	if err := s.Err(); err != nil {
+		return nil, errors.Annotate(err).Reason("error scanning file").Err()
+	}
+	if len(content) == 0 {
+		return nil, nil
+	}
+	if endLine == "" {
+		return nil, errors.New("unterminated inline spec file")
+	}
+
+	// If we have a common begin/end prefix, trim it from each content line that
+	// also has it.
+	prefix := beginLine[:len(beginLine)-len(beginGuard)]
+	if endLine[:len(endLine)-len(endGuard)] != prefix {
+		prefix = ""

[iannucci, 2017/02/21 09:09:54]

should this just be an error?

[dnj, 2017/02/22 07:37:19]

I don't think so. Let's be a little flexible for now.

+	}
+	if prefix != "" {
+		for i, line := range content {
+			if len(line) < len(prefix) {
+				// This line is shorter than the prefix. Does the part of that line that
+				// exists match the prefix up until that point?
+				if line == prefix[:len(line)] {
+					// Yes, so empty line.
+					line = ""
+				}

[iannucci, 2017/02/21 09:09:54]

what happens in the else case? doesn't matter?

[dnj, 2017/02/22 07:37:19]

Yeah, if there is no shorter prefix to trim, we leave the line as-is.

+			} else {
+				line = strings.TrimPrefix(line, prefix)
+			}
+			content[i] = line
+		}
+	}
+
+	// Process the resulting file.
+	spec, err := Parse(strings.Join(content, "\n"))
+	if err != nil {
+		return nil, errors.Annotate(err).Reason("failed to parse spec file from: %(path)s").
+			D("path", path).
+			Err()
+	}
+	return spec, nil
+}