cipd: Implement 'init', 'install' and 'installed' subcommands.

go/src/infra/tools/cipd/apps/cipd/friendly.go

Index: go/src/infra/tools/cipd/apps/cipd/friendly.go
diff --git a/go/src/infra/tools/cipd/apps/cipd/friendly.go b/go/src/infra/tools/cipd/apps/cipd/friendly.go
new file mode 100644
index 0000000000000000000000000000000000000000..f710d62ab26258f0755548bafe540dc298bc4285
--- /dev/null
+++ b/go/src/infra/tools/cipd/apps/cipd/friendly.go
@@ -0,0 +1,452 @@
+// Copyright 2015 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package main
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"github.com/luci/luci-go/client/authcli"
+	"github.com/luci/luci-go/common/auth"
+
+	"github.com/maruel/subcommands"
+
+	"infra/tools/cipd"
+	"infra/tools/cipd/local"
+)
+
+// findSiteRoot returns a directory R such as R/.cipd exists and p is inside
+// R or p is R. Returns empty string if no such directory.
+func findSiteRoot(p string) string {

[tandrii(chromium), 2015/09/21 13:47:41]

Good! I coded this for something already...

+	for {
+		if isSiteRoot(p) {
+			return p
+		}
+		parent := filepath.Dir(p)
+		if parent == p {

[nodir, 2015/09/21 22:26:58]

this seems to be undocumented behavior. Works on linux, but I cannot check on
windows

[Vadim Sh., 2015/09/29 01:43:33]

Why check if you have the source code? :)
https://golang.org/src/path/filepath/path.go?s=13086:13114#L440

The documented bit is "The returned path does not end in a separator unless it
is the root directory.", so I'll use it instead, but it is less readable imho.

+			return ""
+		}
+		p = parent
+	}
+}
+
+// optionalSiteRoot takes a path to a site root or an empty string. If some
+// path is given, it normalizes it and ensures that it is indeed a site root
+// directory. If empty string is given, it discovers a site root for current
+// directory.
+func optionalSiteRoot(siteRoot string) (string, error) {
+	if siteRoot == "" {
+		cwd, err := os.Getwd()
+		if err != nil {
+			return "", err
+		}
+		siteRoot = findSiteRoot(cwd)
+		if siteRoot == "" {
+			return "", fmt.Errorf("directory %s is not in a site root, use 'init' to create one", cwd)
+		}
+		return siteRoot, nil
+	}
+	siteRoot, err := filepath.Abs(siteRoot)
+	if err != nil {
+		return "", err
+	}
+	if !isSiteRoot(siteRoot) {
+		return "", fmt.Errorf("directory %s doesn't look like a site root, use 'init' to create one", siteRoot)
+	}
+	return siteRoot, nil
+}
+
+// isSiteRoot returns true of <p>/.cipd exists.
+func isSiteRoot(p string) bool {
+	fi, err := os.Stat(filepath.Join(p, local.SiteServiceDir))
+	return err == nil && fi.IsDir()
+}
+
+// siteConfig is stored in .cipd/config.json.
+type siteConfig struct {
+	// ServiceURL is https://<hostname> of a backend to use by default.
+	ServiceURL string
+	// DefaultVersion is what version to install if not specified.
+	DefaultVersion string
+	// TrackedVersions is mapping package name -> version to use in 'update'.
+	TrackedVersions map[string]string
+}
+
+// read loads JSON from given path.
+func (c *siteConfig) read(path string) error {
+	*c = siteConfig{}
+	r, err := os.Open(path)
+	if err != nil {
+		return err
+	}
+	defer r.Close()
+	return json.NewDecoder(r).Decode(c)
+}
+
+// write dumps JSON to given path.
+func (c *siteConfig) write(path string) error {
+	blob, err := json.MarshalIndent(c, "", "\t")
+	if err != nil {
+		return err
+	}
+	return ioutil.WriteFile(path, blob, 0666)
+}
+
+// readConfig reads config, returning default one if missing.
+func readConfig(siteRoot string) (siteConfig, error) {
+	path := filepath.Join(siteRoot, local.SiteServiceDir, "config.json")
+	c := siteConfig{}
+	if err := c.read(path); err != nil && !os.IsNotExist(err) {
+		return c, err
+	}
+	return c, nil
+}
+
+// modifyConfig reads config file, calls callback to mutate it, then writes
+// it back.
+func modifyConfig(siteRoot string, cb func(cfg *siteConfig) error) error {
+	path := filepath.Join(siteRoot, local.SiteServiceDir, "config.json")
+	c := siteConfig{}
+	if err := c.read(path); err != nil && !os.IsNotExist(err) {
+		return err
+	}
+	if err := cb(&c); err != nil {
+		return err
+	}
+	return c.write(path)
+}
+
+type siteRootCB func(siteRoot string, cfg *siteConfig) error
+
+// doInSiteRoot finds site root directory, reads config and calls callback.
+// Is siteRoot is "", will find a site root based on current directory,

[tandrii(chromium), 2015/09/21 13:47:41]

s/Is/If

[Vadim Sh., 2015/09/29 01:43:33]

Done.

+// otherwise will use siteRoot.
+func doInSiteRoot(siteRoot string, cb siteRootCB) error {

[nodir, 2015/09/21 22:26:58]

I think just returning siteRoot and cfg would be simpler

[Vadim Sh., 2015/09/29 01:43:33]

Done. Not sure why I went callback-warrior road. Probably overexposure to
Javascript.

+	siteRoot, err := optionalSiteRoot(siteRoot)
+	if err != nil {
+		return err
+	}
+	cfg, err := readConfig(siteRoot)
+	if err != nil {
+		return err
+	}
+	return cb(siteRoot, &cfg)
+}
+
+type siteRootWithClientCB func(siteRoot string, cfg *siteConfig, c cipd.Client) error
+
+// doInSiteRootWithClient is like doInSiteRoot, but also initializes
+// cipd.Client object to talk to the backend.
+func doInSiteRootWithClient(siteRoot string, authFlags authcli.Flags, cb siteRootWithClientCB) error {

[nodir, 2015/09/21 22:26:59]

I think translating this to

(siteConfig) func createClient() (cipd.Client, err)

would be simpler

[Vadim Sh., 2015/09/29 01:43:33]

client needs authFlags. Moved into separate initClient() method.

+	return doInSiteRoot(siteRoot, func(siteRoot string, cfg *siteConfig) error {
+		serviceOpts := ServiceOptions{
+			authFlags:  authFlags,
+			serviceURL: cfg.ServiceURL,
+		}
+		client, err := serviceOpts.makeCipdClient(siteRoot)
+		if err != nil {
+			return err
+		}
+		return cb(siteRoot, cfg, client)
+	})
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// 'init' subcommand.
+
+var cmdInit = &subcommands.Command{
+	UsageLine: "init [options]",
+	ShortDesc: "sets up a new site root directory to install packages to",
+	LongDesc:  "Sets up a new site root directory to install packages to.",
+	CommandRun: func() subcommands.CommandRun {
+		c := &initRun{}
+		c.registerBaseFlags()
+		c.Flags.StringVar(&c.rootDir, "root", "", "Path to an installation site root directory.")

[nodir, 2015/09/21 22:26:59]

optional: use "." as default so it is obvious for the user that it defaults to
the current dir, when they run cipd install -help

[Vadim Sh., 2015/09/29 01:43:33]

Done.

+		c.Flags.BoolVar(&c.force, "force", false, "Create even if directory is not empty.")
+		c.Flags.StringVar(&c.serviceURL, "service-url", "", "URL of a backend to use instead of the default one.")
+		return c
+	},
+}
+
+type initRun struct {
+	Subcommand
+
+	rootDir    string
+	force      bool
+	serviceURL string
+}
+
+func (c *initRun) Run(a subcommands.Application, args []string) int {
+	if !c.init(args, 0, 0) {
+		return 1
+	}
+	rootDir, err := initSiteRoot(c.rootDir, c.force)
+	if err != nil {
+		return c.done(nil, err)
+	}
+	err = modifyConfig(rootDir, func(cfg *siteConfig) error {
+		cfg.ServiceURL = c.serviceURL
+		return nil
+	})
+	return c.done(rootDir, err)
+}
+
+func initSiteRoot(rootDir string, force bool) (string, error) {
+	// Default to cwd.
+	var err error
+	if rootDir == "" {
+		rootDir, err = os.Getwd()
+	} else {
+		rootDir, err = filepath.Abs(rootDir)
+	}
+	if err != nil {
+		return "", err
+	}
+
+	// rootDir is inside an existing site root?
+	existing := findSiteRoot(rootDir)
+	if existing != "" {
+		msg := fmt.Sprintf("directory %s is already inside a site root (%s)", rootDir, existing)

[nodir, 2015/09/21 22:26:59]

Is json-output=- only for debugging? Otherwise Fprintf to stderr here and below

[Vadim Sh., 2015/09/29 01:43:33]

Error printing is done in done().

[nodir, 2015/09/30 21:03:05]

Acknowledged.

+		if !force {

[tandrii(chromium), 2015/09/21 13:47:41]

This is extra meaning for force flag then: doc above says just about non-empty
dir, but here it's about creating .cipd inside a tree with .cipd .

[Vadim Sh., 2015/09/29 01:43:33]

Updated flag help string.

+			return "", errors.New(msg)
+		}
+		fmt.Printf("Warning: %s.\n", msg)
+	}
+
+	// Attempting to use in a non empty directory?
+	items, err := ioutil.ReadDir(rootDir)
+	if err != nil && !os.IsNotExist(err) {
+		return "", err
+	}
+	if len(items) != 0 {

[nodir, 2015/09/21 22:26:59]

what if err != nil?

[Vadim Sh., 2015/09/29 01:43:33]

err != nil here only if os.IsNotExists(err) is true, meaning directory doesn't
exist, which is ok. See 'if' two lines above :)

[nodir, 2015/09/30 21:03:05]

Ack. I read them incorrectly

+		msg := fmt.Sprintf("directory %s is not empty", rootDir)
+		if !force {
+			return "", errors.New(msg)
+		}
+		fmt.Printf("Warning: %s.\n", msg)
+	}
+
+	// Good to go.
+	if err = os.MkdirAll(filepath.Join(rootDir, local.SiteServiceDir), 0777); err != nil {
+		return "", err
+	}
+	fmt.Printf("Site root initialized at %s.\n", rootDir)
+	return rootDir, nil
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// 'install' subcommand.
+
+var cmdInstall = &subcommands.Command{
+	UsageLine: "install <package> [<version>] [options]",
+	ShortDesc: "installs or updates a package",
+	LongDesc:  "Installs or updates a package.",
+	CommandRun: func() subcommands.CommandRun {
+		c := &installRun{}
+		c.registerBaseFlags()
+		c.authFlags.Register(&c.Flags, auth.Options{})
+		c.Flags.StringVar(&c.rootDir, "root", "", "Path to an installation site root directory.")
+		c.Flags.BoolVar(&c.force, "force", false, "Refetch and reinstall even if already installed.")
+		return c
+	},
+}
+
+type installRun struct {
+	Subcommand
+	authFlags authcli.Flags
+
+	rootDir string
+	force   bool
+}
+
+func (c *installRun) Run(a subcommands.Application, args []string) int {
+	if !c.init(args, 1, 2) {
+		return 1
+	}
+
+	// Pkg and version to install.
+	pkgName := args[0]
+	version := ""
+	if len(args) == 2 {
+		version = args[1]
+	}
+
+	// Auto initialize site root directory if necessary. Don't be too aggressive
+	// about it though (do not use force=true). Will do anything only if
+	// c.rootDir points to an empty directory.
+	rootDir, err := optionalSiteRoot(c.rootDir)
+	if err != nil {
+		rootDir, err = initSiteRoot(c.rootDir, false)
+		if err != nil {
+			err = fmt.Errorf("can't auto initialize cipd site root (%s), use 'init'", err)
+			return c.done(nil, err)
+		}
+	}
+
+	var output pinOrError
+	err = doInSiteRootWithClient(rootDir, c.authFlags,
+		func(siteRoot string, cfg *siteConfig, client cipd.Client) error {
+			output = installOrUpdate(pkgName, version, siteRoot, c.force, cfg, client)
+			return nil
+		})
+	return c.doneWithPins([]pinOrError{output}, err)
+}
+
+func installOrUpdate(pkgName, version, siteRoot string, force bool, cfg *siteConfig, client cipd.Client) pinOrError {
+	// Figure out what exactly to install.
+	if version == "" {
+		version = cfg.DefaultVersion
+	}
+	if version == "" {
+		version = "latest"
+	}
+	pin, err := client.ResolveVersion(pkgName, version)

[nodir, 2015/09/21 22:26:58]

could use cfg.makeCipdClient to get client

[Vadim Sh., 2015/09/29 01:43:33]

did something else

+	if err != nil {
+		return pinOrError{
+			Pkg: pkgName,
+			Err: err.Error(),
+		}
+	}
+
+	// Already installed?
+	skipInstall := false

[nodir, 2015/09/21 22:26:59]

I'd prefer positive name: forceInstall := true

[Vadim Sh., 2015/09/29 01:43:33]

Done.

+	if !force {
+		d := local.NewDeployer(siteRoot, log)
+		existing, err := d.CheckDeployed(pkgName)
+		if err == nil && existing == pin {
+			fmt.Printf("Package %s is up-to-date.\n", pkgName)
+			skipInstall = true
+		}
+	}
+
+	// Go for it.
+	if !skipInstall {

[nodir, 2015/09/21 22:26:58]

negative on negative

[Vadim Sh., 2015/09/29 01:43:33]

Done.

+		fmt.Printf("Installing %s (version %q)...\n", pkgName, version)
+		if err = client.FetchAndDeployInstance(pin); err != nil {
+			return pinOrError{
+				Pkg: pkgName,
+				Err: err.Error(),
+			}
+		}
+	}
+
+	// Update config saying what version to track. Remove tracking if an exact
+	// instance ID was requested.
+	trackedVersion := ""
+	if version != pin.InstanceID {
+		trackedVersion = version
+	}
+	err = modifyConfig(siteRoot, func(cfg *siteConfig) error {
+		if cfg.TrackedVersions == nil {
+			cfg.TrackedVersions = map[string]string{}
+		}
+		if cfg.TrackedVersions[pkgName] != trackedVersion {
+			if trackedVersion == "" {
+				fmt.Printf("Package %s is now pinned to %q.\n", pkgName, pin.InstanceID)
+			} else {
+				fmt.Printf("Package %s is now tracking %q.\n", pkgName, trackedVersion)
+			}
+		}
+		if trackedVersion == "" {
+			delete(cfg.TrackedVersions, pkgName)
+		} else {
+			cfg.TrackedVersions[pkgName] = trackedVersion
+		}
+		return nil
+	})
+	if err != nil {
+		return pinOrError{
+			Pkg: pkgName,
+			Err: err.Error(),
+		}
+	}
+
+	// Success.
+	return pinOrError{
+		Pkg:      pkgName,
+		Pin:      &pin,
+		Tracking: trackedVersion,
+	}
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// 'installed' subcommand.
+
+var cmdInstalled = &subcommands.Command{
+	UsageLine: "installed [<package> <package> ...] [options]",
+	ShortDesc: "lists packages installed in the site root",
+	LongDesc:  "Lists packages installed in the site root.",
+	CommandRun: func() subcommands.CommandRun {
+		c := &installedRun{}
+		c.registerBaseFlags()
+		c.Flags.StringVar(&c.rootDir, "root", "", "Path to an installation site root directory.")
+		return c
+	},
+}
+
+type installedRun struct {
+	Subcommand
+
+	rootDir string
+}
+
+func (c *installedRun) Run(a subcommands.Application, args []string) int {
+	if !c.init(args, 0, -1) {
+		return 1
+	}
+	var output []pinOrError
+	err := doInSiteRoot(c.rootDir, func(siteRoot string, cfg *siteConfig) error {
+		var err error
+		output, err = findInstalled(args, siteRoot, cfg)
+		return err
+	})
+	return c.doneWithPins(output, err)
+}
+
+func findInstalled(pkgs []string, siteRoot string, cfg *siteConfig) ([]pinOrError, error) {
+	d := local.NewDeployer(siteRoot, log)
+
+	// List all?
+	if len(pkgs) == 0 {
+		pins, err := d.FindDeployed()
+		if err != nil {
+			return nil, err
+		}
+		output := make([]pinOrError, len(pins))
+		for i, pin := range pins {
+			cpy := pin
+			output[i] = pinOrError{
+				Pkg:      pin.PackageName,
+				Pin:      &cpy,
+				Tracking: cfg.TrackedVersions[pin.PackageName],
+			}
+		}
+		return output, nil
+	}
+
+	// List specific packages only.
+	output := make([]pinOrError, len(pkgs))
+	for i, pkgName := range pkgs {
+		pin, err := d.CheckDeployed(pkgName)
+		if err == nil {
+			output[i] = pinOrError{
+				Pkg:      pkgName,
+				Pin:      &pin,
+				Tracking: cfg.TrackedVersions[pkgName],
+			}
+		} else {
+			output[i] = pinOrError{
+				Pkg:      pkgName,
+				Err:      err.Error(),
+				Tracking: cfg.TrackedVersions[pkgName],
+			}
+		}
+	}
+	return output, nil
+}