deploytool: Add README.md, migrate docs to it.

deploytool/cmd/deploy.go

Index: deploytool/cmd/deploy.go
diff --git a/deploytool/cmd/deploy.go b/deploytool/cmd/deploy.go
deleted file mode 100644
index 71a96ef4e8b8e57bebf77737391fdcd8048d0ed9..0000000000000000000000000000000000000000
--- a/deploytool/cmd/deploy.go
+++ /dev/null
@@ -1,469 +0,0 @@
-// Copyright 2016 The LUCI Authors. All rights reserved.
-// Use of this source code is governed under the Apache License, Version 2.0
-// that can be found in the LICENSE file.
-
-package main
-
-import (
-	"sort"
-
-	"github.com/luci/luci-go/common/cli"
-	"github.com/luci/luci-go/common/errors"
-	"github.com/luci/luci-go/common/flag/flagenum"
-	log "github.com/luci/luci-go/common/logging"
-	"github.com/luci/luci-go/deploytool/managedfs"
-
-	"github.com/maruel/subcommands"
-)
-
-type deployStage int
-
-const (
-	// deployStaging is the staging stage, where local filesystem and build
-	// artifacts are constructed. Build scripts are executed, and generated files
-	// and directories are created. This will leave the user with a directory that
-	// can be used to build from.
-	deployStaging deployStage = iota
-	// deployLocalBuild performs a local build of the components, offering both a
-	// sanity check prior to engaging remote services and creating build artifacts
-	// ready to push remotely.
-	deployLocalBuild deployStage = iota
-	// deployPush pushes local build artifacts to the remote service, but doesn't
-	// enable them.
-	deployPush deployStage = iota
-	// deployCommit commits the pushed artifacts to the remote service, activating
-	// them.
-	deployCommit deployStage = iota
-)
-
-var deployStageFlagEnum = flagenum.Enum{
-	"stage":      deployStaging,
-	"localbuild": deployLocalBuild,
-	"push":       deployPush,
-	"commit":     deployCommit,
-}
-
-func (s *deployStage) Set(v string) error { return deployStageFlagEnum.FlagSet(s, v) }
-func (s deployStage) String() string      { return deployStageFlagEnum.FlagString(s) }
-
-type deployParams struct {
-	// stage is the highest deployment stage to deploy through.
-	stage deployStage
-
-	// ignoreCurrentVersion, if true, instructs depoyment logic to proceed with
-	// deployment even if the currently-deployed version matches the current
-	// source vrsion.
-	ignoreCurrentVersion bool
-	// commitTainted, if true, instructs the deploy tool to perform commit
-	// operations on tainted Components.
-	commitTainted bool
-}
-
-var cmdDeploy = subcommands.Command{
-	UsageLine: "deploy [DEPLOYMENT.[.COMPONENT]]...",
-	ShortDesc: "Deploys some or all of a deployment.",
-	LongDesc:  "Deploys a Deployment. If specific projects are named, only those projects will be deployed.",
-	CommandRun: func() subcommands.CommandRun {
-		var cmd cmdDeployRun
-		cmd.dp.params.stage = deployCommit
-
-		fs := cmd.GetFlags()
-		fs.BoolVar(&cmd.checkout, "checkout", false,
-			"Refresh the checkout prior to deployment.")
-		fs.Var(&cmd.dp.params.stage, "stage",
-			"Run the deployment sequence up through this stage ["+deployStageFlagEnum.Choices()+"].")
-		fs.BoolVar(&cmd.dp.params.commitTainted, "commit-tainted", false,
-			"Install new deployments even if their source is tained.")
-		fs.BoolVar(&cmd.dp.params.ignoreCurrentVersion, "ignore-current-version", false,
-			"Push/install new deployments even if the currently-deployed version would not change or could not be parsed.")
-		return &cmd
-	},
-}
-
-type cmdDeployRun struct {
-	subcommands.CommandRunBase
-
-	// checkout, if true, refreshes the checkout prior to deployment.
-	checkout bool
-
-	dp deploymentPlan
-}
-
-func (cmd *cmdDeployRun) Run(app subcommands.Application, args []string) int {
-	a, c := app.(*application), cli.GetContext(app, cmd)
-
-	err := a.runWork(c, func(w *work) error {
-		// Perform our planned checkout.
-		if cmd.checkout {
-			// Perform a full checkout.
-			if err := checkout(w, &a.layout, false); err != nil {
-				return errors.Annotate(err).Reason("failed to checkout sources").Err()
-			}
-		}
-
-		// Load our frozen checkout.
-		if err := a.layout.loadFrozenLayout(w); err != nil {
-			return errors.Annotate(err).Reason("failed to load frozen checkout").Err()
-		}
-		return nil
-	})
-	if err != nil {
-		logError(c, err, "Failed to perform checkout.")
-		return 1
-	}
-
-	// Figure out what we're going to deploy.
-	for _, arg := range args {
-		matched := false
-		err := a.layout.matchDeploymentComponent(arg, func(dep *layoutDeployment, comp *layoutDeploymentComponent) {
-			matched = true
-
-			if comp != nil {
-				cmd.dp.addProjectComponent(comp)
-			} else {
-				cmd.dp.addDeployment(dep)
-			}
-		})
-		if err != nil {
-			logError(c, err, "Failed during matching.")
-			return 1
-		}
-		if !matched {
-			log.Fields{
-				"target": arg,
-			}.Errorf(c, "Deploy target did not match any configured deployments or components.")
-			return 1
-		}
-	}
-
-	err = a.runWork(c, func(w *work) error {
-		if err := cmd.dp.initialize(w, &a.layout); err != nil {
-			return errors.Annotate(err).Reason("failed to initialize").Err()
-		}
-
-		return cmd.dp.deploy(w)
-	})
-	if err != nil {
-		logError(c, err, "Failed to perform deployment.")
-		return 1
-	}
-	return 0
-}
-
-type deploymentPlan struct {
-	// comps maps DEPLOYMENT/PROJECT/COMPONENT to its specific deployable project
-	// component.
-	comps map[string]*layoutDeploymentComponent
-
-	// deployments tracks which deployments we've specified.
-	deployments map[title]struct{}
-
-	// params are the deployment parameters to forward to deployment operations.
-	params deployParams
-
-	// reg is the registry of deployable Components.
-	reg deploymentRegistry
-
-	// layout is the deployment layout. It is initialized during "initialize".
-	layout *deployLayout
-}
-
-func (dp *deploymentPlan) addProjectComponent(comp *layoutDeploymentComponent) {
-	// Mark this specific component.
-	if dp.comps == nil {
-		dp.comps = make(map[string]*layoutDeploymentComponent)
-	}
-	dp.comps[comp.String()] = comp
-
-	// Mark that we've used this deployment.
-	if dp.deployments == nil {
-		dp.deployments = make(map[title]struct{})
-	}
-	dp.deployments[comp.dep.title] = struct{}{}
-}
-
-func (dp *deploymentPlan) addDeployment(dep *layoutDeployment) {
-	for _, comp := range dep.components {
-		dp.addProjectComponent(comp)
-	}
-}
-
-func (dp *deploymentPlan) deployedComponents() []string {
-	comps := make([]string, 0, len(dp.comps))
-	for path := range dp.comps {
-		comps = append(comps, path)
-	}
-	sort.Strings(comps)
-	return comps
-}
-
-func (dp *deploymentPlan) initialize(w *work, l *deployLayout) error {
-	dp.layout = l
-
-	// Load all deployable Components from checkout.
-	for _, depName := range l.deploymentNames {
-		dep := l.deployments[depName]
-
-		if _, ok := dp.deployments[dep.title]; !ok {
-			continue
-		}
-
-		// Load the application components. We will iterate through them in the
-		// order in which they are defined in the Application's protobuf.
-		for _, compName := range dep.componentNames {
-			comp := dep.components[compName]
-
-			// Only pass the registrar if this component is being deployed.
-			var pReg componentRegistrar
-			if _, ok := dp.comps[comp.String()]; ok {
-				pReg = &dp.reg
-			}
-			if err := comp.loadSourceComponent(pReg); err != nil {
-				return errors.Annotate(err).Reason("failed to load component %(comp)q").
-					D("comp", comp.String()).Err()
-			}
-		}
-		return nil
-	}
-	return nil
-}
-
-func (dp *deploymentPlan) deploy(w *work) error {
-	// Create our working root and staging/build subdirectories.
-	fs, err := dp.layout.workingFilesystem()
-	if err != nil {
-		return errors.Annotate(err).Reason("failed to create working directory").Err()
-	}
-
-	stagingDir, err := fs.Base().EnsureDirectory("staging")
-	if err != nil {
-		return errors.Annotate(err).Reason("failed to create staging directory").Err()
-	}
-
-	// Stage: Staging
-	if err := dp.reg.stage(w, stagingDir, &dp.params); err != nil {
-		return errors.Annotate(err).Reason("failed to stage").Err()
-	}
-	if dp.params.stage <= deployStaging {
-		return nil
-	}
-
-	// Stage: Local Build
-	if err := dp.reg.localBuild(w); err != nil {
-		return errors.Annotate(err).Reason("failed to perform local build").Err()
-	}
-	if dp.params.stage <= deployLocalBuild {
-		return nil
-	}
-
-	// Stage: Push
-	if err := dp.reg.push(w); err != nil {
-		return errors.Annotate(err).Reason("failed to push components").Err()
-	}
-	if dp.params.stage <= deployPush {
-		return nil
-	}
-
-	// Stage: Commit
-	if err := dp.reg.commit(w); err != nil {
-		return errors.Annotate(err).Reason("failed to commit").Err()
-	}
-	return nil
-}
-
-// deploymentRegistry is the registry of all prepared deployments.
-//
-// While deployment is specified and staged at a component level, actual
-// deployment is allowed to happen at a practical level (e.g., a single
-// cloud project's AppEngine configuration, a single Google Container Engine
-// cluster, etc.).
-//
-// This will track individual components, as well as integrate them into larger
-// action plans.
-type deploymentRegistry struct {
-	// components is a map of components that have been deployed.
-	components map[string]*layoutDeploymentComponent
-	// componentNames is the sorted list of component names in components.
-	componentNames []string
-
-	// gaeProjects maps Google AppEngine deployment state to specific GAE
-	// projects.
-	gaeProjects map[string]*gaeDeployment
-	// gaeProjectNames is the sorted list of registered GAE project names.
-	gaeProjectNames []string
-
-	// gkeProjects is the set of Google Container Engine deployments for a given
-	// cloud project.
-	gkeProjects map[string]*containerEngineDeployment
-	// gkeProjectNames is the sorted list of registered GKE project names.
-	gkeProjectNames []string
-}
-
-func (reg *deploymentRegistry) addComponent(comp *layoutDeploymentComponent) {
-	name := comp.comp.Name
-	if _, has := reg.components[name]; has {
-		return
-	}
-
-	if reg.components == nil {
-		reg.components = make(map[string]*layoutDeploymentComponent)
-	}
-	reg.components[name] = comp
-	reg.componentNames = append(reg.componentNames, name)
-}
-
-func (reg *deploymentRegistry) addGAEModule(module *layoutDeploymentGAEModule) {
-	// Get/create the AppEngine project for this module.
-	cloudProjectName := module.comp.dep.cloudProject.Name
-	gaeD := reg.gaeProjects[cloudProjectName]
-	if gaeD == nil {
-		gaeD = makeGAEDeployment(module.comp.dep.cloudProject)
-
-		if reg.gaeProjects == nil {
-			reg.gaeProjects = make(map[string]*gaeDeployment)
-		}
-		reg.gaeProjects[cloudProjectName] = gaeD
-		reg.gaeProjectNames = append(reg.gaeProjectNames, cloudProjectName)
-	}
-
-	gaeD.addModule(module)
-	reg.addComponent(module.comp)
-}
-
-func (reg *deploymentRegistry) addGKEPod(pb *layoutDeploymentGKEPodBinding) {
-	// Get/create the AppEngine project for this module.
-	cloudProjectName := pb.cluster.cloudProject.Name
-	gkeD := reg.gkeProjects[cloudProjectName]
-	if gkeD == nil {
-		gkeD = &containerEngineDeployment{
-			project:  pb.cluster.cloudProject,
-			clusters: make(map[string]*containerEngineDeploymentCluster),
-		}
-
-		if reg.gkeProjects == nil {
-			reg.gkeProjects = make(map[string]*containerEngineDeployment)
-		}
-		reg.gkeProjects[cloudProjectName] = gkeD
-		reg.gkeProjectNames = append(reg.gkeProjectNames, cloudProjectName)
-	}
-
-	// Register this pod in its cluster.
-	gkeD.addCluster(pb.cluster).attachPod(pb)
-}
-
-// appEngineModulesOnly clears all deployment parameters that are not AppEngine
-// modules.
-func (reg *deploymentRegistry) appEngineModulesOnly() {
-	reg.gkeProjects, reg.gkeProjectNames = nil, nil
-}
-
-// stage performs staging, preparing deployment structures on the local
-// filesystem.
-func (reg *deploymentRegistry) stage(w *work, stageDir *managedfs.Dir, params *deployParams) error {
-	// Sort our name lists.
-	sort.Strings(reg.componentNames)
-	sort.Strings(reg.gaeProjectNames)
-	sort.Strings(reg.gkeProjectNames)
-
-	// All components can be independently staged in parallel.
-	//
-	// We will still enumerate over them in sorted order for determinism.
-	return w.RunMulti(func(workC chan<- func() error) {
-		// AppEngine Projects
-		for _, name := range reg.gaeProjectNames {
-			proj := reg.gaeProjects[name]
-			workC <- func() error {
-				depDir, err := stageDir.EnsureDirectory(string(proj.project.dep.title), "appengine")
-				if err != nil {
-					return errors.Annotate(err).Reason("failed to create deployment directory for %(deployment)q").
-						D("deployment", proj.project.dep.title).Err()
-				}
-				return proj.stage(w, depDir, params)
-			}
-		}
-
-		// Container Engine Projects
-		for _, name := range reg.gkeProjectNames {
-			proj := reg.gkeProjects[name]
-			workC <- func() error {
-				depDir, err := stageDir.EnsureDirectory(string(proj.project.dep.title), "container_engine")
-				if err != nil {
-					return errors.Annotate(err).Reason("failed to create deployment directory for %(deployment)q").
-						D("deployment", proj.project.dep.title).Err()
-				}
-				return proj.stage(w, depDir, params)
-			}
-		}
-	})
-}
-
-// build performs the build stage on staged components.
-func (reg *deploymentRegistry) localBuild(w *work) error {
-	// Enumerate over them in sorted order for determinism.
-	return w.RunMulti(func(workC chan<- func() error) {
-		// AppEngine Projects
-		for _, name := range reg.gaeProjectNames {
-			proj := reg.gaeProjects[name]
-			workC <- func() error {
-				return proj.localBuild(w)
-			}
-		}
-
-		// Container Engine Projects
-		for _, name := range reg.gkeProjectNames {
-			proj := reg.gkeProjects[name]
-			workC <- func() error {
-				return proj.localBuild(w)
-			}
-		}
-	})
-}
-
-// push builds and pushes the deployments to remote services, but does not
-// commit to them.
-func (reg *deploymentRegistry) push(w *work) error {
-	// All components can be independently pushed in parallel.
-	//
-	// We will still enumerate over them in sorted order for determinism.
-	return w.RunMulti(func(workC chan<- func() error) {
-		// AppEngine Projects
-		for _, name := range reg.gaeProjectNames {
-			proj := reg.gaeProjects[name]
-			workC <- func() error {
-				return proj.push(w)
-			}
-		}
-
-		// Container Engine Projects
-		for _, name := range reg.gkeProjectNames {
-			proj := reg.gkeProjects[name]
-			workC <- func() error {
-				return proj.push(w)
-			}
-		}
-	})
-}
-
-// commit commits the deployment to the remote services.
-func (reg *deploymentRegistry) commit(w *work) error {
-	// All components can be independently committed in parallel.
-	//
-	// We will still enumerate over them in sorted order for determinism.
-	return w.RunMulti(func(workC chan<- func() error) {
-		// AppEngine Projects
-		for _, name := range reg.gaeProjectNames {
-			proj := reg.gaeProjects[name]
-			workC <- func() error {
-				return proj.commit(w)
-			}
-		}
-
-		// Container Engine Projects
-		for _, name := range reg.gkeProjectNames {
-			proj := reg.gkeProjects[name]
-			workC <- func() error {
-				return proj.commit(w)
-			}
-		}
-	})
-}