luci_deploy: Enable loading "index.yaml" directly.

deploytool/cmd/luci_deploy/appengine.go

Index: deploytool/cmd/luci_deploy/appengine.go
diff --git a/deploytool/cmd/luci_deploy/appengine.go b/deploytool/cmd/luci_deploy/appengine.go
new file mode 100644
index 0000000000000000000000000000000000000000..1356aaf2c6a7976262ea8c9db66262d650d276a5
--- /dev/null
+++ b/deploytool/cmd/luci_deploy/appengine.go
@@ -0,0 +1,337 @@
+// Copyright 2016 The LUCI Authors. All rights reserved.
+// Use of this source code is governed under the Apache License, Version 2.0
+// that can be found in the LICENSE file.
+
+package main
+
+import (
+	"io/ioutil"
+	"path/filepath"
+	"strings"
+
+	"github.com/luci/luci-go/common/errors"
+	"github.com/luci/luci-go/deploytool/api/deploy"
+	"gopkg.in/yaml.v2"
+)
+
+// gaeAppYAML is a YAML struct for an AppEngine "app.yaml".
+type gaeAppYAML struct {
+	Module     string      `yaml:"module,omitempty"`
+	Runtime    string      `yaml:"runtime,omitempty"`
+	ThreadSafe *bool       `yaml:"threadsafe,omitempty"`
+	APIVersion interface{} `yaml:"api_version,omitempty"`
+	VM         bool        `yaml:"vm,omitempty"`
+
+	BetaSettings *gaeAppYAMLBetaSettings `yaml:"beta_settings,omitempty"`
+
+	Handlers []*gaeAppYAMLHandler `yaml:"handlers,omitempty"`
+}
+
+// gaeAppYAMLBetaSettings is a YAML struct for an AppEngine "app.yaml"
+// beta settings.
+type gaeAppYAMLBetaSettings struct {
+	ServiceAccountScopes string `yaml:"service_account_scopes,omitempty"`
+}
+
+// gaeAppYAMLHAndler is a YAML struct for an AppEngine "app.yaml"
+// handler entry.
+type gaeAppYAMLHandler struct {
+	URL    string `yaml:"url"`
+	Script string `yaml:"script,omitempty"`
+
+	Secure string `yaml:"secure,omitempty"`
+	Login  string `yaml:"login,omitempty"`
+
+	StaticDir   string `yaml:"static_dir,omitempty"`
+	StaticFiles string `yaml:"static_files,omitempty"`
+	Upload      string `yaml:"upload,omitempty"`
+}
+
+// gaeCronYAML is a YAML struct for an AppEngine "cron.yaml".
+type gaeCronYAML struct {
+	Cron []*gaeCronYAMLEntry `yaml:"cron,omitempty"`
+}
+
+// gaeCronYAMLEntry is a YAML struct for an AppEngine "cron.yaml" entry.
+type gaeCronYAMLEntry struct {
+	Description string `yaml:"description,omitempty"`
+	URL         string `yaml:"url"`
+	Schedule    string `yaml:"schedule"`
+	Target      string `yaml:"target,omitempty"`
+}
+
+// gaeIndexYAML is a YAML struct for an AppEngine "index.yaml".
+type gaeIndexYAML struct {
+	Indexes []*gaeIndexYAMLEntry `yaml:"indexes,omitempty"`
+}
+
+// gaeIndexYAMLEntry is a YAML struct for an AppEngine "index.yaml" entry.
+type gaeIndexYAMLEntry struct {
+	Kind       string                       `yaml:"kind"`
+	Ancestor   string                       `yaml:"ancestor,omitempty"`
+	Properties []*gaeIndexYAMLEntryProperty `yaml:"properties"`
+}
+
+// gaeIndexYAMLEntryProperty is a YAML struct for an AppEngine "index.yaml"
+// entry's property.
+type gaeIndexYAMLEntryProperty struct {
+	Name      string `yaml:"name"`
+	Direction string `yaml:"direction,omitempty"`
+}
+
+// gaeDispatchYAML is a YAML struct for an AppEngine "dispatch.yaml".
+type gaeDispatchYAML struct {
+	Dispatch []*gaeDispatchYAMLEntry `yaml:"dispatch,omitempty"`
+}
+
+// gaeDispatchYAMLEntry is a YAML struct for an AppEngine "dispatch.yaml" entry.
+type gaeDispatchYAMLEntry struct {
+	Module string `yaml:"module"`
+	URL    string `yaml:"url"`
+}
+
+// gaeQueueYAML is a YAML struct for an AppEngine "queue.yaml".
+type gaeQueueYAML struct {
+	Queue []*gaeQueueYAMLEntry `yaml:"queue,omitempty"`
+}
+
+// gaeQueueYAMLEntry is a YAML struct for an AppEngine "queue.yaml" entry.
+type gaeQueueYAMLEntry struct {
+	Name string `yaml:"name"`
+	Mode string `yaml:"mode"`
+
+	Rate                  string                            `yaml:"rate,omitempty"`
+	BucketSize            int                               `yaml:"bucket_size,omitempty"`
+	MaxConcurrentRequests int                               `yaml:"max_concurrent_requests,omitempty"`
+	RetryParameters       *gaeQueueYAMLEntryRetryParameters `yaml:"retry_parameters,omitempty"`
+	Target                string                            `yaml:"target,omitempty"`
+}
+
+// gaeQueueYAMLEntryRetryParameters is a YAML struct for an AppEngine
+// "queue.yaml" entry push queue retry parameters.
+type gaeQueueYAMLEntryRetryParameters struct {
+	TaskAgeLimit      string `yaml:"task_age_limit,omitempty"`
+	MinBackoffSeconds int    `yaml:"min_backoff_seconds,omitempty"`
+	MaxBackoffSeconds int    `yaml:"max_backoff_seconds,omitempty"`
+	MaxDoublings      int    `yaml:"max_doublings,omitempty"`
+}
+
+func gaeBuildAppYAML(aem *deploy.AppEngineModule, staticMap map[*deploy.BuildPath]string) (*gaeAppYAML, error) {
+	appYAML := gaeAppYAML{
+		Module: aem.ModuleName,
+	}
+
+	var (
+		defaultScript = ""
+		isStub        = false
+	)
+
+	switch aem.GetRuntime().(type) {
+	case *deploy.AppEngineModule_GoModule_:
+		appYAML.Runtime = "go"
+
+		if aem.ManagedVm != nil {
+			appYAML.APIVersion = 1
+			appYAML.VM = true
+		} else {
+			appYAML.APIVersion = "go1"
+		}
+		defaultScript = "_go_app"
+
+	case *deploy.AppEngineModule_StaticModule_:
+		// A static module presents itself as an empty Python AppEngine module. This
+		// doesn't actually need any Python code to support it.
+		appYAML.Runtime = "python27"
+		appYAML.APIVersion = "1"
+		threadSafe := true
+		appYAML.ThreadSafe = &threadSafe
+		isStub = true
+
+	default:
+		return nil, errors.Reason("unsupported runtime %(runtime)q").D("runtime", aem.Runtime).Err()
+	}
+
+	if mvm := aem.ManagedVm; mvm != nil {
+		if len(mvm.Scopes) > 0 {
+			appYAML.BetaSettings = &gaeAppYAMLBetaSettings{
+				ServiceAccountScopes: strings.Join(mvm.Scopes, ","),
+			}
+		}
+	}
+
+	if handlerSet := aem.Handlers; handlerSet != nil {
+		appYAML.Handlers = make([]*gaeAppYAMLHandler, len(handlerSet.Handler))
+		for i, handler := range handlerSet.Handler {
+			entry := gaeAppYAMLHandler{
+				URL:    handler.Url,
+				Secure: handler.Secure.AppYAMLString(),
+				Login:  handler.Login.AppYAMLString(),
+			}
+
+			// Fill in static dir/file paths.
+			switch t := handler.GetContent().(type) {
+			case nil:
+				entry.Script = defaultScript
+
+			case *deploy.AppEngineModule_Handler_Script:
+				entry.Script = t.Script
+				if entry.Script == "" {
+					entry.Script = defaultScript
+				}
+
+			case *deploy.AppEngineModule_Handler_StaticBuildDir:
+				entry.StaticDir = staticMap[t.StaticBuildDir]
+
+			case *deploy.AppEngineModule_Handler_StaticFiles_:
+				sf := t.StaticFiles
+
+				relDir := staticMap[sf.GetBuild()]
+				entry.Upload = filepath.Join(relDir, sf.Upload)
+				entry.StaticFiles = filepath.Join(relDir, sf.UrlMap)
+
+			default:
+				return nil, errors.Reason("don't know how to handle content %(content)T").
+					D("content", t).D("url", handler.Url).Err()
+			}
+
+			if entry.Script != "" && isStub {
+				return nil, errors.Reason("stub module cannot have entry script").Err()
+			}
+
+			appYAML.Handlers[i] = &entry
+		}
+	}
+
+	return &appYAML, nil
+}
+
+func gaeBuildCronYAML(cp *layoutDeploymentCloudProject) *gaeCronYAML {
+	var cronYAML gaeCronYAML
+
+	for _, m := range cp.appEngineModules {
+		for _, cron := range m.resources.Cron {
+			cronYAML.Cron = append(cronYAML.Cron, &gaeCronYAMLEntry{
+				Description: cron.Description,
+				URL:         cron.Url,
+				Schedule:    cron.Schedule,
+				Target:      m.ModuleName,
+			})
+		}
+	}
+	return &cronYAML
+}
+
+func gaeBuildIndexYAML(cp *layoutDeploymentCloudProject) *gaeIndexYAML {
+	var indexYAML gaeIndexYAML
+
+	for _, index := range cp.resources.Index {
+		entry := gaeIndexYAMLEntry{
+			Kind:       index.Kind,
+			Properties: make([]*gaeIndexYAMLEntryProperty, len(index.Property)),
+		}
+		if index.Ancestor {
+			entry.Ancestor = "yes"
+		}
+		for i, prop := range index.Property {
+			entry.Properties[i] = &gaeIndexYAMLEntryProperty{
+				Name:      prop.Name,
+				Direction: prop.Direction.AppYAMLString(),
+			}
+		}
+		indexYAML.Indexes = append(indexYAML.Indexes, &entry)
+	}
+	return &indexYAML
+}
+
+func gaeBuildDispatchYAML(cp *layoutDeploymentCloudProject) (*gaeDispatchYAML, error) {
+	var dispatchYAML gaeDispatchYAML
+
+	for _, m := range cp.appEngineModules {
+		for _, url := range m.resources.Dispatch {
+			dispatchYAML.Dispatch = append(dispatchYAML.Dispatch, &gaeDispatchYAMLEntry{
+				Module: m.ModuleName,
+				URL:    url,
+			})
+		}
+	}
+	if count := len(dispatchYAML.Dispatch); count > 10 {
+		return nil, errors.Reason("dispatch file has more than 10 routing rules (%(count)d)").
+			D("count", count).Err()
+	}
+	return &dispatchYAML, nil
+}
+
+func gaeBuildQueueYAML(cp *layoutDeploymentCloudProject) (*gaeQueueYAML, error) {
+	var queueYAML gaeQueueYAML
+
+	for _, m := range cp.appEngineModules {
+		for _, queue := range m.resources.TaskQueue {
+			entry := gaeQueueYAMLEntry{
+				Name: queue.Name,
+			}
+			switch t := queue.GetType().(type) {
+			case *deploy.AppEngineResources_TaskQueue_Push_:
+				push := t.Push
+
+				entry.Target = m.ModuleName
+				entry.Mode = "push"
+				entry.Rate = push.Rate
+				entry.BucketSize = int(push.BucketSize)
+				entry.RetryParameters = &gaeQueueYAMLEntryRetryParameters{
+					TaskAgeLimit:      push.RetryTaskAgeLimit,
+					MinBackoffSeconds: int(push.RetryMinBackoffSeconds),
+					MaxBackoffSeconds: int(push.RetryMaxBackoffSeconds),
+					MaxDoublings:      int(push.RetryMaxDoublings),
+				}
+
+			default:
+				return nil, errors.Reason("unknown task queue type %(type)T").D("type", t).Err()
+			}
+
+			queueYAML.Queue = append(queueYAML.Queue, &entry)
+		}
+	}
+	return &queueYAML, nil
+}
+
+// loadIndexYAMLResource loads an AppEngine "index.yaml" file from the specified
+// filesystem path and translates it into AppEngineResources Index entries.
+func loadIndexYAMLResource(path string) (*deploy.AppEngineResources, error) {
+	data, err := ioutil.ReadFile(path)
+	if err != nil {
+		return nil, errors.Annotate(err).Reason("failed to load [%(path)s]").D("path", path).Err()
+	}
+
+	var indexYAML gaeIndexYAML
+	if err := yaml.Unmarshal(data, &indexYAML); err != nil {
+		return nil, errors.Annotate(err).Reason("could not load 'index.yaml' from [%(path)s]").
+			D("path", path).Err()
+	}
+
+	var res deploy.AppEngineResources
+	if len(indexYAML.Indexes) > 0 {
+		res.Index = make([]*deploy.AppEngineResources_Index, len(indexYAML.Indexes))
+		for i, idx := range indexYAML.Indexes {
+			entry := deploy.AppEngineResources_Index{
+				Kind:     idx.Kind,
+				Ancestor: (idx.Ancestor == "yes"),
+				Property: make([]*deploy.AppEngineResources_Index_Property, len(idx.Properties)),
+			}
+			for pidx, idxProp := range idx.Properties {
+				dir, err := deploy.IndexDirectionFromAppYAMLString(idxProp.Direction)
+				if err != nil {
+					return nil, errors.Annotate(err).Reason("could not identify direction for entry").
+						D("index", i).D("propertyIndex", pidx).Err()
+				}
+
+				entry.Property[pidx] = &deploy.AppEngineResources_Index_Property{
+					Name:      idxProp.Name,
+					Direction: dir,
+				}
+			}
+
+			res.Index[i] = &entry
+		}
+	}
+	return &res, nil
+}