Milo: Use luci-cfg for defining projects and console view.

milo/appengine/settings/config.go

Index: milo/appengine/settings/config.go
diff --git a/milo/appengine/settings/config.go b/milo/appengine/settings/config.go
new file mode 100644
index 0000000000000000000000000000000000000000..6bb28053386712e6c74546e5b490ca8da512b515
--- /dev/null
+++ b/milo/appengine/settings/config.go
@@ -0,0 +1,134 @@
+// Copyright 2016 The LUCI Authors. All rights reserved.
+// Use of this source code is governed under the Apache License, Version 2.0
+// that can be found in the LICENSE file.
+
+package settings
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/golang/protobuf/proto"
+	"github.com/luci/gae/service/datastore"
+	"github.com/luci/gae/service/info"
+	"github.com/luci/luci-go/common/config"
+	"github.com/luci/luci-go/common/logging"
+	milocfg "github.com/luci/luci-go/milo/common/config"
+	"github.com/luci/luci-go/server/router"
+	"golang.org/x/net/context"
+)
+
+type Project struct {
+	// The ID of the project, as per self defined.  This is not the luci-cfg
+	// name.
+	ID string `gae:"$id"`
+	// The luci-cfg name of the project.
+	Name string
+	// The Project data in protobuf binary format.
+	Data []byte `gae:",noindex"`
+}
+
+func UpdateHandler(ctx *router.Context) {
+	c, h := ctx.Context, ctx.Writer
+	err := update(c)
+	if err != nil {
+		logging.WithError(err).Errorf(c, "Update Handler encountered error")
+		h.WriteHeader(500)
+	}
+	logging.Infof(c, "Successfully completed")
+	h.WriteHeader(200)
+}
+
+// Update internal configuration based off luci-cfg.
+// update updates Milo's configuration based off luci config.  This includes
+// scanning through all project and extract all console configs.
+func update(c context.Context) error {
+	cfgName := info.Get(c).AppID() + ".cfg"
+	cfgs, err := config.GetProjectConfigs(c, cfgName, false)
+	if err != nil {
+		return err
+	}
+	// A map of project ID to project.
+	projects := map[string]*Project{}
+	for _, cfg := range cfgs {
+		pathParts := strings.SplitN(cfg.ConfigSet, "/", 2)
+		if len(pathParts) != 2 {
+			return fmt.Errorf("Invalid config path: %s", cfg.Path)
+		}
+		name := pathParts[1]
+		proj := milocfg.Project{}
+		logging.Infof(c, "Prossing %s", name)
+		if err = proto.UnmarshalText(cfg.Content, &proj); err != nil {
+			return err
+		}
+		if dup, ok := projects[proj.ID]; ok {
+			return fmt.Errorf(
+				"Duplicate project ID: %s. (%s and %s)", proj.ID, dup.Name, name)
+		}
+		p := &Project{
+			ID:   proj.ID,
+			Name: name,
+		}
+		projects[proj.ID] = p
+		p.Data, err = proto.Marshal(&proj)
+		if err != nil {
+			return err
+		}
+	}
+
+	// Now load all the data into the datastore.
+	ds := datastore.Get(c)
+	projs := make([]*Project, 0, len(projects))
+	for _, proj := range projects {
+		projs = append(projs, proj)
+	}
+	err = ds.Put(projs)
+	if err != nil {
+		return err
+	}
+
+	// Delete entries that no longer exist.
+	q := datastore.NewQuery("Project").KeysOnly(true)
+	allProjs := []Project{}
+	ds.GetAll(q, &allProjs)
+	toDelete := []Project{}
+	for _, proj := range allProjs {
+		if _, ok := projects[proj.ID]; !ok {
+			toDelete = append(toDelete, proj)
+		}
+	}
+	ds.Delete(toDelete)
+
+	return nil
+}
+
+func GetProject(c context.Context, projName string) (*milocfg.Project, error) {
+	// TODO(hinoka): First, get it from Memcache

[estaab, 2016/08/18 23:19:37]

Doesn't gae have an automatic memcaching layer? Maybe I'm totally mistaken :P

[Ryan Tseng, 2016/08/18 23:55:30]

Yeah it does... removing now.

+
+	// Next, Try datastore
+	ds := datastore.Get(c)
+	p := Project{ID: projName}
+	if err := ds.Get(&p); err != nil {
+		return nil, err
+	}
+	mp := milocfg.Project{}
+	if err := proto.Unmarshal(p.Data, &mp); err != nil {
+		return nil, err
+	}
+	// TODO(hinoka): Save it in memcache
+
+	return &mp, nil
+}
+
+func GetConsole(c context.Context, projName, consoleName string) (*milocfg.Console, error) {
+	p, err := GetProject(c, projName)
+	if err != nil {
+		return nil, err
+	}
+	for _, cs := range p.Consoles {
+		if cs.Name == consoleName {
+			return cs, nil
+		}
+	}
+	return nil, fmt.Errorf("Console %s not found in project %s", consoleName, projName)
+}