| OLD | NEW |
|---|
| (Empty) |
| 1 // Copyright 2016 The LUCI Authors. All rights reserved. | |
| 2 // Use of this source code is governed under the Apache License, Version 2.0 | |
| 3 // that can be found in the LICENSE file. | |
| 4 | |
| 5 package common | |
| 6 | |
| 7 import ( | |
| 8 "fmt" | |
| 9 "time" | |
| 10 | |
| 11 "github.com/golang/protobuf/proto" | |
| 12 "golang.org/x/net/context" | |
| 13 | |
| 14 "github.com/luci/gae/service/datastore" | |
| 15 "github.com/luci/gae/service/info" | |
| 16 "github.com/luci/luci-go/common/data/caching/proccache" | |
| 17 "github.com/luci/luci-go/common/logging" | |
| 18 "github.com/luci/luci-go/luci_config/server/cfgclient" | |
| 19 "github.com/luci/luci-go/luci_config/server/cfgclient/backend" | |
| 20 "github.com/luci/luci-go/luci_config/server/cfgclient/textproto" | |
| 21 | |
| 22 "github.com/luci/luci-go/milo/api/config" | |
| 23 ) | |
| 24 | |
| 25 // Project is a LUCI project. | |
| 26 type Project struct { | |
| 27 // The ID of the project, as per self defined. This is not the luci-con
fig | |
| 28 // name. | |
| 29 ID string `gae:"$id"` | |
| 30 // The luci-config name of the project. | |
| 31 Name string | |
| 32 // The Project data in protobuf binary format. | |
| 33 Data []byte `gae:",noindex"` | |
| 34 } | |
| 35 | |
| 36 // The key for the service config entity in datastore. | |
| 37 const ServiceConfigID = "service_config" | |
| 38 | |
| 39 // ServiceConfig is a container for the instance's service config. | |
| 40 type ServiceConfig struct { | |
| 41 // ID is the datastore key. This should be static, as there should only
be | |
| 42 // one service config. | |
| 43 ID string `gae:"$id"` | |
| 44 // Revision is the revision of the config, taken from luci-config. This
is used | |
| 45 // to determine if the entry needs to be refreshed. | |
| 46 Revision string | |
| 47 // Data is the binary proto of the config. | |
| 48 Data []byte `gae:",noindex"` | |
| 49 // Text is the text format of the config. For human consumption only. | |
| 50 Text string `gae:",noindex"` | |
| 51 // LastUpdated is the time this config was last updated. | |
| 52 LastUpdated time.Time | |
| 53 } | |
| 54 | |
| 55 // GetServiceConfig returns the service (aka global) config for the current | |
| 56 // instance of Milo from the datastore. Returns an empty config and warn heavil
y | |
| 57 // if none is found. | |
| 58 // TODO(hinoka): Use process cache to cache configs. | |
| 59 func GetSettings(c context.Context) *config.Settings { | |
| 60 settings := config.Settings{ | |
| 61 Buildbot: &config.Settings_Buildbot{}, | |
| 62 } | |
| 63 | |
| 64 msg, err := GetCurrentServiceConfig(c) | |
| 65 if err != nil { | |
| 66 // The service config does not exist, just return an empty confi
g | |
| 67 // and complain loudly in the logs. | |
| 68 logging.WithError(err).Errorf(c, | |
| 69 "Encountered error while loading service config, using e
mpty config.") | |
| 70 return &settings | |
| 71 } | |
| 72 | |
| 73 err = proto.Unmarshal(msg.Data, &settings) | |
| 74 if err != nil { | |
| 75 // The service config is broken, just return an empty config | |
| 76 // and complain loudly in the logs. | |
| 77 logging.WithError(err).Errorf(c, | |
| 78 "Encountered error while unmarshalling service config, u
sing empty config.") | |
| 79 // Zero out the message just incase something got written in. | |
| 80 settings = config.Settings{Buildbot: &config.Settings_Buildbot{}
} | |
| 81 } | |
| 82 | |
| 83 return &settings | |
| 84 } | |
| 85 | |
| 86 // GetCurrentServiceConfig gets the service config for the instance from either | |
| 87 // process cache or datastore cache. | |
| 88 func GetCurrentServiceConfig(c context.Context) (*ServiceConfig, error) { | |
| 89 // This maker function is used to do the actual fetch of the ServiceConf
ig | |
| 90 // from datastore. It is called if the ServiceConfig is not in proc cac
he. | |
| 91 maker := func() (interface{}, time.Duration, error) { | |
| 92 msg := ServiceConfig{ID: ServiceConfigID} | |
| 93 err := datastore.Get(c, &msg) | |
| 94 if err != nil { | |
| 95 return nil, time.Minute, err | |
| 96 } | |
| 97 logging.Infof(c, "loaded service config from datastore") | |
| 98 return msg, time.Minute, nil | |
| 99 } | |
| 100 item, err := proccache.GetOrMake(c, ServiceConfigID, maker) | |
| 101 if err != nil { | |
| 102 return nil, fmt.Errorf("failed to get service config: %s", err.E
rror()) | |
| 103 } | |
| 104 if msg, ok := item.(ServiceConfig); ok { | |
| 105 logging.Infof(c, "loaded config entry from %s", msg.LastUpdated.
Format(time.RFC3339)) | |
| 106 return &msg, nil | |
| 107 } | |
| 108 return nil, fmt.Errorf("could not load service config %#v", item) | |
| 109 } | |
| 110 | |
| 111 // UpdateServiceConfig fetches the service config from luci-config | |
| 112 // and then stores a snapshot of the configuration in datastore. | |
| 113 func UpdateServiceConfig(c context.Context) error { | |
| 114 // Load the settings from luci-config. | |
| 115 cs := string(cfgclient.CurrentServiceConfigSet(c)) | |
| 116 // Acquire the raw config client. | |
| 117 lucicfg := backend.Get(c).GetConfigInterface(c, backend.AsService) | |
| 118 // Our global config name is called settings.cfg. | |
| 119 cfg, err := lucicfg.GetConfig(c, cs, "settings.cfg", false) | |
| 120 if err != nil { | |
| 121 return fmt.Errorf("could not load settings.cfg from luci-config:
%s", err) | |
| 122 } | |
| 123 settings := &config.Settings{} | |
| 124 err = proto.UnmarshalText(cfg.Content, settings) | |
| 125 if err != nil { | |
| 126 return fmt.Errorf("could not unmarshal proto from luci-config:\n
%s", cfg.Content) | |
| 127 } | |
| 128 newConfig := ServiceConfig{ | |
| 129 ID: ServiceConfigID, | |
| 130 Text: cfg.Content, | |
| 131 Revision: cfg.Revision, | |
| 132 LastUpdated: time.Now().UTC(), | |
| 133 } | |
| 134 newConfig.Data, err = proto.Marshal(settings) | |
| 135 if err != nil { | |
| 136 return fmt.Errorf("could not marshal proto into binary\n%s", new
Config.Text) | |
| 137 } | |
| 138 | |
| 139 // Do the revision check & swap in a datastore transaction. | |
| 140 err = datastore.RunInTransaction(c, func(c context.Context) error { | |
| 141 oldConfig := ServiceConfig{ID: ServiceConfigID} | |
| 142 err := datastore.Get(c, &oldConfig) | |
| 143 switch err { | |
| 144 case datastore.ErrNoSuchEntity: | |
| 145 // Might be the first time this has run. | |
| 146 logging.WithError(err).Warningf(c, "No existing service
config.") | |
| 147 case nil: | |
| 148 // Continue | |
| 149 default: | |
| 150 return fmt.Errorf("could not load existing config: %s",
err) | |
| 151 } | |
| 152 // Check to see if we need to update | |
| 153 if oldConfig.Revision == newConfig.Revision { | |
| 154 logging.Infof(c, "revisions matched (%s), no need to upd
ate", oldConfig.Revision) | |
| 155 return nil | |
| 156 } | |
| 157 logging.Infof(c, "revisions differ (old %s, new %s), updating", | |
| 158 oldConfig.Revision, newConfig.Revision) | |
| 159 return datastore.Put(c, &newConfig) | |
| 160 }, nil) | |
| 161 | |
| 162 if err != nil { | |
| 163 return fmt.Errorf("failed to update config entry in transaction"
, err) | |
| 164 } | |
| 165 logging.Infof(c, "successfully updated to new config") | |
| 166 | |
| 167 return nil | |
| 168 } | |
| 169 | |
| 170 // UpdateProjectConfigs internal project configuration based off luci-config. | |
| 171 // update updates Milo's configuration based off luci config. This includes | |
| 172 // scanning through all project and extract all console configs. | |
| 173 func UpdateProjectConfigs(c context.Context) error { | |
| 174 cfgName := info.AppID(c) + ".cfg" | |
| 175 | |
| 176 var ( | |
| 177 configs []*config.Project | |
| 178 metas []*cfgclient.Meta | |
| 179 ) | |
| 180 logging.Debugf(c, "fetching configs for %s", cfgName) | |
| 181 if err := cfgclient.Projects(c, cfgclient.AsService, cfgName, textproto.
Slice(&configs), &metas); err != nil { | |
| 182 logging.WithError(err).Errorf(c, "Encountered error while gettin
g project config for %s", cfgName) | |
| 183 return err | |
| 184 } | |
| 185 | |
| 186 // A map of project ID to project. | |
| 187 projects := map[string]*Project{} | |
| 188 for i, proj := range configs { | |
| 189 projectName, _, _ := metas[i].ConfigSet.SplitProject() | |
| 190 | |
| 191 logging.Infof(c, "Prossing %s", projectName) | |
| 192 if dup, ok := projects[proj.ID]; ok { | |
| 193 return fmt.Errorf( | |
| 194 "Duplicate project ID: %s. (%s and %s)", proj.ID
, dup.Name, projectName) | |
| 195 } | |
| 196 p := &Project{ | |
| 197 ID: proj.ID, | |
| 198 Name: string(projectName), | |
| 199 } | |
| 200 projects[proj.ID] = p | |
| 201 | |
| 202 var err error | |
| 203 p.Data, err = proto.Marshal(proj) | |
| 204 if err != nil { | |
| 205 return err | |
| 206 } | |
| 207 } | |
| 208 | |
| 209 // Now load all the data into the datastore. | |
| 210 projs := make([]*Project, 0, len(projects)) | |
| 211 for _, proj := range projects { | |
| 212 projs = append(projs, proj) | |
| 213 } | |
| 214 if err := datastore.Put(c, projs); err != nil { | |
| 215 return err | |
| 216 } | |
| 217 | |
| 218 // Delete entries that no longer exist. | |
| 219 q := datastore.NewQuery("Project").KeysOnly(true) | |
| 220 allProjs := []Project{} | |
| 221 datastore.GetAll(c, q, &allProjs) | |
| 222 toDelete := []Project{} | |
| 223 for _, proj := range allProjs { | |
| 224 if _, ok := projects[proj.ID]; !ok { | |
| 225 toDelete = append(toDelete, proj) | |
| 226 } | |
| 227 } | |
| 228 datastore.Delete(c, toDelete) | |
| 229 | |
| 230 return nil | |
| 231 } | |
| 232 | |
| 233 // GetAllProjects returns all registered projects. | |
| 234 func GetAllProjects(c context.Context) ([]*config.Project, error) { | |
| 235 q := datastore.NewQuery("Project") | |
| 236 q.Order("ID") | |
| 237 | |
| 238 ps := []*Project{} | |
| 239 err := datastore.GetAll(c, q, &ps) | |
| 240 if err != nil { | |
| 241 return nil, err | |
| 242 } | |
| 243 results := make([]*config.Project, len(ps)) | |
| 244 for i, p := range ps { | |
| 245 results[i] = &config.Project{} | |
| 246 if err := proto.Unmarshal(p.Data, results[i]); err != nil { | |
| 247 return nil, err | |
| 248 } | |
| 249 } | |
| 250 return results, nil | |
| 251 } | |
| 252 | |
| 253 // GetProject returns the requested project. | |
| 254 func GetProject(c context.Context, projName string) (*config.Project, error) { | |
| 255 // Next, Try datastore | |
| 256 p := Project{ID: projName} | |
| 257 if err := datastore.Get(c, &p); err != nil { | |
| 258 return nil, err | |
| 259 } | |
| 260 mp := config.Project{} | |
| 261 if err := proto.Unmarshal(p.Data, &mp); err != nil { | |
| 262 return nil, err | |
| 263 } | |
| 264 | |
| 265 return &mp, nil | |
| 266 } | |
| 267 | |
| 268 // GetConsole returns the requested console instance. | |
| 269 func GetConsole(c context.Context, projName, consoleName string) (*config.Consol
e, error) { | |
| 270 p, err := GetProject(c, projName) | |
| 271 if err != nil { | |
| 272 return nil, err | |
| 273 } | |
| 274 for _, cs := range p.Consoles { | |
| 275 if cs.Name == consoleName { | |
| 276 return cs, nil | |
| 277 } | |
| 278 } | |
| 279 return nil, fmt.Errorf("Console %s not found in project %s", consoleName
, projName) | |
| 280 } | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 2949783002:
[milo] appengine/* -> * (Closed)
