| OLD | NEW |
|---|
| (Empty) |
| 1 // Copyright 2016 The LUCI Authors. All rights reserved. | |
| 2 // Use of this source code is governed under the Apache License, Version 2.0 | |
| 3 // that can be found in the LICENSE file. | |
| 4 | |
| 5 package main | |
| 6 | |
| 7 import ( | |
| 8 "io/ioutil" | |
| 9 "os" | |
| 10 "path/filepath" | |
| 11 "sort" | |
| 12 "strings" | |
| 13 | |
| 14 "github.com/luci/luci-go/common/errors" | |
| 15 log "github.com/luci/luci-go/common/logging" | |
| 16 "github.com/luci/luci-go/deploytool/api/deploy" | |
| 17 "github.com/luci/luci-go/deploytool/managedfs" | |
| 18 | |
| 19 "golang.org/x/net/context" | |
| 20 ) | |
| 21 | |
| 22 const defaultLayoutFilename = "layout.cfg" | |
| 23 | |
| 24 type layoutSource struct { | |
| 25 *deploy.FrozenLayout_Source | |
| 26 | |
| 27 sg *layoutSourceGroup | |
| 28 title title | |
| 29 } | |
| 30 | |
| 31 func (s *layoutSource) String() string { return joinPath(s.sg.title, s.title) } | |
| 32 | |
| 33 func (s *layoutSource) checkoutPath() string { | |
| 34 return s.sg.layout.workingPathTo(s.Relpath) | |
| 35 } | |
| 36 | |
| 37 // pathTo resolves a path, p, specified in a directory with source-root-relative | |
| 38 // path relpath. | |
| 39 // | |
| 40 // - If the path begins with "/", it is taken relative to the source root. | |
| 41 // - If the path does not begin with "/", it is taken relative to "relpath" | |
| 42 // within the source root (e.g., "relpath" is prepended to it). | |
| 43 func (s *layoutSource) pathTo(p, relpath string) string { | |
| 44 if s.Relpath == "" { | |
| 45 panic(errors.Reason("source %(source)q is not checked out").D("s
ource", s).Err()) | |
| 46 } | |
| 47 | |
| 48 // If this is absolute, take it relative to source root. | |
| 49 if strings.HasPrefix(p, "/") { | |
| 50 relpath = "" | |
| 51 } | |
| 52 | |
| 53 // Convert "p" to a source-relative absolute path. | |
| 54 p = strings.Trim(p, "/") | |
| 55 if relpath != "" { | |
| 56 relpath = strings.Trim(relpath, "/") | |
| 57 p = relpath + "/" + p | |
| 58 } | |
| 59 | |
| 60 return deployToNative(s.checkoutPath(), p) | |
| 61 } | |
| 62 | |
| 63 // layoutSourceGroup is a group of named, associated Source entries. | |
| 64 type layoutSourceGroup struct { | |
| 65 *deploy.FrozenLayout_SourceGroup | |
| 66 | |
| 67 // layout is the owning layout. | |
| 68 layout *deployLayout | |
| 69 | |
| 70 title title | |
| 71 sources map[title]*layoutSource | |
| 72 } | |
| 73 | |
| 74 func (sg *layoutSourceGroup) String() string { return string(sg.title) } | |
| 75 | |
| 76 func (sg *layoutSourceGroup) allSources() []*layoutSource { | |
| 77 keys := make([]string, 0, len(sg.sources)) | |
| 78 for key := range sg.sources { | |
| 79 keys = append(keys, string(key)) | |
| 80 } | |
| 81 sort.Strings(keys) | |
| 82 | |
| 83 srcs := make([]*layoutSource, len(keys)) | |
| 84 for i, k := range keys { | |
| 85 srcs[i] = sg.sources[title(k)] | |
| 86 } | |
| 87 return srcs | |
| 88 } | |
| 89 | |
| 90 type layoutApp struct { | |
| 91 *deploy.Application | |
| 92 | |
| 93 title title | |
| 94 components map[title]*layoutAppComponent | |
| 95 } | |
| 96 | |
| 97 type layoutAppComponent struct { | |
| 98 *deploy.Application_Component | |
| 99 | |
| 100 proj *layoutApp | |
| 101 title title | |
| 102 } | |
| 103 | |
| 104 type layoutDeployment struct { | |
| 105 *deploy.Deployment | |
| 106 | |
| 107 l *deployLayout | |
| 108 title title | |
| 109 | |
| 110 // app is the application that this deployment is bound to. | |
| 111 app *layoutApp | |
| 112 // sg is the source group that this deployment is bound to. | |
| 113 sg *layoutSourceGroup | |
| 114 | |
| 115 // components is the set of deployed application components. | |
| 116 components map[title]*layoutDeploymentComponent | |
| 117 // componentNames is a list of keys in the components map, ordered by th
e | |
| 118 // order in which these components appeared in the protobuf. | |
| 119 componentNames []title | |
| 120 // cloudProject is the deployment cloud project, if one is specified. | |
| 121 cloudProject *layoutDeploymentCloudProject | |
| 122 } | |
| 123 | |
| 124 func (d *layoutDeployment) String() string { return string(d.title) } | |
| 125 | |
| 126 // substituteParams applies parameter substitution to the supplied string in a | |
| 127 // left-to-right manner. | |
| 128 func (d *layoutDeployment) substituteParams(vp *string) error { | |
| 129 if err := substitute(vp, d.Parameter); err != nil { | |
| 130 return errors.Annotate(err).Err() | |
| 131 } | |
| 132 return nil | |
| 133 } | |
| 134 | |
| 135 type layoutDeploymentComponent struct { | |
| 136 deploy.Component | |
| 137 | |
| 138 // reldir is the source-relative directory that relative paths in this | |
| 139 // component will reference. This will be the parent directory of the | |
| 140 // component's configuration file. | |
| 141 reldir string | |
| 142 | |
| 143 // dep is the deployment that this component belongs to. | |
| 144 dep *layoutDeployment | |
| 145 // comp is the Component definition. | |
| 146 comp *layoutAppComponent | |
| 147 // sources is the set of sources required to build this Component. This
will | |
| 148 // always have at least one entry: the source where this Component is lo
cated. | |
| 149 sources []*layoutSource | |
| 150 | |
| 151 // buildDirs is a map of build directory keys to their on-disk paths gen
erated | |
| 152 // for them. This is populated during `buildComponent()`. | |
| 153 buildDirs map[string]string | |
| 154 | |
| 155 // buildPathMap is a map of the resolved on-disk paths for a given Build
Path. | |
| 156 buildPathMap map[*deploy.BuildPath]string | |
| 157 | |
| 158 // gkePod is the set of cluster-bound GKE pods that this component is de
ployed | |
| 159 // to. | |
| 160 gkePod *layoutDeploymentGKEPod | |
| 161 | |
| 162 // gkePods is the set of pod/cluster bindings for this Component's pod. | |
| 163 gkePods []*layoutDeploymentGKEPodBinding | |
| 164 } | |
| 165 | |
| 166 func (comp *layoutDeploymentComponent) String() string { | |
| 167 return joinPath(comp.dep.title, comp.comp.title) | |
| 168 } | |
| 169 | |
| 170 func (comp *layoutDeploymentComponent) source() *layoutSource { | |
| 171 return comp.sources[0] | |
| 172 } | |
| 173 | |
| 174 func (comp *layoutDeploymentComponent) pathTo(relpath string) string { | |
| 175 return comp.source().pathTo(relpath, comp.reldir) | |
| 176 } | |
| 177 | |
| 178 func (comp *layoutDeploymentComponent) buildPath(bp *deploy.BuildPath) (string,
error) { | |
| 179 if path, ok := comp.buildPathMap[bp]; ok { | |
| 180 return path, nil | |
| 181 } | |
| 182 return "", errors.Reason("no build path resolved for: %(bp)+v").D("bp",
bp, "%+v").Err() | |
| 183 } | |
| 184 | |
| 185 func (comp *layoutDeploymentComponent) loadSourceComponent(reg componentRegistra
r) error { | |
| 186 if err := unmarshalTextProtobuf(comp.source().pathTo(comp.comp.Path, "")
, &comp.Component); err != nil { | |
| 187 return errors.Annotate(err).Reason("failed to load source compon
ent %(component)q").D("component", comp).Err() | |
| 188 } | |
| 189 | |
| 190 // Referenced build paths. | |
| 191 for i, p := range comp.BuildPath { | |
| 192 var msg deploy.Component_Build | |
| 193 if err := unmarshalTextProtobuf(comp.pathTo(p), &msg); err != ni
l { | |
| 194 return errors.Annotate(err).Reason("failed to load compo
nent Build #%(index)d from [%(path)s]"). | |
| 195 D("index", i).D("path", p).Err() | |
| 196 } | |
| 197 comp.Build = append(comp.Build, &msg) | |
| 198 } | |
| 199 | |
| 200 // Load any referenced data and normalize it for internal usage. | |
| 201 dep := comp.dep | |
| 202 switch t := comp.GetComponent().(type) { | |
| 203 case *deploy.Component_AppengineModule: | |
| 204 // Normalize AppEngine module: | |
| 205 // - Modules explicitly named "default" will have their ModuleNa
me changed | |
| 206 // to the empty string. | |
| 207 // - All referenced path parameters will be loaded and appended
onto their | |
| 208 // non-path members. | |
| 209 if dep.cloudProject == nil { | |
| 210 return errors.Reason("AppEngine module %(comp)q requires
a cloud project"). | |
| 211 D("comp", comp).Err() | |
| 212 } | |
| 213 | |
| 214 aem := t.AppengineModule | |
| 215 if aem.ModuleName == "default" { | |
| 216 aem.ModuleName = "" | |
| 217 } | |
| 218 | |
| 219 module := layoutDeploymentGAEModule{ | |
| 220 AppEngineModule: aem, | |
| 221 comp: comp, | |
| 222 } | |
| 223 | |
| 224 // Referenced handler paths. | |
| 225 for i, p := range aem.HandlerPath { | |
| 226 var msg deploy.AppEngineModule_HandlerSet | |
| 227 if err := unmarshalTextProtobuf(comp.pathTo(p), &msg); e
rr != nil { | |
| 228 return errors.Annotate(err).Reason("failed to lo
ad HandlerSet #%(index)d for %(component)q"). | |
| 229 D("index", i).D("component", comp).Err() | |
| 230 } | |
| 231 module.Handlers.Handler = append(module.Handlers.Handler
, msg.Handler...) | |
| 232 } | |
| 233 | |
| 234 // Append GAE Resources. | |
| 235 if r := module.Resources; r != nil { | |
| 236 dep.cloudProject.appendResources(r, &module) | |
| 237 } | |
| 238 | |
| 239 for i, p := range module.ResourcePath { | |
| 240 if err := comp.dep.substituteParams(&p); err != nil { | |
| 241 return errors.Annotate(err).Reason("failed to su
bstitute parameters for resource path"). | |
| 242 D("path", p).Err() | |
| 243 } | |
| 244 | |
| 245 var res deploy.AppEngineResources | |
| 246 if err := unmarshalTextProtobuf(comp.pathTo(p), &res); e
rr != nil { | |
| 247 return errors.Annotate(err).Reason("failed to lo
ad Resources #%(index)d for %(component)"). | |
| 248 D("index", i).D("path", p).D("component"
, comp).Err() | |
| 249 } | |
| 250 dep.cloudProject.appendResources(&res, &module) | |
| 251 } | |
| 252 | |
| 253 // Add this module to our cloud project's AppEngine modules list
. | |
| 254 dep.cloudProject.appEngineModules = append(dep.cloudProject.appE
ngineModules, &module) | |
| 255 if reg != nil { | |
| 256 reg.addGAEModule(&module) | |
| 257 } | |
| 258 | |
| 259 case *deploy.Component_GkePod: | |
| 260 if len(comp.gkePods) == 0 { | |
| 261 return errors.Reason("GKE Container %(comp)q is not boun
d to a GKE cluster"). | |
| 262 D("comp", comp.String()).Err() | |
| 263 } | |
| 264 | |
| 265 comp.gkePod = &layoutDeploymentGKEPod{ | |
| 266 ContainerEnginePod: t.GkePod, | |
| 267 comp: comp, | |
| 268 } | |
| 269 | |
| 270 // None of the labels may use our "deploytool" prefix. | |
| 271 var invalidLabels []string | |
| 272 for k := range comp.gkePod.KubePod.Labels { | |
| 273 if isKubeDeployToolKey(k) { | |
| 274 invalidLabels = append(invalidLabels, k) | |
| 275 } | |
| 276 } | |
| 277 if len(invalidLabels) > 0 { | |
| 278 sort.Strings(invalidLabels) | |
| 279 return errors.Reason("user-supplied labels may not use d
eploytool prefix"). | |
| 280 D("labels", invalidLabels).Err() | |
| 281 } | |
| 282 | |
| 283 for _, bp := range comp.gkePods { | |
| 284 bp.pod = comp.gkePod | |
| 285 if reg != nil { | |
| 286 reg.addGKEPod(bp) | |
| 287 } | |
| 288 } | |
| 289 } | |
| 290 | |
| 291 return nil | |
| 292 } | |
| 293 | |
| 294 // expandPaths iterates through the Component-defined directory fields and | |
| 295 // expands their paths into actual filesystem paths. | |
| 296 // | |
| 297 // This must be performed after the build instructions have been executed so | |
| 298 // that the "build_dir" map will be available if needed by a Component. | |
| 299 func (comp *layoutDeploymentComponent) expandPaths() error { | |
| 300 resolveBuildPath := func(bp *deploy.BuildPath) error { | |
| 301 if _, ok := comp.buildPathMap[bp]; ok { | |
| 302 // Already resolved. | |
| 303 return nil | |
| 304 } | |
| 305 | |
| 306 var resolved string | |
| 307 if bp.DirKey != "" { | |
| 308 dir, ok := comp.buildDirs[bp.DirKey] | |
| 309 if !ok { | |
| 310 return errors.Reason("Invalid `dir_key` value: %
(dirKey)q").D("dirKey", bp.DirKey).Err() | |
| 311 } | |
| 312 resolved = deployToNative(dir, bp.Path) | |
| 313 } else { | |
| 314 resolved = comp.pathTo(bp.Path) | |
| 315 } | |
| 316 | |
| 317 if comp.buildPathMap == nil { | |
| 318 comp.buildPathMap = make(map[*deploy.BuildPath]string) | |
| 319 } | |
| 320 comp.buildPathMap[bp] = resolved | |
| 321 return nil | |
| 322 } | |
| 323 | |
| 324 switch t := comp.Component.Component.(type) { | |
| 325 case *deploy.Component_AppengineModule: | |
| 326 aem := t.AppengineModule | |
| 327 if aem.Handlers != nil { | |
| 328 for _, handler := range aem.Handlers.Handler { | |
| 329 switch c := handler.Content.(type) { | |
| 330 case *deploy.AppEngineModule_Handler_StaticFiles
_: | |
| 331 sf := c.StaticFiles | |
| 332 switch bd := sf.BaseDir.(type) { | |
| 333 case *deploy.AppEngineModule_Handler_Sta
ticFiles_Path: | |
| 334 // Normalize our static files di
rectory to a BuildPath rooted at our | |
| 335 // source. | |
| 336 sf.BaseDir = &deploy.AppEngineMo
dule_Handler_StaticFiles_Build{ | |
| 337 Build: &deploy.BuildPath
{Path: bd.Path}, | |
| 338 } | |
| 339 if err := resolveBuildPath(sf.Ge
tBuild()); err != nil { | |
| 340 return errors.Annotate(e
rr).Err() | |
| 341 } | |
| 342 | |
| 343 case *deploy.AppEngineModule_Handler_Sta
ticFiles_Build: | |
| 344 if err := resolveBuildPath(bd.Bu
ild); err != nil { | |
| 345 return errors.Annotate(e
rr).Err() | |
| 346 } | |
| 347 | |
| 348 default: | |
| 349 return errors.Reason("unknown `b
ase_dir` type %(type)T").D("type", bd).Err() | |
| 350 } | |
| 351 | |
| 352 case *deploy.AppEngineModule_Handler_StaticDir: | |
| 353 // Normalize our static directory (sourc
e-relative) to a BuildPath | |
| 354 // rooted at our source. | |
| 355 handler.Content = &deploy.AppEngineModul
e_Handler_StaticBuildDir{ | |
| 356 StaticBuildDir: &deploy.BuildPat
h{Path: c.StaticDir}, | |
| 357 } | |
| 358 if err := resolveBuildPath(handler.GetSt
aticBuildDir()); err != nil { | |
| 359 return errors.Annotate(err).Err(
) | |
| 360 } | |
| 361 | |
| 362 case *deploy.AppEngineModule_Handler_StaticBuild
Dir: | |
| 363 if err := resolveBuildPath(c.StaticBuild
Dir); err != nil { | |
| 364 return errors.Annotate(err).Err(
) | |
| 365 } | |
| 366 } | |
| 367 } | |
| 368 } | |
| 369 | |
| 370 case *deploy.Component_GkePod: | |
| 371 for _, container := range t.GkePod.KubePod.Container { | |
| 372 switch df := container.Dockerfile.(type) { | |
| 373 case *deploy.KubernetesPod_Container_Path: | |
| 374 // Convert to BuildPath. | |
| 375 container.Dockerfile = &deploy.KubernetesPod_Con
tainer_Build{ | |
| 376 Build: &deploy.BuildPath{Path: df.Path}, | |
| 377 } | |
| 378 if err := resolveBuildPath(container.GetBuild())
; err != nil { | |
| 379 return errors.Annotate(err).Err() | |
| 380 } | |
| 381 | |
| 382 case *deploy.KubernetesPod_Container_Build: | |
| 383 if err := resolveBuildPath(df.Build); err != nil
{ | |
| 384 return errors.Annotate(err).Err() | |
| 385 } | |
| 386 | |
| 387 default: | |
| 388 return errors.Reason("unknown `dockerfile` type
%(type)T").D("type", df).Err() | |
| 389 } | |
| 390 } | |
| 391 } | |
| 392 | |
| 393 return nil | |
| 394 } | |
| 395 | |
| 396 // layoutDeploymentCloudProject tracks a cloud project element, as well as | |
| 397 // any cloud project configurations referenced by this Deployment's Components. | |
| 398 type layoutDeploymentCloudProject struct { | |
| 399 *deploy.Deployment_CloudProject | |
| 400 | |
| 401 // dep is the Deployment that owns this cloud project. | |
| 402 dep *layoutDeployment | |
| 403 // gkeCluster is a map of GKE cluster names to their definitions. | |
| 404 gkeClusters map[string]*layoutDeploymentGKECluster | |
| 405 // appEngineModules is the set of AppEngine modules in this Deployment t
hat | |
| 406 // reference this cloud project. | |
| 407 appEngineModules []*layoutDeploymentGAEModule | |
| 408 | |
| 409 // resources is the accumulated set of AppEngine Resources, loaded from | |
| 410 // component and deployment source configs. | |
| 411 resources deploy.AppEngineResources | |
| 412 } | |
| 413 | |
| 414 func (cp *layoutDeploymentCloudProject) String() string { | |
| 415 return joinPath(title(cp.dep.String()), title(cp.Name)) | |
| 416 } | |
| 417 | |
| 418 func (cp *layoutDeploymentCloudProject) appendResources(res *deploy.AppEngineRes
ources, | |
| 419 module *layoutDeploymentGAEModule) { | |
| 420 | |
| 421 // Accumulate global (non-module-bound) resources in our cloud project's | |
| 422 // resources protobuf. | |
| 423 cp.resources.Index = append(cp.resources.Index, res.Index...) | |
| 424 | |
| 425 // Accumulate module-specific resources in our module's resources protob
uf. | |
| 426 module.resources.Dispatch = append(module.resources.Dispatch, res.Dispat
ch...) | |
| 427 module.resources.TaskQueue = append(module.resources.TaskQueue, res.Task
Queue...) | |
| 428 module.resources.Cron = append(module.resources.Cron, res.Cron...) | |
| 429 } | |
| 430 | |
| 431 // layoutDeploymentGAEModule is a single configured AppEngine module. | |
| 432 type layoutDeploymentGAEModule struct { | |
| 433 *deploy.AppEngineModule | |
| 434 | |
| 435 // comp is the component that describes this module. | |
| 436 comp *layoutDeploymentComponent | |
| 437 | |
| 438 // resources is the set of module-bound resources. | |
| 439 resources deploy.AppEngineResources | |
| 440 } | |
| 441 | |
| 442 // layoutDeploymentGKECluster tracks a GKE cluster element. | |
| 443 type layoutDeploymentGKECluster struct { | |
| 444 *deploy.Deployment_CloudProject_GKECluster | |
| 445 | |
| 446 // cloudProject is the cloud project that this GKE cluster belongs to. | |
| 447 cloudProject *layoutDeploymentCloudProject | |
| 448 // pods is the set of GKE pods in this Deployment. | |
| 449 pods []*layoutDeploymentGKEPodBinding | |
| 450 } | |
| 451 | |
| 452 func (c *layoutDeploymentGKECluster) String() string { | |
| 453 return joinPath(title(c.cloudProject.String()), title(c.Name)) | |
| 454 } | |
| 455 | |
| 456 // layoutDeploymentGKEPodBinding tracks a GKE pod bound to a GKE cluster. | |
| 457 type layoutDeploymentGKEPodBinding struct { | |
| 458 *deploy.Deployment_CloudProject_GKECluster_PodBinding | |
| 459 | |
| 460 // cluster is the cluster that the pod is deployed to. | |
| 461 cluster *layoutDeploymentGKECluster | |
| 462 // pod is the pod that is deployed. This is filled in when project conte
nts | |
| 463 // are loaded. | |
| 464 pod *layoutDeploymentGKEPod | |
| 465 } | |
| 466 | |
| 467 func (pb *layoutDeploymentGKEPodBinding) String() string { | |
| 468 return joinPath(title(pb.pod.String()), title(pb.cluster.String())) | |
| 469 } | |
| 470 | |
| 471 // layoutDeploymentGKEPod tracks a deployed pod component bound to a GKE | |
| 472 // cluster. | |
| 473 type layoutDeploymentGKEPod struct { | |
| 474 // The container engine pod definition. This won't be filled in until th
e | |
| 475 // deployment's source configuration is loaded in loadSourceComponent. | |
| 476 *deploy.ContainerEnginePod | |
| 477 | |
| 478 // comp is the component that this pod was described by. | |
| 479 comp *layoutDeploymentComponent | |
| 480 } | |
| 481 | |
| 482 func (pb *layoutDeploymentGKEPod) String() string { return pb.comp.String() } | |
| 483 | |
| 484 // deployLayout is the loaded and configured layout, populated with any state | |
| 485 // that has been loaded during operation. | |
| 486 type deployLayout struct { | |
| 487 deploy.Layout | |
| 488 | |
| 489 // base is the base path of the layout file. By default, all other sourc
es | |
| 490 // will be relative to this path. | |
| 491 basePath string | |
| 492 | |
| 493 // user is the user configuration protobuf. | |
| 494 user deploy.UserConfig | |
| 495 // userSourceOverrides is a map of user checkout URL overrides. | |
| 496 userSourceOverrides map[string]*deploy.Source | |
| 497 | |
| 498 // sourceGroups is the set of source group files, mapped to their source
group | |
| 499 // name. | |
| 500 sourceGroups map[title]*layoutSourceGroup | |
| 501 // apps is the set of applications, mapped to their application name. | |
| 502 apps map[title]*layoutApp | |
| 503 | |
| 504 // deployments is the set of deployments, mapped to their deployment nam
e. | |
| 505 deployments map[title]*layoutDeployment | |
| 506 // deploymentNames is a list of keys in the deployments map, ordered by
the | |
| 507 // order in which the deployments were loaded. | |
| 508 deploymentNames []title | |
| 509 // map of cloud project names to their deployments. | |
| 510 cloudProjects map[string]*layoutDeploymentCloudProject | |
| 511 } | |
| 512 | |
| 513 // workingFilesystem creates a new managed filesystem at our working directory | |
| 514 // root. | |
| 515 // | |
| 516 // This adds layout-defined components to the filesystem. | |
| 517 func (l *deployLayout) workingFilesystem() (*managedfs.Filesystem, error) { | |
| 518 fs, err := managedfs.New(l.WorkingPath) | |
| 519 if err != nil { | |
| 520 return nil, errors.Annotate(err).Err() | |
| 521 } | |
| 522 return fs, nil | |
| 523 } | |
| 524 | |
| 525 func (l *deployLayout) workingPathTo(relpath string) string { | |
| 526 return filepath.Join(l.WorkingPath, relpath) | |
| 527 } | |
| 528 | |
| 529 func (l *deployLayout) getDeploymentComponent(v string) (*layoutDeployment, *lay
outDeploymentComponent, error) { | |
| 530 deployment, component := splitComponentPath(v) | |
| 531 | |
| 532 dep := l.deployments[deployment] | |
| 533 if dep == nil { | |
| 534 return nil, nil, errors.Reason("unknown Deployment %(dep)q"). | |
| 535 D("value", v).D("dep", deployment).Err() | |
| 536 } | |
| 537 | |
| 538 // If a component was specified, only add that component. | |
| 539 if component != "" { | |
| 540 comp := dep.components[component] | |
| 541 if comp == nil { | |
| 542 return nil, nil, errors.Reason("unknown Deployment Compo
nent %(value)q"). | |
| 543 D("value", v).D("dep", deployment).D("comp", com
ponent).Err() | |
| 544 } | |
| 545 return dep, comp, nil | |
| 546 } | |
| 547 return dep, nil, nil | |
| 548 } | |
| 549 | |
| 550 func (l *deployLayout) matchDeploymentComponent(m string, cb func(*layoutDeploym
ent, *layoutDeploymentComponent)) error { | |
| 551 for _, depName := range l.deploymentNames { | |
| 552 dep := l.deployments[depName] | |
| 553 | |
| 554 matched, err := filepath.Match(m, dep.String()) | |
| 555 if err != nil { | |
| 556 return errors.Annotate(err).Reason("failed to match %(pa
ttern)q").D("pattern", m).Err() | |
| 557 } | |
| 558 if matched { | |
| 559 // Matches entire deployment. | |
| 560 cb(dep, nil) | |
| 561 continue | |
| 562 } | |
| 563 | |
| 564 // Try each of the deployment's components. | |
| 565 for _, compName := range dep.componentNames { | |
| 566 comp := dep.components[compName] | |
| 567 | |
| 568 matched, err := filepath.Match(m, comp.String()) | |
| 569 if err != nil { | |
| 570 return errors.Annotate(err).Reason("failed to ma
tch %(pattern)q").D("pattern", m).Err() | |
| 571 } | |
| 572 if matched { | |
| 573 cb(dep, comp) | |
| 574 } | |
| 575 } | |
| 576 } | |
| 577 | |
| 578 return nil | |
| 579 } | |
| 580 | |
| 581 func (l *deployLayout) load(c context.Context, path string) error { | |
| 582 if path == "" { | |
| 583 wd, err := os.Getwd() | |
| 584 if err != nil { | |
| 585 return err | |
| 586 } | |
| 587 path, err = findLayout(defaultLayoutFilename, wd) | |
| 588 if err != nil { | |
| 589 return err | |
| 590 } | |
| 591 } | |
| 592 | |
| 593 if err := unmarshalTextProtobuf(path, &l.Layout); err != nil { | |
| 594 return err | |
| 595 } | |
| 596 | |
| 597 // Load the user config, if available. | |
| 598 if err := loadUserConfig(c, &l.user); err != nil { | |
| 599 return errors.Annotate(err).Reason("failed to load user config")
.Err() | |
| 600 } | |
| 601 if len(l.user.SourceOverride) > 0 { | |
| 602 l.userSourceOverrides = make(map[string]*deploy.Source, len(l.us
er.SourceOverride)) | |
| 603 for k, v := range l.user.SourceOverride { | |
| 604 l.userSourceOverrides[k] = v | |
| 605 } | |
| 606 } | |
| 607 | |
| 608 // Populate with defaults. | |
| 609 l.basePath = filepath.Dir(path) | |
| 610 if l.SourcesPath == "" { | |
| 611 l.SourcesPath = filepath.Join(l.basePath, "sources") | |
| 612 } | |
| 613 if l.ApplicationsPath == "" { | |
| 614 l.ApplicationsPath = filepath.Join(l.basePath, "applications") | |
| 615 } | |
| 616 if l.DeploymentsPath == "" { | |
| 617 l.DeploymentsPath = filepath.Join(l.basePath, "deployments") | |
| 618 } | |
| 619 | |
| 620 if l.WorkingPath == "" { | |
| 621 l.WorkingPath = filepath.Join(l.basePath, ".working") | |
| 622 } else { | |
| 623 if !filepath.IsAbs(l.WorkingPath) { | |
| 624 l.WorkingPath = filepath.Join(l.basePath, l.WorkingPath) | |
| 625 } | |
| 626 } | |
| 627 | |
| 628 absWorkingPath, err := filepath.Abs(l.WorkingPath) | |
| 629 if err != nil { | |
| 630 return errors.Annotate(err).Reason("failed to resolve absolute p
ath for %(path)q"). | |
| 631 D("path", l.WorkingPath).Err() | |
| 632 } | |
| 633 l.WorkingPath = absWorkingPath | |
| 634 return nil | |
| 635 } | |
| 636 | |
| 637 func (l *deployLayout) initFrozenCheckout(c context.Context) (*deploy.FrozenLayo
ut, error) { | |
| 638 fis, err := ioutil.ReadDir(l.SourcesPath) | |
| 639 if err != nil { | |
| 640 return nil, errors.Annotate(err).Reason("failed to read director
y").Err() | |
| 641 } | |
| 642 | |
| 643 // Build internal and frozen layout in parallel. | |
| 644 var frozen deploy.FrozenLayout | |
| 645 frozen.SourceGroup = make(map[string]*deploy.FrozenLayout_SourceGroup, l
en(fis)) | |
| 646 | |
| 647 for _, fi := range fis { | |
| 648 name := title(fi.Name()) | |
| 649 path := filepath.Join(l.SourcesPath, string(name)) | |
| 650 | |
| 651 if !fi.IsDir() { | |
| 652 log.Fields{ | |
| 653 "path": path, | |
| 654 }.Warningf(c, "Skipping non-directory in source group di
rectory.") | |
| 655 continue | |
| 656 } | |
| 657 | |
| 658 if err := name.validate(); err != nil { | |
| 659 log.Fields{ | |
| 660 log.ErrorKey: err, | |
| 661 "title": name, | |
| 662 "path": path, | |
| 663 }.Warningf(c, "Skipping invalid source group title.") | |
| 664 continue | |
| 665 } | |
| 666 | |
| 667 // Load the Sources. | |
| 668 srcInfos, err := ioutil.ReadDir(path) | |
| 669 if err != nil { | |
| 670 log.Fields{ | |
| 671 log.ErrorKey: err, | |
| 672 "sourceGroup": name, | |
| 673 "path": path, | |
| 674 }.Warningf(c, "Could not read source group directory.") | |
| 675 continue | |
| 676 } | |
| 677 | |
| 678 sg := deploy.FrozenLayout_SourceGroup{ | |
| 679 Source: make(map[string]*deploy.FrozenLayout_Source, len
(srcInfos)), | |
| 680 } | |
| 681 | |
| 682 var srcBase deploy.Source | |
| 683 err = unmarshalTextProtobufDir(path, srcInfos, &srcBase, func(na
me string) error { | |
| 684 cpy := srcBase | |
| 685 | |
| 686 t, err := titleFromConfigPath(name) | |
| 687 if err != nil { | |
| 688 return errors.Annotate(err).Reason("invalid sour
ce title").Err() | |
| 689 } | |
| 690 | |
| 691 src := deploy.FrozenLayout_Source{ | |
| 692 Source: &cpy, | |
| 693 } | |
| 694 sg.Source[string(t)] = &src | |
| 695 return nil | |
| 696 }) | |
| 697 if err != nil { | |
| 698 return nil, errors.Annotate(err).Reason("failed to load
source group %(name)q from [%(path)s]"). | |
| 699 D("name", name).D("path", path).Err() | |
| 700 } | |
| 701 | |
| 702 frozen.SourceGroup[string(name)] = &sg | |
| 703 } | |
| 704 | |
| 705 return &frozen, nil | |
| 706 } | |
| 707 | |
| 708 func (l *deployLayout) loadFrozenLayout(c context.Context) error { | |
| 709 // Load the frozen configuration file from disk. | |
| 710 frozen, err := checkoutFrozen(l) | |
| 711 if err != nil { | |
| 712 return errors.Annotate(err).Reason("failed to load frozen checko
ut").Err() | |
| 713 } | |
| 714 | |
| 715 // Load our source groups and sources. | |
| 716 l.sourceGroups = make(map[title]*layoutSourceGroup, len(frozen.SourceGro
up)) | |
| 717 for sgName, fsg := range frozen.SourceGroup { | |
| 718 sg := layoutSourceGroup{ | |
| 719 FrozenLayout_SourceGroup: fsg, | |
| 720 layout: l, | |
| 721 title: title(sgName), | |
| 722 sources: make(map[title]*layoutSource, len(fsg.Source)), | |
| 723 } | |
| 724 | |
| 725 for srcName, fs := range fsg.Source { | |
| 726 src := layoutSource{ | |
| 727 FrozenLayout_Source: fs, | |
| 728 sg: &sg, | |
| 729 title: title(srcName), | |
| 730 } | |
| 731 sg.sources[src.title] = &src | |
| 732 } | |
| 733 | |
| 734 l.sourceGroups[sg.title] = &sg | |
| 735 } | |
| 736 | |
| 737 // Build our internally-connected structures from the frozen layout. | |
| 738 if err := l.loadApps(c); err != nil { | |
| 739 return errors.Annotate(err).Reason("failed to load applications
from [%(path)s]"). | |
| 740 D("path", l.ApplicationsPath).Err() | |
| 741 } | |
| 742 if err := l.loadDeployments(c); err != nil { | |
| 743 return errors.Annotate(err).Reason("failed to load deployments f
rom [%(path)s]"). | |
| 744 D("path", l.DeploymentsPath).Err() | |
| 745 } | |
| 746 return nil | |
| 747 } | |
| 748 | |
| 749 func (l *deployLayout) loadApps(c context.Context) error { | |
| 750 fis, err := ioutil.ReadDir(l.ApplicationsPath) | |
| 751 if err != nil { | |
| 752 return errors.Annotate(err).Reason("failed to read directory").E
rr() | |
| 753 } | |
| 754 apps := make(map[title]*deploy.Application, len(fis)) | |
| 755 var appBase deploy.Application | |
| 756 err = unmarshalTextProtobufDir(l.ApplicationsPath, fis, &appBase, func(n
ame string) error { | |
| 757 cpy := appBase | |
| 758 | |
| 759 t, err := titleFromConfigPath(name) | |
| 760 if err != nil { | |
| 761 return errors.Annotate(err).Reason("invalid application
title").Err() | |
| 762 } | |
| 763 | |
| 764 apps[t] = &cpy | |
| 765 return nil | |
| 766 }) | |
| 767 if err != nil { | |
| 768 return err | |
| 769 } | |
| 770 | |
| 771 l.apps = make(map[title]*layoutApp, len(apps)) | |
| 772 for t, app := range apps { | |
| 773 proj := layoutApp{ | |
| 774 Application: app, | |
| 775 title: t, | |
| 776 components: make(map[title]*layoutAppComponent, len(app
.Component)), | |
| 777 } | |
| 778 | |
| 779 // Initialize components. These can't actually be populated unti
l we have | |
| 780 // loaded our sources. | |
| 781 for _, comp := range proj.Component { | |
| 782 compT := title(comp.Name) | |
| 783 if err := compT.validate(); err != nil { | |
| 784 return errors.Annotate(err).Reason("application
%(app)q component %(component)q is not a valid component title"). | |
| 785 D("app", t).D("component", comp.Name).Er
r() | |
| 786 } | |
| 787 proj.components[compT] = &layoutAppComponent{ | |
| 788 Application_Component: comp, | |
| 789 proj: &proj, | |
| 790 title: compT, | |
| 791 } | |
| 792 } | |
| 793 | |
| 794 l.apps[t] = &proj | |
| 795 } | |
| 796 return nil | |
| 797 } | |
| 798 | |
| 799 func (l *deployLayout) loadDeployments(c context.Context) error { | |
| 800 fis, err := ioutil.ReadDir(l.DeploymentsPath) | |
| 801 if err != nil { | |
| 802 return errors.Annotate(err).Reason("failed to read directory").E
rr() | |
| 803 } | |
| 804 deployments := make(map[title]*deploy.Deployment, len(fis)) | |
| 805 var deploymentBase deploy.Deployment | |
| 806 err = unmarshalTextProtobufDir(l.DeploymentsPath, fis, &deploymentBase,
func(name string) error { | |
| 807 cpy := deploymentBase | |
| 808 | |
| 809 t, err := titleFromConfigPath(name) | |
| 810 if err != nil { | |
| 811 return errors.Annotate(err).Reason("invalid deployment t
itle").Err() | |
| 812 } | |
| 813 | |
| 814 deployments[t] = &cpy | |
| 815 return nil | |
| 816 }) | |
| 817 if err != nil { | |
| 818 return err | |
| 819 } | |
| 820 | |
| 821 l.deployments = make(map[title]*layoutDeployment, len(deployments)) | |
| 822 for t, d := range deployments { | |
| 823 dep, err := l.loadDeployment(t, d) | |
| 824 if err != nil { | |
| 825 return errors.Annotate(err).Reason("failed to load deplo
yment (%(deployment)q)"). | |
| 826 D("deployment", t).Err() | |
| 827 } | |
| 828 l.deployments[t] = dep | |
| 829 l.deploymentNames = append(l.deploymentNames, dep.title) | |
| 830 } | |
| 831 | |
| 832 return nil | |
| 833 } | |
| 834 | |
| 835 func (l *deployLayout) loadDeployment(t title, d *deploy.Deployment) (*layoutDep
loyment, error) { | |
| 836 dep := layoutDeployment{ | |
| 837 Deployment: d, | |
| 838 l: l, | |
| 839 title: t, | |
| 840 sg: l.sourceGroups[title(d.SourceGroup)], | |
| 841 } | |
| 842 if dep.sg == nil { | |
| 843 return nil, errors.Reason("unknown source group %(sourceGroup)q"
). | |
| 844 D("sourceGroup", d.SourceGroup).Err() | |
| 845 } | |
| 846 | |
| 847 // Resolve our application. | |
| 848 dep.app = l.apps[title(dep.Application)] | |
| 849 if dep.app == nil { | |
| 850 return nil, errors.Reason("unknown application %(app)q").D("app"
, dep.Application).Err() | |
| 851 } | |
| 852 | |
| 853 // Initialize our Components. Their protobufs cannot not be loaded until | |
| 854 // we have a checkout. | |
| 855 dep.components = make(map[title]*layoutDeploymentComponent, len(dep.app.
components)) | |
| 856 for compTitle, projComp := range dep.app.components { | |
| 857 comp := layoutDeploymentComponent{ | |
| 858 reldir: deployDirname(projComp.Path), | |
| 859 dep: &dep, | |
| 860 comp: projComp, | |
| 861 sources: []*layoutSource{dep.sg.sources[title(projComp.S
ource)]}, | |
| 862 } | |
| 863 if comp.sources[0] == nil { | |
| 864 return nil, errors.Reason("application references non-ex
istent source %(source)q"). | |
| 865 D("source", projComp.Source).Err() | |
| 866 } | |
| 867 for _, os := range projComp.OtherSource { | |
| 868 src := dep.sg.sources[title(os)] | |
| 869 if src == nil { | |
| 870 return nil, errors.Reason("application reference
s non-existent other source %(source)q"). | |
| 871 D("source", os).Err() | |
| 872 } | |
| 873 comp.sources = append(comp.sources, src) | |
| 874 } | |
| 875 | |
| 876 dep.components[compTitle] = &comp | |
| 877 dep.componentNames = append(dep.componentNames, compTitle) | |
| 878 } | |
| 879 | |
| 880 // Build a map of cloud project names. | |
| 881 if dep.CloudProject != nil { | |
| 882 cp := layoutDeploymentCloudProject{ | |
| 883 Deployment_CloudProject: dep.CloudProject, | |
| 884 dep: &dep, | |
| 885 } | |
| 886 if l.cloudProjects == nil { | |
| 887 l.cloudProjects = make(map[string]*layoutDeploymentCloud
Project) | |
| 888 } | |
| 889 if cur, ok := l.cloudProjects[cp.Name]; ok { | |
| 890 return nil, errors.Reason("cloud project %(name)q define
d by both %(curDep)q and %(thisDep)q"). | |
| 891 D("name", cp.Name).D("curDep", cur.dep.title).D(
"thisDep", dep.title).Err() | |
| 892 } | |
| 893 l.cloudProjects[cp.Name] = &cp | |
| 894 | |
| 895 if len(dep.CloudProject.GkeCluster) > 0 { | |
| 896 cp.gkeClusters = make(map[string]*layoutDeploymentGKEClu
ster, len(dep.CloudProject.GkeCluster)) | |
| 897 for _, gke := range dep.CloudProject.GkeCluster { | |
| 898 gkeCluster := layoutDeploymentGKECluster{ | |
| 899 Deployment_CloudProject_GKECluster: gke, | |
| 900 cloudProject: &cp, | |
| 901 } | |
| 902 | |
| 903 // Bind Components to their GKE cluster. | |
| 904 for _, b := range gke.Pod { | |
| 905 comp := dep.components[title(b.Name)] | |
| 906 switch { | |
| 907 case comp == nil: | |
| 908 return nil, errors.Reason("unkno
wn component %(comp)q for cluster %(name)q"). | |
| 909 D("comp", b.Name).D("nam
e", gke.Name).Err() | |
| 910 | |
| 911 case b.Replicas <= 0: | |
| 912 return nil, errors.Reason("GKE c
omponent %(comp)q must have at least 1 replica"). | |
| 913 D("comp", b.Name).Err() | |
| 914 } | |
| 915 | |
| 916 bp := &layoutDeploymentGKEPodBinding{ | |
| 917 Deployment_CloudProject_GKEClust
er_PodBinding: b, | |
| 918 cluster: &gkeCluster, | |
| 919 } | |
| 920 comp.gkePods = append(comp.gkePods, bp) | |
| 921 gkeCluster.pods = append(gkeCluster.pods
, bp) | |
| 922 } | |
| 923 | |
| 924 cp.gkeClusters[gke.Name] = &gkeCluster | |
| 925 } | |
| 926 } | |
| 927 | |
| 928 dep.cloudProject = &cp | |
| 929 } | |
| 930 | |
| 931 return &dep, nil | |
| 932 } | |
| 933 | |
| 934 // allSourceGroups returns a sorted list of all of the source group names | |
| 935 // in the layout. | |
| 936 func (l *deployLayout) allSourceGroups() []*layoutSourceGroup { | |
| 937 titles := make([]string, 0, len(l.sourceGroups)) | |
| 938 for t := range l.sourceGroups { | |
| 939 titles = append(titles, string(t)) | |
| 940 } | |
| 941 sort.Strings(titles) | |
| 942 | |
| 943 result := make([]*layoutSourceGroup, len(titles)) | |
| 944 for i, t := range titles { | |
| 945 result[i] = l.sourceGroups[title(t)] | |
| 946 } | |
| 947 return result | |
| 948 } | |
| 949 | |
| 950 // findLayout looks in the current working directory and ascends towards the | |
| 951 // root filesystem looking for a "layout.cfg" file. | |
| 952 func findLayout(filename, dir string) (string, error) { | |
| 953 for { | |
| 954 path := filepath.Join(dir, filename) | |
| 955 | |
| 956 switch st, err := os.Stat(path); { | |
| 957 case err == nil: | |
| 958 if !st.IsDir() { | |
| 959 return path, nil | |
| 960 } | |
| 961 | |
| 962 case isNotExist(err): | |
| 963 break | |
| 964 | |
| 965 default: | |
| 966 return "", errors.Annotate(err).Reason("failed to state
%(path)q"). | |
| 967 D("path", path).Err() | |
| 968 } | |
| 969 | |
| 970 // Walk up one directory. | |
| 971 oldDir := dir | |
| 972 dir, _ = filepath.Split(dir) | |
| 973 if oldDir == dir { | |
| 974 return "", errors.Reason("could not find %(filename)q st
arting from %(dir)q"). | |
| 975 D("filename", filename).D("dir", dir).Err() | |
| 976 } | |
| 977 } | |
| 978 } | |
| 979 | |
| 980 type componentRegistrar interface { | |
| 981 addGAEModule(*layoutDeploymentGAEModule) | |
| 982 addGKEPod(*layoutDeploymentGKEPodBinding) | |
| 983 } | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 2182213002:
deploytool: Add README.md, migrate docs to it. (Closed)
Base URL: https://github.com/luci/luci-go@master