| OLD | NEW |
|---|
| (Empty) |
| 1 // Copyright 2014 The Chromium Authors. All rights reserved. | |
| 2 // Use of this source code is governed by a BSD-style license that can be | |
| 3 // found in the LICENSE file. | |
| 4 | |
| 5 package cipd | |
| 6 | |
| 7 import ( | |
| 8 "crypto/sha1" | |
| 9 "encoding/base64" | |
| 10 "fmt" | |
| 11 "io/ioutil" | |
| 12 "os" | |
| 13 "path/filepath" | |
| 14 "sort" | |
| 15 "strings" | |
| 16 "sync" | |
| 17 "time" | |
| 18 ) | |
| 19 | |
| 20 // TODO(vadimsh): Make it work on Windows, verify it works on Mac. | |
| 21 | |
| 22 // TODO(vadimsh): How to handle path conflicts between two packages? Currently | |
| 23 // the last one installed wins. | |
| 24 | |
| 25 // File system layout of a site directory <root>: | |
| 26 // <root>/.cipd/pkgs/ | |
| 27 // <package name digest>/ | |
| 28 // _current -> symlink to fea3ab83440e9dfb813785e16d4101f331ed44f4 | |
| 29 // fea3ab83440e9dfb813785e16d4101f331ed44f4/ | |
| 30 // bin/ | |
| 31 // tool | |
| 32 // ... | |
| 33 // ... | |
| 34 // bin/ | |
| 35 // tool -> symlink to ../.cipd/pkgs/<package name digest>/_current/bin/tool | |
| 36 // ... | |
| 37 // | |
| 38 // Where <package name digest> is derived from a package name. It doesn't have | |
| 39 // to be reversible though, since the package name is still stored in the | |
| 40 // installed package manifest and can be read from there. | |
| 41 // | |
| 42 // Some efforts are made to make sure that during the deployment a window of | |
| 43 // inconsistency in the file system is as small as possible. | |
| 44 | |
| 45 // Subdirectory of site root to extract packages to. | |
| 46 const packagesDir = siteServiceDir + "/pkgs" | |
| 47 | |
| 48 // Name of a symlink that points to latest deployed version. | |
| 49 const currentSymlink = "_current" | |
| 50 | |
| 51 // PackageState contains information about single deployed package. | |
| 52 type PackageState struct { | |
| 53 // PackageName identifies the package. | |
| 54 PackageName string | |
| 55 // InstanceID is ID of the installed package instance (SHA1 of package c
ontents). | |
| 56 InstanceID string | |
| 57 } | |
| 58 | |
| 59 // DeployInstance installs a specific instance of a package (identified by | |
| 60 // InstanceID()) into a site root directory. It unpacks the package into | |
| 61 // <root>/.cipd/pkgs/*, and rearranges symlinks to point to unpacked files. | |
| 62 // It tries to make it as "atomic" as possibly. | |
| 63 func DeployInstance(root string, inst PackageInstance) (state PackageState, err
error) { | |
| 64 root, err = filepath.Abs(filepath.Clean(root)) | |
| 65 if err != nil { | |
| 66 return | |
| 67 } | |
| 68 log.Infof("Deploying %s:%s into %s", inst.PackageName(), inst.InstanceID
(), root) | |
| 69 | |
| 70 // Be paranoid. | |
| 71 err = ValidatePackageName(inst.PackageName()) | |
| 72 if err != nil { | |
| 73 return | |
| 74 } | |
| 75 err = ValidateInstanceID(inst.InstanceID()) | |
| 76 if err != nil { | |
| 77 return | |
| 78 } | |
| 79 | |
| 80 // Remember currently deployed version (to remove it later). Do not frea
k out | |
| 81 // if it's not there (prevID is "" in that case). | |
| 82 oldFiles := makeStringSet() | |
| 83 prevID := findDeployedInstance(root, inst.PackageName(), oldFiles) | |
| 84 | |
| 85 // Extract new version to a final destination. | |
| 86 newFiles := makeStringSet() | |
| 87 destPath, err := deployInstance(root, inst, newFiles) | |
| 88 if err != nil { | |
| 89 return | |
| 90 } | |
| 91 | |
| 92 // Switch '_current' symlink to point to a new package instance. It is a | |
| 93 // point of no return. The function must not fail going forward. | |
| 94 mainSymlinkPath := packagePath(root, inst.PackageName(), currentSymlink) | |
| 95 err = ensureSymlink(mainSymlinkPath, inst.InstanceID()) | |
| 96 if err != nil { | |
| 97 ensureDirectoryGone(destPath) | |
| 98 return | |
| 99 } | |
| 100 | |
| 101 // Asynchronously remove previous version (best effort). | |
| 102 wg := sync.WaitGroup{} | |
| 103 defer wg.Wait() | |
| 104 if prevID != "" && prevID != inst.InstanceID() { | |
| 105 wg.Add(1) | |
| 106 go func() { | |
| 107 defer wg.Done() | |
| 108 ensureDirectoryGone(packagePath(root, inst.PackageName()
, prevID)) | |
| 109 }() | |
| 110 } | |
| 111 | |
| 112 log.Infof("Adjusting symlinks for %s", inst.PackageName()) | |
| 113 | |
| 114 // Make symlinks in the site directory for all new files. Reference a pa
ckage | |
| 115 // root via '_current' symlink (instead of direct destPath), to make | |
| 116 // subsequent updates 'more atomic' (since they'll need to switch only | |
| 117 // '_current' symlink to update _all_ files in the site root at once). | |
| 118 linkFilesToRoot(root, mainSymlinkPath, newFiles) | |
| 119 | |
| 120 // Delete symlinks to files no longer needed i.e. set(old) - set(new). | |
| 121 for relPath := range oldFiles.diff(newFiles) { | |
| 122 ensureFileGone(filepath.Join(root, relPath)) | |
| 123 } | |
| 124 | |
| 125 // Verify it's all right, read the manifest. | |
| 126 state, err = CheckDeployed(root, inst.PackageName()) | |
| 127 if err == nil && state.InstanceID != inst.InstanceID() { | |
| 128 err = fmt.Errorf("Other package instance (%s) was deployed concu
rrently", state.InstanceID) | |
| 129 } | |
| 130 if err == nil { | |
| 131 log.Infof("Successfully deployed %s:%s", inst.PackageName(), ins
t.InstanceID()) | |
| 132 } else { | |
| 133 log.Errorf("Failed to deploy %s:%s: %s", inst.PackageName(), ins
t.InstanceID(), err.Error()) | |
| 134 } | |
| 135 return | |
| 136 } | |
| 137 | |
| 138 // CheckDeployed checks whether a given package is deployed and returns | |
| 139 // information about it if it is. | |
| 140 func CheckDeployed(root string, pkg string) (state PackageState, err error) { | |
| 141 state, err = readPackageState(packagePath(root, pkg)) | |
| 142 if err != nil { | |
| 143 return | |
| 144 } | |
| 145 if state.PackageName != pkg { | |
| 146 err = fmt.Errorf("Package path and package name in the manifest
do not match") | |
| 147 } | |
| 148 return | |
| 149 } | |
| 150 | |
| 151 // FindDeployed returns a list of packages deployed to a site root. | |
| 152 func FindDeployed(root string) (out []PackageState, err error) { | |
| 153 root, err = filepath.Abs(filepath.Clean(root)) | |
| 154 if err != nil { | |
| 155 return | |
| 156 } | |
| 157 | |
| 158 // Directories with packages are direct children of .cipd/pkgs/. | |
| 159 pkgs := filepath.Join(root, filepath.FromSlash(packagesDir)) | |
| 160 infos, err := ioutil.ReadDir(pkgs) | |
| 161 if err != nil { | |
| 162 if os.IsNotExist(err) { | |
| 163 err = nil | |
| 164 return | |
| 165 } | |
| 166 return | |
| 167 } | |
| 168 | |
| 169 // Read the package name from the package manifest. Skip broken stuff. | |
| 170 found := map[string]PackageState{} | |
| 171 keys := []string{} | |
| 172 for _, info := range infos { | |
| 173 // Attempt to read the manifest. If it is there -> valid package
is found. | |
| 174 if info.IsDir() { | |
| 175 state, err := readPackageState(filepath.Join(pkgs, info.
Name())) | |
| 176 if err == nil { | |
| 177 // Ignore duplicate entries, they can appear if
someone messes with | |
| 178 // pkgs/* structure manually. | |
| 179 if _, ok := found[state.PackageName]; !ok { | |
| 180 keys = append(keys, state.PackageName) | |
| 181 found[state.PackageName] = state | |
| 182 } | |
| 183 } | |
| 184 } | |
| 185 } | |
| 186 | |
| 187 // Sort by package name. | |
| 188 sort.Strings(keys) | |
| 189 out = make([]PackageState, len(found)) | |
| 190 for i, k := range keys { | |
| 191 out[i] = found[k] | |
| 192 } | |
| 193 return | |
| 194 } | |
| 195 | |
| 196 // RemoveDeployed deletes a package given its name. | |
| 197 func RemoveDeployed(root string, packageName string) error { | |
| 198 root, err := filepath.Abs(filepath.Clean(root)) | |
| 199 if err != nil { | |
| 200 return err | |
| 201 } | |
| 202 log.Infof("Removing %s from %s", packageName, root) | |
| 203 | |
| 204 // Be paranoid. | |
| 205 err = ValidatePackageName(packageName) | |
| 206 if err != nil { | |
| 207 return err | |
| 208 } | |
| 209 | |
| 210 // Grab list of files in currently deployed package to unlink them from
root. | |
| 211 files := makeStringSet() | |
| 212 instanceID := findDeployedInstance(root, packageName, files) | |
| 213 | |
| 214 // If was installed, removed symlinks pointing to the package files. | |
| 215 if instanceID != "" { | |
| 216 for relPath := range files { | |
| 217 ensureFileGone(filepath.Join(root, relPath)) | |
| 218 } | |
| 219 } | |
| 220 | |
| 221 // Ensure all garbage is gone even if instanceID == "" was returned. | |
| 222 return ensureDirectoryGone(packagePath(root, packageName)) | |
| 223 } | |
| 224 | |
| 225 //////////////////////////////////////////////////////////////////////////////// | |
| 226 // Utility functions. | |
| 227 | |
| 228 // findDeployedInstance returns instanceID of a currently deployed package | |
| 229 // instance and finds all files in it (adding them to 'files' set). Returns "" | |
| 230 // if nothing is deployed. File paths in 'files' are relative to package root. | |
| 231 func findDeployedInstance(root string, pkg string, files stringSet) string { | |
| 232 state, err := CheckDeployed(root, pkg) | |
| 233 if err != nil { | |
| 234 return "" | |
| 235 } | |
| 236 scanPackageDir(packagePath(root, pkg, state.InstanceID), files) | |
| 237 return state.InstanceID | |
| 238 } | |
| 239 | |
| 240 // deployInstance atomically extracts a package instance to its final | |
| 241 // destination and returns a path to it. It writes a list of extracted files | |
| 242 // to 'files'. File paths in 'files' are relative to package root. | |
| 243 func deployInstance(root string, inst PackageInstance, files stringSet) (string,
error) { | |
| 244 // Extract new version to a final destination. ExtractPackageInstance kn
ows | |
| 245 // how to build full paths and how to atomically extract a package. No n
eed | |
| 246 // to delete garbage if it fails. | |
| 247 destPath := packagePath(root, inst.PackageName(), inst.InstanceID()) | |
| 248 err := ExtractInstance(inst, NewFileSystemDestination(destPath)) | |
| 249 if err != nil { | |
| 250 return "", err | |
| 251 } | |
| 252 // Enumerate files inside. Nuke it and fail if it's unreadable. | |
| 253 err = scanPackageDir(packagePath(root, inst.PackageName(), inst.Instance
ID()), files) | |
| 254 if err != nil { | |
| 255 ensureDirectoryGone(destPath) | |
| 256 return "", err | |
| 257 } | |
| 258 return destPath, err | |
| 259 } | |
| 260 | |
| 261 // linkFilesToRoot makes symlinks in root that point to files in packageRoot. | |
| 262 // All targets are specified by 'files' as paths relative to packageRoot. This | |
| 263 // function is best effort and thus doesn't return errors. | |
| 264 func linkFilesToRoot(root string, packageRoot string, files stringSet) { | |
| 265 for relPath := range files { | |
| 266 // E.g <root>/bin/tool. | |
| 267 symlinkAbs := filepath.Join(root, relPath) | |
| 268 // E.g. <root>/.cipd/pkgs/name/_current/bin/tool. | |
| 269 targetAbs := filepath.Join(packageRoot, relPath) | |
| 270 // E.g. ../.cipd/pkgs/name/_current/bin/tool. | |
| 271 targetRel, err := filepath.Rel(filepath.Dir(symlinkAbs), targetA
bs) | |
| 272 if err != nil { | |
| 273 log.Warnf("Can't get relative path from %s to %s", filep
ath.Dir(symlinkAbs), targetAbs) | |
| 274 continue | |
| 275 } | |
| 276 err = ensureSymlink(symlinkAbs, targetRel) | |
| 277 if err != nil { | |
| 278 log.Warnf("Failed to create symlink for %s", relPath) | |
| 279 continue | |
| 280 } | |
| 281 } | |
| 282 } | |
| 283 | |
| 284 // packagePath joins paths together to return absolute path to .cipd/pkgs sub pa
th. | |
| 285 func packagePath(root string, pkg string, rest ...string) string { | |
| 286 root, err := filepath.Abs(filepath.Clean(root)) | |
| 287 if err != nil { | |
| 288 panic(fmt.Sprintf("Can't get absolute path of '%s'", root)) | |
| 289 } | |
| 290 root = filepath.Join(root, filepath.FromSlash(packagesDir), packageNameD
igest(pkg)) | |
| 291 result := filepath.Join(append([]string{root}, rest...)...) | |
| 292 | |
| 293 // Be paranoid and check that everything is inside .cipd directory. | |
| 294 abs, err := filepath.Abs(result) | |
| 295 if err != nil { | |
| 296 panic(fmt.Sprintf("Can't get absolute path of '%s'", result)) | |
| 297 } | |
| 298 if !isSubpath(abs, root) { | |
| 299 panic(fmt.Sprintf("Wrong path %s outside of root %s", abs, root)
) | |
| 300 } | |
| 301 return result | |
| 302 } | |
| 303 | |
| 304 // packageNameDigest returns a filename to use for naming a package directory in | |
| 305 // the file system. Using package names as is can introduce problems on file | |
| 306 // systems with path length limits (on Windows in particular). Returns last two | |
| 307 // components of the package name + stripped SHA1 of the whole package name. | |
| 308 func packageNameDigest(pkg string) string { | |
| 309 // Be paranoid. | |
| 310 err := ValidatePackageName(pkg) | |
| 311 if err != nil { | |
| 312 panic(err.Error()) | |
| 313 } | |
| 314 | |
| 315 // Grab stripped SHA1 of the full package name. | |
| 316 h := sha1.New() | |
| 317 h.Write([]byte(pkg)) | |
| 318 hash := base64.URLEncoding.EncodeToString(h.Sum(nil))[:10] | |
| 319 | |
| 320 // Grab last <= 2 components of the package path. | |
| 321 chunks := strings.Split(pkg, "/") | |
| 322 if len(chunks) > 2 { | |
| 323 chunks = chunks[len(chunks)-2:] | |
| 324 } | |
| 325 | |
| 326 // Join together with '_' as separator. | |
| 327 chunks = append(chunks, hash) | |
| 328 return strings.Join(chunks, "_") | |
| 329 } | |
| 330 | |
| 331 // readPackageState reads package manifest of a deployed package instance and | |
| 332 // returns corresponding PackageState object. | |
| 333 func readPackageState(packageDir string) (state PackageState, err error) { | |
| 334 // Resolve _current symlink to a concrete instance ID. | |
| 335 current, err := os.Readlink(filepath.Join(packageDir, currentSymlink)) | |
| 336 if err != nil { | |
| 337 return | |
| 338 } | |
| 339 err = ValidateInstanceID(current) | |
| 340 if err != nil { | |
| 341 err = fmt.Errorf("Symlink target doesn't look like a valid insta
nce id") | |
| 342 return | |
| 343 } | |
| 344 // Read the manifest from the instance directory. | |
| 345 manifestPath := filepath.Join(packageDir, current, filepath.FromSlash(ma
nifestName)) | |
| 346 r, err := os.Open(manifestPath) | |
| 347 if err != nil { | |
| 348 return | |
| 349 } | |
| 350 defer r.Close() | |
| 351 manifest, err := readManifest(r) | |
| 352 if err != nil { | |
| 353 return | |
| 354 } | |
| 355 state = PackageState{ | |
| 356 PackageName: manifest.PackageName, | |
| 357 InstanceID: current, | |
| 358 } | |
| 359 return | |
| 360 } | |
| 361 | |
| 362 // ensureSymlink atomically creates a symlink pointing to a target. It will | |
| 363 // create full directory path if necessary. | |
| 364 func ensureSymlink(symlink string, target string) error { | |
| 365 // Already set? | |
| 366 existing, err := os.Readlink(symlink) | |
| 367 if err != nil && existing == target { | |
| 368 return nil | |
| 369 } | |
| 370 | |
| 371 // Make sure path exists. | |
| 372 err = os.MkdirAll(filepath.Dir(symlink), 0777) | |
| 373 if err != nil { | |
| 374 return err | |
| 375 } | |
| 376 | |
| 377 // Create a new symlink file, can't modify existing one. | |
| 378 temp := fmt.Sprintf("%s_%v", symlink, time.Now().UnixNano()) | |
| 379 err = os.Symlink(target, temp) | |
| 380 if err != nil { | |
| 381 return err | |
| 382 } | |
| 383 | |
| 384 // Atomically replace current symlink with a new one. | |
| 385 err = os.Rename(temp, symlink) | |
| 386 if err != nil { | |
| 387 os.Remove(temp) | |
| 388 return err | |
| 389 } | |
| 390 | |
| 391 return nil | |
| 392 } | |
| 393 | |
| 394 // scanPackageDir finds a set of regular files (and symlinks) in a package | |
| 395 // instance directory to be symlinked into the site root. Adds paths relative | |
| 396 // to 'root' to 'out'. Skips package service directories (.cipdpkg and .cipd) | |
| 397 // since they contain package deployer gut files, not something that needs | |
| 398 // to be deployed. | |
| 399 func scanPackageDir(root string, out stringSet) error { | |
| 400 return filepath.Walk(root, func(path string, info os.FileInfo, err error
) error { | |
| 401 if err != nil { | |
| 402 return err | |
| 403 } | |
| 404 rel, err := filepath.Rel(root, path) | |
| 405 if err != nil { | |
| 406 return err | |
| 407 } | |
| 408 if rel == packageServiceDir || rel == siteServiceDir { | |
| 409 return filepath.SkipDir | |
| 410 } | |
| 411 if info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 { | |
| 412 out.add(rel) | |
| 413 } | |
| 414 return nil | |
| 415 }) | |
| 416 } | |
| 417 | |
| 418 // ensureDirectoryGone removes the directory as instantly as possible by | |
| 419 // renaming it first and only then recursively deleting. | |
| 420 func ensureDirectoryGone(path string) error { | |
| 421 temp := fmt.Sprintf("%s_%v", path, time.Now().UnixNano()) | |
| 422 err := os.Rename(path, temp) | |
| 423 if err != nil { | |
| 424 if !os.IsNotExist(err) { | |
| 425 log.Warnf("Failed to rename directory %s: %v", path, err
) | |
| 426 return err | |
| 427 } | |
| 428 return nil | |
| 429 } | |
| 430 err = os.RemoveAll(temp) | |
| 431 if err != nil { | |
| 432 log.Warnf("Failed to remove directory %s: %v", temp, err) | |
| 433 return err | |
| 434 } | |
| 435 return nil | |
| 436 } | |
| 437 | |
| 438 // ensureFileGone removes file, logging the errors (if any). | |
| 439 func ensureFileGone(path string) error { | |
| 440 err := os.Remove(path) | |
| 441 if err != nil && !os.IsNotExist(err) { | |
| 442 log.Warnf("Failed to remove %s", path) | |
| 443 return err | |
| 444 } | |
| 445 return nil | |
| 446 } | |
| 447 | |
| 448 //////////////////////////////////////////////////////////////////////////////// | |
| 449 // Simple stringSet implementation for keeping a set of filenames. | |
| 450 | |
| 451 type stringSet map[string]struct{} | |
| 452 | |
| 453 func makeStringSet() stringSet { | |
| 454 return make(stringSet) | |
| 455 } | |
| 456 | |
| 457 // add adds an element to the string set. | |
| 458 func (a stringSet) add(elem string) { | |
| 459 a[elem] = struct{}{} | |
| 460 } | |
| 461 | |
| 462 // diff returns set(a) - set(b). | |
| 463 func (a stringSet) diff(b stringSet) stringSet { | |
| 464 out := makeStringSet() | |
| 465 for elem := range a { | |
| 466 if _, ok := b[elem]; !ok { | |
| 467 out.add(elem) | |
| 468 } | |
| 469 } | |
| 470 return out | |
| 471 } | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 1129043003:
cipd: Refactor client to make it more readable. (Closed)
Base URL: https://chromium.googlesource.com/infra/infra.git@master