| OLD | NEW |
|---|
| (Empty) |
| 1 // Copyright 2015 The Chromium Authors. All rights reserved. | |
| 2 // Use of this source code is governed by a BSD-style license that can be | |
| 3 // found in the LICENSE file. | |
| 4 | |
| 5 package cipd | |
| 6 | |
| 7 import ( | |
| 8 "fmt" | |
| 9 "io" | |
| 10 "io/ioutil" | |
| 11 "path/filepath" | |
| 12 "regexp" | |
| 13 "sort" | |
| 14 | |
| 15 "github.com/go-yaml/yaml" | |
| 16 ) | |
| 17 | |
| 18 // PackageDef defines how exactly to build a package: what files to put into it, | |
| 19 // how to name them, how to name the package itself, etc. It is loaded from | |
| 20 // *.yaml file. | |
| 21 type PackageDef struct { | |
| 22 // Package defines a name of the package. | |
| 23 Package string | |
| 24 // Root defines where to search for files, relative to package file itse
lf. | |
| 25 Root string | |
| 26 // Data describe what to add to the package. | |
| 27 Data []PackageChunkDef | |
| 28 } | |
| 29 | |
| 30 // PackageChunkDef represents one entry in 'data' section of package definition | |
| 31 // file. It is either a single file, or a recursively scanned directory (with | |
| 32 // optional list of regexps for files to skip). | |
| 33 type PackageChunkDef struct { | |
| 34 // Dir is a directory to add to the package (recursively). | |
| 35 Dir string | |
| 36 // File is a single file to add to the package. | |
| 37 File string | |
| 38 // Exclude is a list of glob patterns to exclude when scanning a directo
ry. | |
| 39 Exclude []string | |
| 40 } | |
| 41 | |
| 42 // LoadPackageDef loads package definition from a YAML source code. In | |
| 43 // substitutes %{...} strings in the definition with corresponding values | |
| 44 // from 'vars' map. | |
| 45 func LoadPackageDef(r io.Reader, vars map[string]string) (out PackageDef, err er
ror) { | |
| 46 data, err := ioutil.ReadAll(r) | |
| 47 if err != nil { | |
| 48 return | |
| 49 } | |
| 50 err = yaml.Unmarshal(data, &out) | |
| 51 if err != nil { | |
| 52 return | |
| 53 } | |
| 54 | |
| 55 // Substitute variables in all strings. | |
| 56 for _, str := range out.strings() { | |
| 57 *str, err = subVars(*str, vars) | |
| 58 if err != nil { | |
| 59 return | |
| 60 } | |
| 61 } | |
| 62 | |
| 63 // Validate package name right away. | |
| 64 err = ValidatePackageName(out.Package) | |
| 65 if err != nil { | |
| 66 return | |
| 67 } | |
| 68 | |
| 69 // Make sure "file" or "dir" are used, but not both. | |
| 70 for i, chunk := range out.Data { | |
| 71 if chunk.File == "" && chunk.Dir == "" { | |
| 72 return out, fmt.Errorf("files entry #%d needs 'file' or
'dir' key", i) | |
| 73 } | |
| 74 if chunk.File != "" && chunk.Dir != "" { | |
| 75 return out, fmt.Errorf("files entry #%d can't have both
'files' and 'dir' keys", i) | |
| 76 } | |
| 77 } | |
| 78 | |
| 79 // Default 'root' to a directory with the package def file. | |
| 80 if out.Root == "" { | |
| 81 out.Root = "." | |
| 82 } | |
| 83 | |
| 84 return | |
| 85 } | |
| 86 | |
| 87 // FindFiles scans files system and returns all files to be added to the | |
| 88 // package. It uses a path to package definition file directory ('cwd' argument) | |
| 89 // to find a root of the package. | |
| 90 func (def *PackageDef) FindFiles(cwd string) ([]File, error) { | |
| 91 // Root of the package is defined relative to package def YAML file. | |
| 92 absCwd, err := filepath.Abs(cwd) | |
| 93 if err != nil { | |
| 94 return nil, err | |
| 95 } | |
| 96 root := filepath.Clean(filepath.Join(absCwd, filepath.FromSlash(def.Root
))) | |
| 97 | |
| 98 // Helper to get absolute path to a file given path relative to root. | |
| 99 makeAbs := func(p string) string { | |
| 100 return filepath.Join(root, filepath.FromSlash(p)) | |
| 101 } | |
| 102 | |
| 103 // Used to skip duplicates. | |
| 104 seen := map[string]File{} | |
| 105 add := func(f File) { | |
| 106 if seen[f.Name()] == nil { | |
| 107 seen[f.Name()] = f | |
| 108 } | |
| 109 } | |
| 110 | |
| 111 log.Info("Enumerating files to zip...") | |
| 112 for _, chunk := range def.Data { | |
| 113 // Individual file. | |
| 114 if chunk.File != "" { | |
| 115 file, err := WrapFile(makeAbs(chunk.File), root, nil) | |
| 116 if err != nil { | |
| 117 return nil, err | |
| 118 } | |
| 119 add(file) | |
| 120 continue | |
| 121 } | |
| 122 | |
| 123 // A subdirectory to scan (with filtering). | |
| 124 if chunk.Dir != "" { | |
| 125 // Absolute path to directory to scan. | |
| 126 startDir := makeAbs(chunk.Dir) | |
| 127 // Exclude files as specified in 'exclude' section. | |
| 128 exclude, err := makeExclusionFilter(startDir, chunk.Excl
ude) | |
| 129 if err != nil { | |
| 130 return nil, err | |
| 131 } | |
| 132 // Run the scan. | |
| 133 files, err := ScanFileSystem(startDir, root, exclude) | |
| 134 if err != nil { | |
| 135 return nil, err | |
| 136 } | |
| 137 for _, f := range files { | |
| 138 add(f) | |
| 139 } | |
| 140 continue | |
| 141 } | |
| 142 | |
| 143 // LoadPackageDef does validation, so this should not happen. | |
| 144 return nil, fmt.Errorf("Unexpected definition: %v", chunk) | |
| 145 } | |
| 146 | |
| 147 // Sort by Name(). | |
| 148 names := make([]string, 0, len(seen)) | |
| 149 for n := range seen { | |
| 150 names = append(names, n) | |
| 151 } | |
| 152 sort.Strings(names) | |
| 153 | |
| 154 // Final sorted array of File. | |
| 155 out := make([]File, 0, len(names)) | |
| 156 for _, n := range names { | |
| 157 out = append(out, seen[n]) | |
| 158 } | |
| 159 return out, nil | |
| 160 } | |
| 161 | |
| 162 // makeExclusionFilter produces a predicate that checks an absolute file path | |
| 163 // against a list of regexps (defined against slash separated paths relative to | |
| 164 // 'startDir'). The predicate takes absolute native path, converts it to slash | |
| 165 // separated path relative to 'startDir' and checks against list of regexps in | |
| 166 // 'patterns'. Returns true on match. | |
| 167 func makeExclusionFilter(startDir string, patterns []string) (ScanFilter, error)
{ | |
| 168 if len(patterns) == 0 { | |
| 169 return nil, nil | |
| 170 } | |
| 171 | |
| 172 // Compile regular expressions. | |
| 173 exps := []*regexp.Regexp{} | |
| 174 for _, expr := range patterns { | |
| 175 if expr == "" { | |
| 176 continue | |
| 177 } | |
| 178 if expr[0] != '^' { | |
| 179 expr = "^" + expr | |
| 180 } | |
| 181 if expr[len(expr)-1] != '$' { | |
| 182 expr = expr + "$" | |
| 183 } | |
| 184 re, err := regexp.Compile(expr) | |
| 185 if err != nil { | |
| 186 return nil, err | |
| 187 } | |
| 188 exps = append(exps, re) | |
| 189 } | |
| 190 | |
| 191 return func(abs string) bool { | |
| 192 rel, err := filepath.Rel(startDir, abs) | |
| 193 if err != nil { | |
| 194 log.Warnf("Unexpected error when evaluating %s: %s", abs
, err) | |
| 195 return true | |
| 196 } | |
| 197 // Do not evaluate paths outside of startDir. | |
| 198 rel = filepath.ToSlash(rel) | |
| 199 if rel[:3] == "../" { | |
| 200 return false | |
| 201 } | |
| 202 for _, exp := range exps { | |
| 203 if exp.MatchString(rel) { | |
| 204 return true | |
| 205 } | |
| 206 } | |
| 207 return false | |
| 208 }, nil | |
| 209 } | |
| 210 | |
| 211 //////////////////////////////////////////////////////////////////////////////// | |
| 212 // Variable substitution. | |
| 213 | |
| 214 var subVarsRe = regexp.MustCompile(`\$\{[^\}]+\}`) | |
| 215 | |
| 216 // strings return array of pointers to all strings in PackageDef that can | |
| 217 // contain ${var} variables. | |
| 218 func (def *PackageDef) strings() []*string { | |
| 219 out := []*string{ | |
| 220 &def.Package, | |
| 221 &def.Root, | |
| 222 } | |
| 223 // Important to use index here, to get a point to a real object, not its
copy. | |
| 224 for i := range def.Data { | |
| 225 out = append(out, def.Data[i].strings()...) | |
| 226 } | |
| 227 return out | |
| 228 } | |
| 229 | |
| 230 // strings return array of pointers to all strings in PackageChunkDef that can | |
| 231 // contain ${var} variables. | |
| 232 func (def *PackageChunkDef) strings() []*string { | |
| 233 out := []*string{ | |
| 234 &def.Dir, | |
| 235 &def.File, | |
| 236 } | |
| 237 for i := range def.Exclude { | |
| 238 out = append(out, &def.Exclude[i]) | |
| 239 } | |
| 240 return out | |
| 241 } | |
| 242 | |
| 243 // subVars replaces "${key}" in strings with values from 'vars' map. Returns | |
| 244 // error if some keys weren't found in 'vars' map. | |
| 245 func subVars(s string, vars map[string]string) (string, error) { | |
| 246 badKeys := []string{} | |
| 247 res := subVarsRe.ReplaceAllStringFunc(s, func(match string) string { | |
| 248 // Strip '${' and '}'. | |
| 249 key := match[2 : len(match)-1] | |
| 250 val, ok := vars[key] | |
| 251 if !ok { | |
| 252 badKeys = append(badKeys, key) | |
| 253 return match | |
| 254 } | |
| 255 return val | |
| 256 }) | |
| 257 if len(badKeys) != 0 { | |
| 258 return res, fmt.Errorf("Values for some variables are not provid
ed: %v", badKeys) | |
| 259 } | |
| 260 return res, nil | |
| 261 } | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 1129043003:
cipd: Refactor client to make it more readable. (Closed)
Base URL: https://chromium.googlesource.com/infra/infra.git@master