| OLD | NEW |
| 1 // Copyright 2017 The LUCI Authors. All rights reserved. | 1 // Copyright 2017 The LUCI Authors. All rights reserved. |
| 2 // Use of this source code is governed under the Apache License, Version 2.0 | 2 // Use of this source code is governed under the Apache License, Version 2.0 |
| 3 // that can be found in the LICENSE file. | 3 // that can be found in the LICENSE file. |
| 4 | 4 |
| 5 package spec | 5 package spec |
| 6 | 6 |
| 7 import ( | 7 import ( |
| 8 "bufio" | 8 "bufio" |
| 9 "io/ioutil" | 9 "io/ioutil" |
| 10 "os" | 10 "os" |
| (...skipping 27 matching lines...) Expand all Loading... |
| 38 // DefaultInlineBeginGuard is the default loader InlineBeginGuard value. | 38 // DefaultInlineBeginGuard is the default loader InlineBeginGuard value. |
| 39 DefaultInlineBeginGuard = "[VPYTHON:BEGIN]" | 39 DefaultInlineBeginGuard = "[VPYTHON:BEGIN]" |
| 40 // DefaultInlineEndGuard is the default loader InlineEndGuard value. | 40 // DefaultInlineEndGuard is the default loader InlineEndGuard value. |
| 41 DefaultInlineEndGuard = "[VPYTHON:END]" | 41 DefaultInlineEndGuard = "[VPYTHON:END]" |
| 42 ) | 42 ) |
| 43 | 43 |
| 44 // Load loads an specification file text protobuf from the supplied path. | 44 // Load loads an specification file text protobuf from the supplied path. |
| 45 func Load(path string, spec *vpython.Spec) error { | 45 func Load(path string, spec *vpython.Spec) error { |
| 46 content, err := ioutil.ReadFile(path) | 46 content, err := ioutil.ReadFile(path) |
| 47 if err != nil { | 47 if err != nil { |
| 48 » » return errors.Annotate(err).Reason("failed to load file from: %(
path)s"). | 48 » » return errors.Annotate(err, "failed to load file from: %s", path
).Err() |
| 49 » » » D("path", path). | |
| 50 » » » Err() | |
| 51 } | 49 } |
| 52 | 50 |
| 53 return Parse(string(content), spec) | 51 return Parse(string(content), spec) |
| 54 } | 52 } |
| 55 | 53 |
| 56 // Parse loads a specification message from a content string. | 54 // Parse loads a specification message from a content string. |
| 57 func Parse(content string, spec *vpython.Spec) error { | 55 func Parse(content string, spec *vpython.Spec) error { |
| 58 if err := cproto.UnmarshalTextML(content, spec); err != nil { | 56 if err := cproto.UnmarshalTextML(content, spec); err != nil { |
| 59 » » return errors.Annotate(err).Reason("failed to unmarshal vpython.
Spec").Err() | 57 » » return errors.Annotate(err, "failed to unmarshal vpython.Spec").
Err() |
| 60 } | 58 } |
| 61 return nil | 59 return nil |
| 62 } | 60 } |
| 63 | 61 |
| 64 // Loader implements the generic ability to load a "vpython" spec file. | 62 // Loader implements the generic ability to load a "vpython" spec file. |
| 65 type Loader struct { | 63 type Loader struct { |
| 66 // InlineBeginGuard is a string that signifies the beginning of an inlin
e | 64 // InlineBeginGuard is a string that signifies the beginning of an inlin
e |
| 67 // specification. If empty, DefaultInlineBeginGuard will be used. | 65 // specification. If empty, DefaultInlineBeginGuard will be used. |
| 68 InlineBeginGuard string | 66 InlineBeginGuard string |
| 69 // InlineEndGuard is a string that signifies the end of an inline | 67 // InlineEndGuard is a string that signifies the end of an inline |
| (...skipping 81 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 151 // ====== | 149 // ====== |
| 152 // | 150 // |
| 153 // LoadForScript will examine successive parent directories starting from the | 151 // LoadForScript will examine successive parent directories starting from the |
| 154 // script's location, looking for a file named CommonName. If it finds one, it | 152 // script's location, looking for a file named CommonName. If it finds one, it |
| 155 // will use that as the specification file. This enables scripts to implicitly | 153 // will use that as the specification file. This enables scripts to implicitly |
| 156 // share an specification. | 154 // share an specification. |
| 157 func (l *Loader) LoadForScript(c context.Context, path string, isModule bool) (*
vpython.Spec, error) { | 155 func (l *Loader) LoadForScript(c context.Context, path string, isModule bool) (*
vpython.Spec, error) { |
| 158 // Partner File: Try loading the spec from an adjacent file. | 156 // Partner File: Try loading the spec from an adjacent file. |
| 159 specPath, err := l.findForScript(path, isModule) | 157 specPath, err := l.findForScript(path, isModule) |
| 160 if err != nil { | 158 if err != nil { |
| 161 » » return nil, errors.Annotate(err).Reason("failed to scan for file
system spec").Err() | 159 » » return nil, errors.Annotate(err, "failed to scan for filesystem
spec").Err() |
| 162 } | 160 } |
| 163 if specPath != "" { | 161 if specPath != "" { |
| 164 var spec vpython.Spec | 162 var spec vpython.Spec |
| 165 if err := Load(specPath, &spec); err != nil { | 163 if err := Load(specPath, &spec); err != nil { |
| 166 return nil, err | 164 return nil, err |
| 167 } | 165 } |
| 168 | 166 |
| 169 logging.Infof(c, "Loaded specification from: %s", specPath) | 167 logging.Infof(c, "Loaded specification from: %s", specPath) |
| 170 return &spec, nil | 168 return &spec, nil |
| 171 } | 169 } |
| 172 | 170 |
| 173 // Inline: Try and parse the main script for the spec file. | 171 // Inline: Try and parse the main script for the spec file. |
| 174 mainScript := path | 172 mainScript := path |
| 175 if isModule { | 173 if isModule { |
| 176 // Module. | 174 // Module. |
| 177 mainScript = filepath.Join(mainScript, "__main__.py") | 175 mainScript = filepath.Join(mainScript, "__main__.py") |
| 178 } | 176 } |
| 179 switch spec, err := l.parseFrom(mainScript); { | 177 switch spec, err := l.parseFrom(mainScript); { |
| 180 case err != nil: | 178 case err != nil: |
| 181 » » return nil, errors.Annotate(err).Reason("failed to parse inline
spec from: %(script)s"). | 179 » » return nil, errors.Annotate(err, "failed to parse inline spec fr
om: %s", mainScript).Err() |
| 182 » » » D("script", mainScript). | |
| 183 » » » Err() | |
| 184 | 180 |
| 185 case spec != nil: | 181 case spec != nil: |
| 186 logging.Infof(c, "Loaded inline spec from: %s", mainScript) | 182 logging.Infof(c, "Loaded inline spec from: %s", mainScript) |
| 187 return spec, nil | 183 return spec, nil |
| 188 } | 184 } |
| 189 | 185 |
| 190 // Common: Try and identify a common specification file. | 186 // Common: Try and identify a common specification file. |
| 191 switch path, err := l.findCommonWalkingFrom(filepath.Dir(mainScript)); { | 187 switch path, err := l.findCommonWalkingFrom(filepath.Dir(mainScript)); { |
| 192 case err != nil: | 188 case err != nil: |
| 193 return nil, err | 189 return nil, err |
| (...skipping 27 matching lines...) Expand all Loading... |
| 221 for { | 217 for { |
| 222 prev := path | 218 prev := path |
| 223 | 219 |
| 224 // Directory must be a Python module. | 220 // Directory must be a Python module. |
| 225 initPath := filepath.Join(path, "__init__.py") | 221 initPath := filepath.Join(path, "__init__.py") |
| 226 if _, err := os.Stat(initPath); err != nil { | 222 if _, err := os.Stat(initPath); err != nil { |
| 227 if os.IsNotExist(err) { | 223 if os.IsNotExist(err) { |
| 228 // Not a Python module, so we're done our search
. | 224 // Not a Python module, so we're done our search
. |
| 229 return "", nil | 225 return "", nil |
| 230 } | 226 } |
| 231 » » » return "", errors.Annotate(err).Reason("failed to stat f
or: %(path)"). | 227 » » » return "", errors.Annotate(err, "failed to stat for: %s"
, path).Err() |
| 232 » » » » D("path", initPath). | |
| 233 » » » » Err() | |
| 234 } | 228 } |
| 235 | 229 |
| 236 // Does a spec file exist for this path? | 230 // Does a spec file exist for this path? |
| 237 specPath := path + Suffix | 231 specPath := path + Suffix |
| 238 switch _, err := os.Stat(specPath); { | 232 switch _, err := os.Stat(specPath); { |
| 239 case err == nil: | 233 case err == nil: |
| 240 // Found the file. | 234 // Found the file. |
| 241 return specPath, nil | 235 return specPath, nil |
| 242 | 236 |
| 243 case os.IsNotExist(err): | 237 case os.IsNotExist(err): |
| 244 // Recurse to parent. | 238 // Recurse to parent. |
| 245 path = filepath.Dir(path) | 239 path = filepath.Dir(path) |
| 246 if path == prev { | 240 if path == prev { |
| 247 // Finished recursing, no ES file. | 241 // Finished recursing, no ES file. |
| 248 return "", nil | 242 return "", nil |
| 249 } | 243 } |
| 250 | 244 |
| 251 default: | 245 default: |
| 252 » » » return "", errors.Annotate(err).Reason("failed to check
for spec file at: %(path)s"). | 246 » » » return "", errors.Annotate(err, "failed to check for spe
c file at: %s", specPath).Err() |
| 253 » » » » D("path", specPath). | |
| 254 » » » » Err() | |
| 255 } | 247 } |
| 256 } | 248 } |
| 257 } | 249 } |
| 258 | 250 |
| 259 func (l *Loader) parseFrom(path string) (*vpython.Spec, error) { | 251 func (l *Loader) parseFrom(path string) (*vpython.Spec, error) { |
| 260 fd, err := os.Open(path) | 252 fd, err := os.Open(path) |
| 261 if err != nil { | 253 if err != nil { |
| 262 » » return nil, errors.Annotate(err).Reason("failed to open file").E
rr() | 254 » » return nil, errors.Annotate(err, "failed to open file").Err() |
| 263 } | 255 } |
| 264 defer fd.Close() | 256 defer fd.Close() |
| 265 | 257 |
| 266 // Determine our guards. | 258 // Determine our guards. |
| 267 beginGuard := l.InlineBeginGuard | 259 beginGuard := l.InlineBeginGuard |
| 268 if beginGuard == "" { | 260 if beginGuard == "" { |
| 269 beginGuard = DefaultInlineBeginGuard | 261 beginGuard = DefaultInlineBeginGuard |
| 270 } | 262 } |
| 271 | 263 |
| 272 endGuard := l.InlineEndGuard | 264 endGuard := l.InlineEndGuard |
| (...skipping 16 matching lines...) Expand all Loading... |
| 289 } else { | 281 } else { |
| 290 if strings.HasSuffix(line, endGuard) { | 282 if strings.HasSuffix(line, endGuard) { |
| 291 // Finished processing. | 283 // Finished processing. |
| 292 endLine = line | 284 endLine = line |
| 293 break | 285 break |
| 294 } | 286 } |
| 295 content = append(content, line) | 287 content = append(content, line) |
| 296 } | 288 } |
| 297 } | 289 } |
| 298 if err := s.Err(); err != nil { | 290 if err := s.Err(); err != nil { |
| 299 » » return nil, errors.Annotate(err).Reason("error scanning file").E
rr() | 291 » » return nil, errors.Annotate(err, "error scanning file").Err() |
| 300 } | 292 } |
| 301 if len(content) == 0 { | 293 if len(content) == 0 { |
| 302 return nil, nil | 294 return nil, nil |
| 303 } | 295 } |
| 304 if endLine == "" { | 296 if endLine == "" { |
| 305 return nil, errors.New("unterminated inline spec file") | 297 return nil, errors.New("unterminated inline spec file") |
| 306 } | 298 } |
| 307 | 299 |
| 308 // If we have a common begin/end prefix, trim it from each content line
that | 300 // If we have a common begin/end prefix, trim it from each content line
that |
| 309 // also has it. | 301 // also has it. |
| (...skipping 13 matching lines...) Expand all Loading... |
| 323 } else { | 315 } else { |
| 324 line = strings.TrimPrefix(line, prefix) | 316 line = strings.TrimPrefix(line, prefix) |
| 325 } | 317 } |
| 326 content[i] = line | 318 content[i] = line |
| 327 } | 319 } |
| 328 } | 320 } |
| 329 | 321 |
| 330 // Process the resulting file. | 322 // Process the resulting file. |
| 331 var spec vpython.Spec | 323 var spec vpython.Spec |
| 332 if err := Parse(strings.Join(content, "\n"), &spec); err != nil { | 324 if err := Parse(strings.Join(content, "\n"), &spec); err != nil { |
| 333 » » return nil, errors.Annotate(err).Reason("failed to parse spec fi
le from: %(path)s"). | 325 » » return nil, errors.Annotate(err, "failed to parse spec file from
: %s", path).Err() |
| 334 » » » D("path", path). | |
| 335 » » » Err() | |
| 336 } | 326 } |
| 337 return &spec, nil | 327 return &spec, nil |
| 338 } | 328 } |
| 339 | 329 |
| 340 func (l *Loader) findCommonWalkingFrom(startDir string) (string, error) { | 330 func (l *Loader) findCommonWalkingFrom(startDir string) (string, error) { |
| 341 // Walk until we hit root. | 331 // Walk until we hit root. |
| 342 prevDir := "" | 332 prevDir := "" |
| 343 for prevDir != startDir { | 333 for prevDir != startDir { |
| 344 checkPath := filepath.Join(startDir, CommonName) | 334 checkPath := filepath.Join(startDir, CommonName) |
| 345 | 335 |
| 346 switch _, err := os.Stat(checkPath); { | 336 switch _, err := os.Stat(checkPath); { |
| 347 case err == nil: | 337 case err == nil: |
| 348 return checkPath, nil | 338 return checkPath, nil |
| 349 | 339 |
| 350 case filesystem.IsNotExist(err): | 340 case filesystem.IsNotExist(err): |
| 351 // Not in this directory. | 341 // Not in this directory. |
| 352 | 342 |
| 353 default: | 343 default: |
| 354 // Failed to load specification from this file. | 344 // Failed to load specification from this file. |
| 355 » » » return "", errors.Annotate(err).Reason("failed to stat c
ommon spec file at: %(path)s"). | 345 » » » return "", errors.Annotate(err, "failed to stat common s
pec file at: %s", checkPath).Err() |
| 356 » » » » D("path", checkPath). | |
| 357 » » » » Err() | |
| 358 } | 346 } |
| 359 | 347 |
| 360 // If we have any barrier files, check to see if they are presen
t in this | 348 // If we have any barrier files, check to see if they are presen
t in this |
| 361 // directory. | 349 // directory. |
| 362 for _, name := range l.CommonFilesystemBarriers { | 350 for _, name := range l.CommonFilesystemBarriers { |
| 363 barrierName := filepath.Join(startDir, name) | 351 barrierName := filepath.Join(startDir, name) |
| 364 if _, err := os.Stat(barrierName); err == nil { | 352 if _, err := os.Stat(barrierName); err == nil { |
| 365 // Identified a barrier file in this directory. | 353 // Identified a barrier file in this directory. |
| 366 return "", nil | 354 return "", nil |
| 367 } | 355 } |
| 368 } | 356 } |
| 369 | 357 |
| 370 // Walk up a directory. | 358 // Walk up a directory. |
| 371 startDir, prevDir = filepath.Dir(startDir), startDir | 359 startDir, prevDir = filepath.Dir(startDir), startDir |
| 372 } | 360 } |
| 373 | 361 |
| 374 // Couldn't find the file. | 362 // Couldn't find the file. |
| 375 return "", nil | 363 return "", nil |
| 376 } | 364 } |
| OLD | NEW |