Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(67)

Side by Side Diff: vpython/venv/venv_resources_test.go

Issue 2963503003: [errors] Greatly simplify common/errors package. (Closed)
Patch Set: fix nits Created 3 years, 5 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
« no previous file with comments | « vpython/venv/venv.go ('k') | vpython/venv/venv_test.go » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 // Copyright 2017 The LUCI Authors. All rights reserved. 1 // Copyright 2017 The LUCI Authors. All rights reserved.
2 // Use of this source code is governed under the Apache License, Version 2.0 2 // Use of this source code is governed under the Apache License, Version 2.0
3 // that can be found in the LICENSE file. 3 // that can be found in the LICENSE file.
4 4
5 package venv 5 package venv
6 6
7 import ( 7 import (
8 "archive/zip" 8 "archive/zip"
9 "crypto/sha256" 9 "crypto/sha256"
10 "encoding/hex" 10 "encoding/hex"
(...skipping 73 matching lines...) Expand 10 before | Expand all | Expand 10 after
84 // 84 //
85 // This online setup is preferred to actually checking these binary files into 85 // This online setup is preferred to actually checking these binary files into
86 // Git, as it offers more versatility and doesn't clutter Git with binary junk. 86 // Git, as it offers more versatility and doesn't clutter Git with binary junk.
87 // 87 //
88 // To optimize repeated test re-executions, withTestEnvironment will also cache 88 // To optimize repeated test re-executions, withTestEnvironment will also cache
89 // the downloaded artifacts in a cache directory. All artifacts will be verified 89 // the downloaded artifacts in a cache directory. All artifacts will be verified
90 // by their SHA256 hashes, which will be baked into the source here. 90 // by their SHA256 hashes, which will be baked into the source here.
91 func loadTestEnvironment(ctx context.Context, t *testing.T) (*testingLoader, err or) { 91 func loadTestEnvironment(ctx context.Context, t *testing.T) (*testingLoader, err or) {
92 wd, err := os.Getwd() 92 wd, err := os.Getwd()
93 if err != nil { 93 if err != nil {
94 » » return nil, errors.Annotate(err).Reason("failed to get working d irectory").Err() 94 » » return nil, errors.Annotate(err, "failed to get working director y").Err()
95 } 95 }
96 96
97 cacheDir := filepath.Join(wd, ".venv_test_cache") 97 cacheDir := filepath.Join(wd, ".venv_test_cache")
98 if err := filesystem.MakeDirs(cacheDir); err != nil { 98 if err := filesystem.MakeDirs(cacheDir); err != nil {
99 » » return nil, errors.Annotate(err).Reason("failed to create cache dir").Err() 99 » » return nil, errors.Annotate(err, "failed to create cache dir").E rr()
100 } 100 }
101 101
102 tl := testingLoader{ 102 tl := testingLoader{
103 cacheDir: cacheDir, 103 cacheDir: cacheDir,
104 } 104 }
105 return &tl, tl.withCacheLock(t, func() error { 105 return &tl, tl.withCacheLock(t, func() error {
106 return tl.ensureRemoteFilesLocked(ctx, t) 106 return tl.ensureRemoteFilesLocked(ctx, t)
107 }) 107 })
108 } 108 }
109 109
(...skipping 41 matching lines...) Expand 10 before | Expand all | Expand 10 after
151 func (tl *testingLoader) installPackage(name, root string) error { 151 func (tl *testingLoader) installPackage(name, root string) error {
152 switch name { 152 switch name {
153 case "foo/bar/virtualenv": 153 case "foo/bar/virtualenv":
154 return unzip(tl.virtualEnvZIP, root) 154 return unzip(tl.virtualEnvZIP, root)
155 case "foo/bar/shirt": 155 case "foo/bar/shirt":
156 return copyFileIntoDir(tl.shirtWheelPath, root) 156 return copyFileIntoDir(tl.shirtWheelPath, root)
157 case "foo/bar/pants": 157 case "foo/bar/pants":
158 return copyFileIntoDir(tl.pantsWheelPath, root) 158 return copyFileIntoDir(tl.pantsWheelPath, root)
159 159
160 default: 160 default:
161 » » return errors.Reason("don't know how to install %(package)q"). 161 » » return errors.Reason("don't know how to install %q", name).Err()
162 » » » D("package", name).
163 » » » Err()
164 } 162 }
165 } 163 }
166 164
167 func (tl *testingLoader) buildWheelLocked(t *testing.T, py *python.Interpreter, name, outDir string) (string, error) { 165 func (tl *testingLoader) buildWheelLocked(t *testing.T, py *python.Interpreter, name, outDir string) (string, error) {
168 ctx := context.Background() 166 ctx := context.Background()
169 w, err := wheel.ParseName(name) 167 w, err := wheel.ParseName(name)
170 if err != nil { 168 if err != nil {
171 » » return "", errors.Annotate(err).Reason("failed to parse wheel na me %(name)q"). 169 » » return "", errors.Annotate(err, "failed to parse wheel name %q", name).Err()
172 » » » D("name", name).
173 » » » Err()
174 } 170 }
175 171
176 outWheelPath := filepath.Join(outDir, w.String()) 172 outWheelPath := filepath.Join(outDir, w.String())
177 switch _, err := os.Stat(outWheelPath); { 173 switch _, err := os.Stat(outWheelPath); {
178 case err == nil: 174 case err == nil:
179 t.Logf("Using cached wheel for %q: %s", name, outWheelPath) 175 t.Logf("Using cached wheel for %q: %s", name, outWheelPath)
180 return outWheelPath, nil 176 return outWheelPath, nil
181 177
182 case os.IsNotExist(err): 178 case os.IsNotExist(err):
183 // Will build a new wheel. 179 // Will build a new wheel.
184 break 180 break
185 181
186 default: 182 default:
187 » » return "", errors.Annotate(err).Reason("failed to stat wheel pat h [%(path)s]"). 183 » » return "", errors.Annotate(err, "failed to stat wheel path [%s]" , outWheelPath).Err()
188 » » » D("path", outWheelPath).
189 » » » Err()
190 } 184 }
191 185
192 srcDir := filepath.Join(testDataDir, w.Distribution+".src") 186 srcDir := filepath.Join(testDataDir, w.Distribution+".src")
193 187
194 // Create a bootstrap wheel-generating VirtualEnv! 188 // Create a bootstrap wheel-generating VirtualEnv!
195 cfg := Config{ 189 cfg := Config{
196 MaxHashLen: 1, // Only going to be 1 enviroment. 190 MaxHashLen: 1, // Only going to be 1 enviroment.
197 BaseDir: filepath.Join(outDir, ".env"), 191 BaseDir: filepath.Join(outDir, ".env"),
198 Python: py.Python, 192 Python: py.Python,
199 Package: vpython.Spec_Package{ 193 Package: vpython.Spec_Package{
(...skipping 27 matching lines...) Expand all
227 // expected forms on all systems. 221 // expected forms on all systems.
228 err := With(ctx, cfg, true, func(ctx context.Context, env *Env) error { 222 err := With(ctx, cfg, true, func(ctx context.Context, env *Env) error {
229 cmd := env.Interpreter().IsolatedCommand(ctx, 223 cmd := env.Interpreter().IsolatedCommand(ctx,
230 "setup.py", 224 "setup.py",
231 "--no-user-cfg", 225 "--no-user-cfg",
232 "bdist_wheel", 226 "bdist_wheel",
233 "--bdist-dir", buildDir, 227 "--bdist-dir", buildDir,
234 "--dist-dir", distDir) 228 "--dist-dir", distDir)
235 cmd.Dir = srcDir 229 cmd.Dir = srcDir
236 if err := cmd.Run(); err != nil { 230 if err := cmd.Run(); err != nil {
237 » » » » return errors.Annotate(err).Reason("failed to bu ild wheel").Err() 231 » » » » return errors.Annotate(err, "failed to build whe el").Err()
238 } 232 }
239 return nil 233 return nil
240 }) 234 })
241 if err != nil { 235 if err != nil {
242 » » » return errors.Annotate(err).Reason("failed to build whee l").Err() 236 » » » return errors.Annotate(err, "failed to build wheel").Err ()
243 } 237 }
244 238
245 // Assert that the expected wheel file was generated, and copy i t into 239 // Assert that the expected wheel file was generated, and copy i t into
246 // outDir. 240 // outDir.
247 wheelPath := filepath.Join(distDir, w.String()) 241 wheelPath := filepath.Join(distDir, w.String())
248 if _, err := os.Stat(wheelPath); err != nil { 242 if _, err := os.Stat(wheelPath); err != nil {
249 » » » return errors.Annotate(err).Reason("failed to generate w heel").Err() 243 » » » return errors.Annotate(err, "failed to generate wheel"). Err()
250 } 244 }
251 if err := copyFileIntoDir(wheelPath, outDir); err != nil { 245 if err := copyFileIntoDir(wheelPath, outDir); err != nil {
252 » » » return errors.Annotate(err).Reason("failed to install wh eel").Err() 246 » » » return errors.Annotate(err, "failed to install wheel").E rr()
253 } 247 }
254 248
255 return nil 249 return nil
256 }) 250 })
257 if err != nil { 251 if err != nil {
258 return "", err 252 return "", err
259 } 253 }
260 254
261 t.Logf("Generated wheel file %q: %s", name, outWheelPath) 255 t.Logf("Generated wheel file %q: %s", name, outWheelPath)
262 return outWheelPath, nil 256 return outWheelPath, nil
(...skipping 28 matching lines...) Expand all
291 for _, url := range rf.urls { 285 for _, url := range rf.urls {
292 err := cacheFromURLLocked(t, cachePath, rf.contentHash, url) 286 err := cacheFromURLLocked(t, cachePath, rf.contentHash, url)
293 if err == nil { 287 if err == nil {
294 t.Logf("Cached remote file [%s] from URL [%s]: [ %s]", rf.name, url, cachePath) 288 t.Logf("Cached remote file [%s] from URL [%s]: [ %s]", rf.name, url, cachePath)
295 rf.install(tl, cachePath) 289 rf.install(tl, cachePath)
296 continue MainLoop 290 continue MainLoop
297 } 291 }
298 t.Logf("Failed to load from URL %q: %s", url, err) 292 t.Logf("Failed to load from URL %q: %s", url, err)
299 } 293 }
300 294
301 » » return errors.Reason("failed to acquire remote file %(name)q"). 295 » » return errors.Reason("failed to acquire remote file %q", rf.name ).Err()
302 » » » D("name", rf.name).
303 » » » Err()
304 } 296 }
305 297
306 return nil 298 return nil
307 } 299 }
308 300
309 func getCachedFileLocked(t *testing.T, cachePath, hash string) error { 301 func getCachedFileLocked(t *testing.T, cachePath, hash string) error {
310 return validateHash(t, cachePath, hash, true) 302 return validateHash(t, cachePath, hash, true)
311 } 303 }
312 304
313 func validateHash(t *testing.T, path, hash string, deleteIfInvalid bool) error { 305 func validateHash(t *testing.T, path, hash string, deleteIfInvalid bool) error {
314 fd, err := os.Open(path) 306 fd, err := os.Open(path)
315 if err != nil { 307 if err != nil {
316 » » return errors.Annotate(err).Reason("failed to open file").Err() 308 » » return errors.Annotate(err, "failed to open file").Err()
317 } 309 }
318 defer fd.Close() 310 defer fd.Close()
319 311
320 h := sha256.New() 312 h := sha256.New()
321 if _, err := io.Copy(h, fd); err != nil { 313 if _, err := io.Copy(h, fd); err != nil {
322 » » return errors.Annotate(err).Reason("failed to hash file").Err() 314 » » return errors.Annotate(err, "failed to hash file").Err()
323 } 315 }
324 316
325 if err := hashesEqual(h, hash); err != nil { 317 if err := hashesEqual(h, hash); err != nil {
326 t.Logf("File [%s] has invalid hash: %s", path, err) 318 t.Logf("File [%s] has invalid hash: %s", path, err)
327 319
328 if deleteIfInvalid { 320 if deleteIfInvalid {
329 if err := os.Remove(path); err != nil { 321 if err := os.Remove(path); err != nil {
330 t.Logf("Failed to delete invalid hash file [%s]: %s", path, err) 322 t.Logf("Failed to delete invalid hash file [%s]: %s", path, err)
331 } 323 }
332 } 324 }
333 return err 325 return err
334 } 326 }
335 327
336 return nil 328 return nil
337 } 329 }
338 330
339 func hashesEqual(h hash.Hash, expected string) error { 331 func hashesEqual(h hash.Hash, expected string) error {
340 if v := hex.EncodeToString(h.Sum(nil)); v != expected { 332 if v := hex.EncodeToString(h.Sum(nil)); v != expected {
341 » » return errors.Reason("hash %(actual)q doesn't match expected %(e xpected)q"). 333 » » return errors.Reason("hash %q doesn't match expected %q", v, exp ected).Err()
342 » » » D("actual", v).
343 » » » D("expected", expected).
344 » » » Err()
345 } 334 }
346 return nil 335 return nil
347 } 336 }
348 337
349 var testCIPDClientOptions = cipd.ClientOptions{ 338 var testCIPDClientOptions = cipd.ClientOptions{
350 ServiceURL: chromeinfra.CIPDServiceURL, 339 ServiceURL: chromeinfra.CIPDServiceURL,
351 UserAgent: "vpython venv tests", 340 UserAgent: "vpython venv tests",
352 } 341 }
353 342
354 func cacheFromCIPDLocked(ctx context.Context, t *testing.T, cachePath, name, has h, pkg, version string) error { 343 func cacheFromCIPDLocked(ctx context.Context, t *testing.T, cachePath, name, has h, pkg, version string) error {
355 return testfs.WithTempDir(t, "vpython_venv_cipd", func(tdir string) erro r { 344 return testfs.WithTempDir(t, "vpython_venv_cipd", func(tdir string) erro r {
356 opts := testCIPDClientOptions 345 opts := testCIPDClientOptions
357 opts.Root = tdir 346 opts.Root = tdir
358 347
359 client, err := cipd.NewClient(opts) 348 client, err := cipd.NewClient(opts)
360 if err != nil { 349 if err != nil {
361 » » » return errors.Annotate(err).Reason("failed to create CIP D client").Err() 350 » » » return errors.Annotate(err, "failed to create CIPD clien t").Err()
362 } 351 }
363 352
364 pin, err := client.ResolveVersion(ctx, pkg, version) 353 pin, err := client.ResolveVersion(ctx, pkg, version)
365 if err != nil { 354 if err != nil {
366 » » » return errors.Annotate(err).Reason("failed to resolve CI PD version for %(pkg)s @%(version)s"). 355 » » » return errors.Annotate(err, "failed to resolve CIPD vers ion for %s @%s", pkg, version).Err()
367 » » » » D("pkg", pkg).
368 » » » » D("version", version).
369 » » » » Err()
370 } 356 }
371 357
372 if err := client.FetchAndDeployInstance(ctx, "", pin); err != ni l { 358 if err := client.FetchAndDeployInstance(ctx, "", pin); err != ni l {
373 » » » return errors.Annotate(err).Reason("failed to fetch/depl oy CIPD package").Err() 359 » » » return errors.Annotate(err, "failed to fetch/deploy CIPD package").Err()
374 } 360 }
375 361
376 path := filepath.Join(opts.Root, name) 362 path := filepath.Join(opts.Root, name)
377 if err := validateHash(t, path, hash, false); err != nil { 363 if err := validateHash(t, path, hash, false); err != nil {
378 // Do not export the invalid path. 364 // Do not export the invalid path.
379 return err 365 return err
380 } 366 }
381 367
382 if err := copyFile(path, cachePath, nil); err != nil { 368 if err := copyFile(path, cachePath, nil); err != nil {
383 » » » return errors.Annotate(err).Reason("failed to install CI PD package file").Err() 369 » » » return errors.Annotate(err, "failed to install CIPD pack age file").Err()
384 } 370 }
385 371
386 return nil 372 return nil
387 }) 373 })
388 } 374 }
389 375
390 func cacheFromURLLocked(t *testing.T, cachePath, hash, url string) (err error) { 376 func cacheFromURLLocked(t *testing.T, cachePath, hash, url string) (err error) {
391 resp, err := http.Get(url) 377 resp, err := http.Get(url)
392 if err != nil { 378 if err != nil {
393 t.Logf("Failed to GET file from URL [%s]: %s", url, err) 379 t.Logf("Failed to GET file from URL [%s]: %s", url, err)
394 } 380 }
395 defer resp.Body.Close() 381 defer resp.Body.Close()
396 382
397 fd, err := os.Create(cachePath) 383 fd, err := os.Create(cachePath)
398 if err != nil { 384 if err != nil {
399 t.Logf("Failed to create output file [%s]: %s", cachePath, err) 385 t.Logf("Failed to create output file [%s]: %s", cachePath, err)
400 } 386 }
401 defer func() { 387 defer func() {
402 if closeErr := fd.Close(); closeErr != nil && err == nil { 388 if closeErr := fd.Close(); closeErr != nil && err == nil {
403 » » » err = errors.Annotate(closeErr).Reason("failed to close file").Err() 389 » » » err = errors.Annotate(closeErr, "failed to close file"). Err()
404 } 390 }
405 }() 391 }()
406 392
407 h := sha256.New() 393 h := sha256.New()
408 tr := io.TeeReader(resp.Body, h) 394 tr := io.TeeReader(resp.Body, h)
409 if _, err := io.Copy(fd, tr); err != nil { 395 if _, err := io.Copy(fd, tr); err != nil {
410 » » return errors.Annotate(err).Reason("failed to download").Err() 396 » » return errors.Annotate(err, "failed to download").Err()
411 } 397 }
412 398
413 if err = hashesEqual(h, hash); err != nil { 399 if err = hashesEqual(h, hash); err != nil {
414 return 400 return
415 } 401 }
416 return nil 402 return nil
417 } 403 }
418 404
419 func unzip(src, dst string) error { 405 func unzip(src, dst string) error {
420 fd, err := zip.OpenReader(src) 406 fd, err := zip.OpenReader(src)
421 if err != nil { 407 if err != nil {
422 » » return errors.Annotate(err).Reason("failed to open ZIP reader"). Err() 408 » » return errors.Annotate(err, "failed to open ZIP reader").Err()
423 } 409 }
424 defer fd.Close() 410 defer fd.Close()
425 411
426 for _, f := range fd.File { 412 for _, f := range fd.File {
427 path := filepath.Join(dst, filepath.FromSlash(f.Name)) 413 path := filepath.Join(dst, filepath.FromSlash(f.Name))
428 fi := f.FileInfo() 414 fi := f.FileInfo()
429 415
430 // Unzip this entry. 416 // Unzip this entry.
431 if fi.IsDir() { 417 if fi.IsDir() {
432 if err := os.MkdirAll(path, 0755); err != nil { 418 if err := os.MkdirAll(path, 0755); err != nil {
433 » » » » return errors.Annotate(err).Reason("failed to mk dir").Err() 419 » » » » return errors.Annotate(err, "failed to mkdir").E rr()
434 } 420 }
435 } else { 421 } else {
436 if err := copyFileOpener(f.Open, path, fi); err != nil { 422 if err := copyFileOpener(f.Open, path, fi); err != nil {
437 return err 423 return err
438 } 424 }
439 } 425 }
440 } 426 }
441 return nil 427 return nil
442 } 428 }
443 429
444 func copyFileIntoDir(src, dstDir string) error { 430 func copyFileIntoDir(src, dstDir string) error {
445 return copyFile(src, filepath.Join(dstDir, filepath.Base(src)), nil) 431 return copyFile(src, filepath.Join(dstDir, filepath.Base(src)), nil)
446 } 432 }
447 433
448 func copyFile(src, dst string, fi os.FileInfo) error { 434 func copyFile(src, dst string, fi os.FileInfo) error {
449 opener := func() (io.ReadCloser, error) { return os.Open(src) } 435 opener := func() (io.ReadCloser, error) { return os.Open(src) }
450 return copyFileOpener(opener, dst, fi) 436 return copyFileOpener(opener, dst, fi)
451 } 437 }
452 438
453 func copyFileOpener(opener func() (io.ReadCloser, error), dst string, fi os.File Info) (err error) { 439 func copyFileOpener(opener func() (io.ReadCloser, error), dst string, fi os.File Info) (err error) {
454 sfd, err := opener() 440 sfd, err := opener()
455 if err != nil { 441 if err != nil {
456 » » return errors.Annotate(err).Reason("failed to open source").Err( ) 442 » » return errors.Annotate(err, "failed to open source").Err()
457 } 443 }
458 defer sfd.Close() 444 defer sfd.Close()
459 445
460 dfd, err := os.Create(dst) 446 dfd, err := os.Create(dst)
461 if err != nil { 447 if err != nil {
462 » » return errors.Annotate(err).Reason("failed to create destination ").Err() 448 » » return errors.Annotate(err, "failed to create destination").Err( )
463 } 449 }
464 defer func() { 450 defer func() {
465 if closeErr := dfd.Close(); closeErr != nil && err == nil { 451 if closeErr := dfd.Close(); closeErr != nil && err == nil {
466 » » » err = errors.Annotate(closeErr).Reason("failed to close destination").Err() 452 » » » err = errors.Annotate(closeErr, "failed to close destina tion").Err()
467 } 453 }
468 }() 454 }()
469 455
470 if _, err := io.Copy(dfd, sfd); err != nil { 456 if _, err := io.Copy(dfd, sfd); err != nil {
471 » » return errors.Annotate(err).Reason("failed to copy file").Err() 457 » » return errors.Annotate(err, "failed to copy file").Err()
472 } 458 }
473 if fi != nil { 459 if fi != nil {
474 if err := os.Chmod(dst, fi.Mode()); err != nil { 460 if err := os.Chmod(dst, fi.Mode()); err != nil {
475 » » » return errors.Annotate(err).Reason("failed to chmod").Er r() 461 » » » return errors.Annotate(err, "failed to chmod").Err()
476 } 462 }
477 } 463 }
478 return nil 464 return nil
479 } 465 }
OLDNEW
« no previous file with comments | « vpython/venv/venv.go ('k') | vpython/venv/venv_test.go » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698