OLD | NEW |
1 // Copyright 2017 The LUCI Authors. All rights reserved. | 1 // Copyright 2017 The LUCI Authors. All rights reserved. |
2 // Use of this source code is governed under the Apache License, Version 2.0 | 2 // Use of this source code is governed under the Apache License, Version 2.0 |
3 // that can be found in the LICENSE file. | 3 // that can be found in the LICENSE file. |
4 | 4 |
5 package venv | 5 package venv |
6 | 6 |
7 import ( | 7 import ( |
8 "archive/zip" | 8 "archive/zip" |
9 "crypto/sha256" | 9 "crypto/sha256" |
10 "encoding/hex" | 10 "encoding/hex" |
(...skipping 73 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
84 // | 84 // |
85 // This online setup is preferred to actually checking these binary files into | 85 // This online setup is preferred to actually checking these binary files into |
86 // Git, as it offers more versatility and doesn't clutter Git with binary junk. | 86 // Git, as it offers more versatility and doesn't clutter Git with binary junk. |
87 // | 87 // |
88 // To optimize repeated test re-executions, withTestEnvironment will also cache | 88 // To optimize repeated test re-executions, withTestEnvironment will also cache |
89 // the downloaded artifacts in a cache directory. All artifacts will be verified | 89 // the downloaded artifacts in a cache directory. All artifacts will be verified |
90 // by their SHA256 hashes, which will be baked into the source here. | 90 // by their SHA256 hashes, which will be baked into the source here. |
91 func loadTestEnvironment(ctx context.Context, t *testing.T) (*testingLoader, err
or) { | 91 func loadTestEnvironment(ctx context.Context, t *testing.T) (*testingLoader, err
or) { |
92 wd, err := os.Getwd() | 92 wd, err := os.Getwd() |
93 if err != nil { | 93 if err != nil { |
94 » » return nil, errors.Annotate(err).Reason("failed to get working d
irectory").Err() | 94 » » return nil, errors.Annotate(err, "failed to get working director
y").Err() |
95 } | 95 } |
96 | 96 |
97 cacheDir := filepath.Join(wd, ".venv_test_cache") | 97 cacheDir := filepath.Join(wd, ".venv_test_cache") |
98 if err := filesystem.MakeDirs(cacheDir); err != nil { | 98 if err := filesystem.MakeDirs(cacheDir); err != nil { |
99 » » return nil, errors.Annotate(err).Reason("failed to create cache
dir").Err() | 99 » » return nil, errors.Annotate(err, "failed to create cache dir").E
rr() |
100 } | 100 } |
101 | 101 |
102 tl := testingLoader{ | 102 tl := testingLoader{ |
103 cacheDir: cacheDir, | 103 cacheDir: cacheDir, |
104 } | 104 } |
105 return &tl, tl.withCacheLock(t, func() error { | 105 return &tl, tl.withCacheLock(t, func() error { |
106 return tl.ensureRemoteFilesLocked(ctx, t) | 106 return tl.ensureRemoteFilesLocked(ctx, t) |
107 }) | 107 }) |
108 } | 108 } |
109 | 109 |
(...skipping 41 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
151 func (tl *testingLoader) installPackage(name, root string) error { | 151 func (tl *testingLoader) installPackage(name, root string) error { |
152 switch name { | 152 switch name { |
153 case "foo/bar/virtualenv": | 153 case "foo/bar/virtualenv": |
154 return unzip(tl.virtualEnvZIP, root) | 154 return unzip(tl.virtualEnvZIP, root) |
155 case "foo/bar/shirt": | 155 case "foo/bar/shirt": |
156 return copyFileIntoDir(tl.shirtWheelPath, root) | 156 return copyFileIntoDir(tl.shirtWheelPath, root) |
157 case "foo/bar/pants": | 157 case "foo/bar/pants": |
158 return copyFileIntoDir(tl.pantsWheelPath, root) | 158 return copyFileIntoDir(tl.pantsWheelPath, root) |
159 | 159 |
160 default: | 160 default: |
161 » » return errors.Reason("don't know how to install %(package)q"). | 161 » » return errors.Reason("don't know how to install %q", name).Err() |
162 » » » D("package", name). | |
163 » » » Err() | |
164 } | 162 } |
165 } | 163 } |
166 | 164 |
167 func (tl *testingLoader) buildWheelLocked(t *testing.T, py *python.Interpreter,
name, outDir string) (string, error) { | 165 func (tl *testingLoader) buildWheelLocked(t *testing.T, py *python.Interpreter,
name, outDir string) (string, error) { |
168 ctx := context.Background() | 166 ctx := context.Background() |
169 w, err := wheel.ParseName(name) | 167 w, err := wheel.ParseName(name) |
170 if err != nil { | 168 if err != nil { |
171 » » return "", errors.Annotate(err).Reason("failed to parse wheel na
me %(name)q"). | 169 » » return "", errors.Annotate(err, "failed to parse wheel name %q",
name).Err() |
172 » » » D("name", name). | |
173 » » » Err() | |
174 } | 170 } |
175 | 171 |
176 outWheelPath := filepath.Join(outDir, w.String()) | 172 outWheelPath := filepath.Join(outDir, w.String()) |
177 switch _, err := os.Stat(outWheelPath); { | 173 switch _, err := os.Stat(outWheelPath); { |
178 case err == nil: | 174 case err == nil: |
179 t.Logf("Using cached wheel for %q: %s", name, outWheelPath) | 175 t.Logf("Using cached wheel for %q: %s", name, outWheelPath) |
180 return outWheelPath, nil | 176 return outWheelPath, nil |
181 | 177 |
182 case os.IsNotExist(err): | 178 case os.IsNotExist(err): |
183 // Will build a new wheel. | 179 // Will build a new wheel. |
184 break | 180 break |
185 | 181 |
186 default: | 182 default: |
187 » » return "", errors.Annotate(err).Reason("failed to stat wheel pat
h [%(path)s]"). | 183 » » return "", errors.Annotate(err, "failed to stat wheel path [%s]"
, outWheelPath).Err() |
188 » » » D("path", outWheelPath). | |
189 » » » Err() | |
190 } | 184 } |
191 | 185 |
192 srcDir := filepath.Join(testDataDir, w.Distribution+".src") | 186 srcDir := filepath.Join(testDataDir, w.Distribution+".src") |
193 | 187 |
194 // Create a bootstrap wheel-generating VirtualEnv! | 188 // Create a bootstrap wheel-generating VirtualEnv! |
195 cfg := Config{ | 189 cfg := Config{ |
196 MaxHashLen: 1, // Only going to be 1 enviroment. | 190 MaxHashLen: 1, // Only going to be 1 enviroment. |
197 BaseDir: filepath.Join(outDir, ".env"), | 191 BaseDir: filepath.Join(outDir, ".env"), |
198 Python: py.Python, | 192 Python: py.Python, |
199 Package: vpython.Spec_Package{ | 193 Package: vpython.Spec_Package{ |
(...skipping 27 matching lines...) Expand all Loading... |
227 // expected forms on all systems. | 221 // expected forms on all systems. |
228 err := With(ctx, cfg, true, func(ctx context.Context, env *Env)
error { | 222 err := With(ctx, cfg, true, func(ctx context.Context, env *Env)
error { |
229 cmd := env.Interpreter().IsolatedCommand(ctx, | 223 cmd := env.Interpreter().IsolatedCommand(ctx, |
230 "setup.py", | 224 "setup.py", |
231 "--no-user-cfg", | 225 "--no-user-cfg", |
232 "bdist_wheel", | 226 "bdist_wheel", |
233 "--bdist-dir", buildDir, | 227 "--bdist-dir", buildDir, |
234 "--dist-dir", distDir) | 228 "--dist-dir", distDir) |
235 cmd.Dir = srcDir | 229 cmd.Dir = srcDir |
236 if err := cmd.Run(); err != nil { | 230 if err := cmd.Run(); err != nil { |
237 » » » » return errors.Annotate(err).Reason("failed to bu
ild wheel").Err() | 231 » » » » return errors.Annotate(err, "failed to build whe
el").Err() |
238 } | 232 } |
239 return nil | 233 return nil |
240 }) | 234 }) |
241 if err != nil { | 235 if err != nil { |
242 » » » return errors.Annotate(err).Reason("failed to build whee
l").Err() | 236 » » » return errors.Annotate(err, "failed to build wheel").Err
() |
243 } | 237 } |
244 | 238 |
245 // Assert that the expected wheel file was generated, and copy i
t into | 239 // Assert that the expected wheel file was generated, and copy i
t into |
246 // outDir. | 240 // outDir. |
247 wheelPath := filepath.Join(distDir, w.String()) | 241 wheelPath := filepath.Join(distDir, w.String()) |
248 if _, err := os.Stat(wheelPath); err != nil { | 242 if _, err := os.Stat(wheelPath); err != nil { |
249 » » » return errors.Annotate(err).Reason("failed to generate w
heel").Err() | 243 » » » return errors.Annotate(err, "failed to generate wheel").
Err() |
250 } | 244 } |
251 if err := copyFileIntoDir(wheelPath, outDir); err != nil { | 245 if err := copyFileIntoDir(wheelPath, outDir); err != nil { |
252 » » » return errors.Annotate(err).Reason("failed to install wh
eel").Err() | 246 » » » return errors.Annotate(err, "failed to install wheel").E
rr() |
253 } | 247 } |
254 | 248 |
255 return nil | 249 return nil |
256 }) | 250 }) |
257 if err != nil { | 251 if err != nil { |
258 return "", err | 252 return "", err |
259 } | 253 } |
260 | 254 |
261 t.Logf("Generated wheel file %q: %s", name, outWheelPath) | 255 t.Logf("Generated wheel file %q: %s", name, outWheelPath) |
262 return outWheelPath, nil | 256 return outWheelPath, nil |
(...skipping 28 matching lines...) Expand all Loading... |
291 for _, url := range rf.urls { | 285 for _, url := range rf.urls { |
292 err := cacheFromURLLocked(t, cachePath, rf.contentHash,
url) | 286 err := cacheFromURLLocked(t, cachePath, rf.contentHash,
url) |
293 if err == nil { | 287 if err == nil { |
294 t.Logf("Cached remote file [%s] from URL [%s]: [
%s]", rf.name, url, cachePath) | 288 t.Logf("Cached remote file [%s] from URL [%s]: [
%s]", rf.name, url, cachePath) |
295 rf.install(tl, cachePath) | 289 rf.install(tl, cachePath) |
296 continue MainLoop | 290 continue MainLoop |
297 } | 291 } |
298 t.Logf("Failed to load from URL %q: %s", url, err) | 292 t.Logf("Failed to load from URL %q: %s", url, err) |
299 } | 293 } |
300 | 294 |
301 » » return errors.Reason("failed to acquire remote file %(name)q"). | 295 » » return errors.Reason("failed to acquire remote file %q", rf.name
).Err() |
302 » » » D("name", rf.name). | |
303 » » » Err() | |
304 } | 296 } |
305 | 297 |
306 return nil | 298 return nil |
307 } | 299 } |
308 | 300 |
309 func getCachedFileLocked(t *testing.T, cachePath, hash string) error { | 301 func getCachedFileLocked(t *testing.T, cachePath, hash string) error { |
310 return validateHash(t, cachePath, hash, true) | 302 return validateHash(t, cachePath, hash, true) |
311 } | 303 } |
312 | 304 |
313 func validateHash(t *testing.T, path, hash string, deleteIfInvalid bool) error { | 305 func validateHash(t *testing.T, path, hash string, deleteIfInvalid bool) error { |
314 fd, err := os.Open(path) | 306 fd, err := os.Open(path) |
315 if err != nil { | 307 if err != nil { |
316 » » return errors.Annotate(err).Reason("failed to open file").Err() | 308 » » return errors.Annotate(err, "failed to open file").Err() |
317 } | 309 } |
318 defer fd.Close() | 310 defer fd.Close() |
319 | 311 |
320 h := sha256.New() | 312 h := sha256.New() |
321 if _, err := io.Copy(h, fd); err != nil { | 313 if _, err := io.Copy(h, fd); err != nil { |
322 » » return errors.Annotate(err).Reason("failed to hash file").Err() | 314 » » return errors.Annotate(err, "failed to hash file").Err() |
323 } | 315 } |
324 | 316 |
325 if err := hashesEqual(h, hash); err != nil { | 317 if err := hashesEqual(h, hash); err != nil { |
326 t.Logf("File [%s] has invalid hash: %s", path, err) | 318 t.Logf("File [%s] has invalid hash: %s", path, err) |
327 | 319 |
328 if deleteIfInvalid { | 320 if deleteIfInvalid { |
329 if err := os.Remove(path); err != nil { | 321 if err := os.Remove(path); err != nil { |
330 t.Logf("Failed to delete invalid hash file [%s]:
%s", path, err) | 322 t.Logf("Failed to delete invalid hash file [%s]:
%s", path, err) |
331 } | 323 } |
332 } | 324 } |
333 return err | 325 return err |
334 } | 326 } |
335 | 327 |
336 return nil | 328 return nil |
337 } | 329 } |
338 | 330 |
339 func hashesEqual(h hash.Hash, expected string) error { | 331 func hashesEqual(h hash.Hash, expected string) error { |
340 if v := hex.EncodeToString(h.Sum(nil)); v != expected { | 332 if v := hex.EncodeToString(h.Sum(nil)); v != expected { |
341 » » return errors.Reason("hash %(actual)q doesn't match expected %(e
xpected)q"). | 333 » » return errors.Reason("hash %q doesn't match expected %q", v, exp
ected).Err() |
342 » » » D("actual", v). | |
343 » » » D("expected", expected). | |
344 » » » Err() | |
345 } | 334 } |
346 return nil | 335 return nil |
347 } | 336 } |
348 | 337 |
349 var testCIPDClientOptions = cipd.ClientOptions{ | 338 var testCIPDClientOptions = cipd.ClientOptions{ |
350 ServiceURL: chromeinfra.CIPDServiceURL, | 339 ServiceURL: chromeinfra.CIPDServiceURL, |
351 UserAgent: "vpython venv tests", | 340 UserAgent: "vpython venv tests", |
352 } | 341 } |
353 | 342 |
354 func cacheFromCIPDLocked(ctx context.Context, t *testing.T, cachePath, name, has
h, pkg, version string) error { | 343 func cacheFromCIPDLocked(ctx context.Context, t *testing.T, cachePath, name, has
h, pkg, version string) error { |
355 return testfs.WithTempDir(t, "vpython_venv_cipd", func(tdir string) erro
r { | 344 return testfs.WithTempDir(t, "vpython_venv_cipd", func(tdir string) erro
r { |
356 opts := testCIPDClientOptions | 345 opts := testCIPDClientOptions |
357 opts.Root = tdir | 346 opts.Root = tdir |
358 | 347 |
359 client, err := cipd.NewClient(opts) | 348 client, err := cipd.NewClient(opts) |
360 if err != nil { | 349 if err != nil { |
361 » » » return errors.Annotate(err).Reason("failed to create CIP
D client").Err() | 350 » » » return errors.Annotate(err, "failed to create CIPD clien
t").Err() |
362 } | 351 } |
363 | 352 |
364 pin, err := client.ResolveVersion(ctx, pkg, version) | 353 pin, err := client.ResolveVersion(ctx, pkg, version) |
365 if err != nil { | 354 if err != nil { |
366 » » » return errors.Annotate(err).Reason("failed to resolve CI
PD version for %(pkg)s @%(version)s"). | 355 » » » return errors.Annotate(err, "failed to resolve CIPD vers
ion for %s @%s", pkg, version).Err() |
367 » » » » D("pkg", pkg). | |
368 » » » » D("version", version). | |
369 » » » » Err() | |
370 } | 356 } |
371 | 357 |
372 if err := client.FetchAndDeployInstance(ctx, "", pin); err != ni
l { | 358 if err := client.FetchAndDeployInstance(ctx, "", pin); err != ni
l { |
373 » » » return errors.Annotate(err).Reason("failed to fetch/depl
oy CIPD package").Err() | 359 » » » return errors.Annotate(err, "failed to fetch/deploy CIPD
package").Err() |
374 } | 360 } |
375 | 361 |
376 path := filepath.Join(opts.Root, name) | 362 path := filepath.Join(opts.Root, name) |
377 if err := validateHash(t, path, hash, false); err != nil { | 363 if err := validateHash(t, path, hash, false); err != nil { |
378 // Do not export the invalid path. | 364 // Do not export the invalid path. |
379 return err | 365 return err |
380 } | 366 } |
381 | 367 |
382 if err := copyFile(path, cachePath, nil); err != nil { | 368 if err := copyFile(path, cachePath, nil); err != nil { |
383 » » » return errors.Annotate(err).Reason("failed to install CI
PD package file").Err() | 369 » » » return errors.Annotate(err, "failed to install CIPD pack
age file").Err() |
384 } | 370 } |
385 | 371 |
386 return nil | 372 return nil |
387 }) | 373 }) |
388 } | 374 } |
389 | 375 |
390 func cacheFromURLLocked(t *testing.T, cachePath, hash, url string) (err error) { | 376 func cacheFromURLLocked(t *testing.T, cachePath, hash, url string) (err error) { |
391 resp, err := http.Get(url) | 377 resp, err := http.Get(url) |
392 if err != nil { | 378 if err != nil { |
393 t.Logf("Failed to GET file from URL [%s]: %s", url, err) | 379 t.Logf("Failed to GET file from URL [%s]: %s", url, err) |
394 } | 380 } |
395 defer resp.Body.Close() | 381 defer resp.Body.Close() |
396 | 382 |
397 fd, err := os.Create(cachePath) | 383 fd, err := os.Create(cachePath) |
398 if err != nil { | 384 if err != nil { |
399 t.Logf("Failed to create output file [%s]: %s", cachePath, err) | 385 t.Logf("Failed to create output file [%s]: %s", cachePath, err) |
400 } | 386 } |
401 defer func() { | 387 defer func() { |
402 if closeErr := fd.Close(); closeErr != nil && err == nil { | 388 if closeErr := fd.Close(); closeErr != nil && err == nil { |
403 » » » err = errors.Annotate(closeErr).Reason("failed to close
file").Err() | 389 » » » err = errors.Annotate(closeErr, "failed to close file").
Err() |
404 } | 390 } |
405 }() | 391 }() |
406 | 392 |
407 h := sha256.New() | 393 h := sha256.New() |
408 tr := io.TeeReader(resp.Body, h) | 394 tr := io.TeeReader(resp.Body, h) |
409 if _, err := io.Copy(fd, tr); err != nil { | 395 if _, err := io.Copy(fd, tr); err != nil { |
410 » » return errors.Annotate(err).Reason("failed to download").Err() | 396 » » return errors.Annotate(err, "failed to download").Err() |
411 } | 397 } |
412 | 398 |
413 if err = hashesEqual(h, hash); err != nil { | 399 if err = hashesEqual(h, hash); err != nil { |
414 return | 400 return |
415 } | 401 } |
416 return nil | 402 return nil |
417 } | 403 } |
418 | 404 |
419 func unzip(src, dst string) error { | 405 func unzip(src, dst string) error { |
420 fd, err := zip.OpenReader(src) | 406 fd, err := zip.OpenReader(src) |
421 if err != nil { | 407 if err != nil { |
422 » » return errors.Annotate(err).Reason("failed to open ZIP reader").
Err() | 408 » » return errors.Annotate(err, "failed to open ZIP reader").Err() |
423 } | 409 } |
424 defer fd.Close() | 410 defer fd.Close() |
425 | 411 |
426 for _, f := range fd.File { | 412 for _, f := range fd.File { |
427 path := filepath.Join(dst, filepath.FromSlash(f.Name)) | 413 path := filepath.Join(dst, filepath.FromSlash(f.Name)) |
428 fi := f.FileInfo() | 414 fi := f.FileInfo() |
429 | 415 |
430 // Unzip this entry. | 416 // Unzip this entry. |
431 if fi.IsDir() { | 417 if fi.IsDir() { |
432 if err := os.MkdirAll(path, 0755); err != nil { | 418 if err := os.MkdirAll(path, 0755); err != nil { |
433 » » » » return errors.Annotate(err).Reason("failed to mk
dir").Err() | 419 » » » » return errors.Annotate(err, "failed to mkdir").E
rr() |
434 } | 420 } |
435 } else { | 421 } else { |
436 if err := copyFileOpener(f.Open, path, fi); err != nil { | 422 if err := copyFileOpener(f.Open, path, fi); err != nil { |
437 return err | 423 return err |
438 } | 424 } |
439 } | 425 } |
440 } | 426 } |
441 return nil | 427 return nil |
442 } | 428 } |
443 | 429 |
444 func copyFileIntoDir(src, dstDir string) error { | 430 func copyFileIntoDir(src, dstDir string) error { |
445 return copyFile(src, filepath.Join(dstDir, filepath.Base(src)), nil) | 431 return copyFile(src, filepath.Join(dstDir, filepath.Base(src)), nil) |
446 } | 432 } |
447 | 433 |
448 func copyFile(src, dst string, fi os.FileInfo) error { | 434 func copyFile(src, dst string, fi os.FileInfo) error { |
449 opener := func() (io.ReadCloser, error) { return os.Open(src) } | 435 opener := func() (io.ReadCloser, error) { return os.Open(src) } |
450 return copyFileOpener(opener, dst, fi) | 436 return copyFileOpener(opener, dst, fi) |
451 } | 437 } |
452 | 438 |
453 func copyFileOpener(opener func() (io.ReadCloser, error), dst string, fi os.File
Info) (err error) { | 439 func copyFileOpener(opener func() (io.ReadCloser, error), dst string, fi os.File
Info) (err error) { |
454 sfd, err := opener() | 440 sfd, err := opener() |
455 if err != nil { | 441 if err != nil { |
456 » » return errors.Annotate(err).Reason("failed to open source").Err(
) | 442 » » return errors.Annotate(err, "failed to open source").Err() |
457 } | 443 } |
458 defer sfd.Close() | 444 defer sfd.Close() |
459 | 445 |
460 dfd, err := os.Create(dst) | 446 dfd, err := os.Create(dst) |
461 if err != nil { | 447 if err != nil { |
462 » » return errors.Annotate(err).Reason("failed to create destination
").Err() | 448 » » return errors.Annotate(err, "failed to create destination").Err(
) |
463 } | 449 } |
464 defer func() { | 450 defer func() { |
465 if closeErr := dfd.Close(); closeErr != nil && err == nil { | 451 if closeErr := dfd.Close(); closeErr != nil && err == nil { |
466 » » » err = errors.Annotate(closeErr).Reason("failed to close
destination").Err() | 452 » » » err = errors.Annotate(closeErr, "failed to close destina
tion").Err() |
467 } | 453 } |
468 }() | 454 }() |
469 | 455 |
470 if _, err := io.Copy(dfd, sfd); err != nil { | 456 if _, err := io.Copy(dfd, sfd); err != nil { |
471 » » return errors.Annotate(err).Reason("failed to copy file").Err() | 457 » » return errors.Annotate(err, "failed to copy file").Err() |
472 } | 458 } |
473 if fi != nil { | 459 if fi != nil { |
474 if err := os.Chmod(dst, fi.Mode()); err != nil { | 460 if err := os.Chmod(dst, fi.Mode()); err != nil { |
475 » » » return errors.Annotate(err).Reason("failed to chmod").Er
r() | 461 » » » return errors.Annotate(err, "failed to chmod").Err() |
476 } | 462 } |
477 } | 463 } |
478 return nil | 464 return nil |
479 } | 465 } |
OLD | NEW |