luci-py/isolateserver.py: Add archive support when downloading.

client/cipd.py

 # Copyright 2016 The LUCI Authors. All rights reserved.
 # Use of this source code is governed under the Apache License, Version 2.0
 # that can be found in the LICENSE file.
 
 """Fetches CIPD client and installs packages."""
 
 __version__ = '0.2'
 
 import collections
 import contextlib
@@ skipping 338 matching lines @@
     # It does not take a lot of disk space.
     version_cache = isolateserver.DiskCache(
         unicode(os.path.join(cache_dir, 'versions')),
         isolateserver.CachePolicies(0, 0, 300),
         hashlib.sha1)
     with version_cache:
       # Convert |version| to a string that may be used as a filename in disk
       # cache by hashing it.
       version_digest = hashlib.sha1(version).hexdigest()
       try:
-        instance_id = version_cache.read(version_digest)
+        instance_id = version_cache.getfileobj(version_digest).read()

[nodir, 2016/06/20 15:54:34]

DiskCache.getfileobject returns a file and it is responsibility of the caller to
close it
this and some other call sites do not close the file

[mithro, 2016/06/21 06:42:33]

I've changed this one to a "with xxxx".

See my comment about close later...

       except isolateserver.CacheMiss:
         instance_id = resolve_version(
             service_url, package_name, version, timeout=timeoutfn())
         version_cache.write(version_digest, instance_id)
 
   # instance_cache is {instance_id -> client binary} mapping.
   # It is bounded by 5 client versions.
   instance_cache = isolateserver.DiskCache(
       unicode(os.path.join(cache_dir, 'clients')),
       isolateserver.CachePolicies(0, 0, 5),
       hashlib.sha1)
   with instance_cache:
     if instance_id not in instance_cache:
       logging.info('Fetching CIPD client %s:%s', package_name, instance_id)
       fetch_url = get_client_fetch_url(
           service_url, package_name, instance_id, timeout=timeoutfn())
       _fetch_cipd_client(instance_cache, instance_id, fetch_url, timeoutfn)
 
     # A single host can run multiple swarming bots, but ATM they do not share
     # same root bot directory. Thus, it is safe to use the same name for the
     # binary.
     binary_path = unicode(os.path.join(cache_dir, 'cipd' + EXECUTABLE_SUFFIX))
     if fs.isfile(binary_path):
       file_path.remove(binary_path)
-    instance_cache.hardlink(instance_id, binary_path, 0511)  # -r-x--x--x
-
+    isolateserver.putfile(
+        instance_cache.getfileobj(instance_id),
+        binary_path,
+        0511)  # -r-x--x--x
     yield CipdClient(binary_path)
 
 
 def parse_package_list_file(path):
   """Returns a map {site_root_path: [(package, version)]} read from file.
 
   Slashes in site_root_path are replaced with os.path.sep.
   """
   with open(path) as f:
     try:
       parsed = json.load(f)
     except ValueError as ex:
       raise Error('Invalid package list file: %s' % ex)
 
   packages = collections.defaultdict(list)
   for package in parsed.get('packages') or []:
     path = package.get('path') or '.'
     path = path.replace('/', os.path.sep)
 
     name = package.get('package_name')
     if not name:
       raise Error('Invalid package list file: package name is not specified')
     version = package.get('version')
     if not version:
       raise Error('Invalid package list file: package version is not specified')
     packages[path].append((name, version))
   return packages