Cache/retrieve extracted CIPD packages in local isolate cache

client/isolateserver.py

 #!/usr/bin/env python
 # Copyright 2013 The LUCI Authors. All rights reserved.
 # Use of this source code is governed under the Apache License, Version 2.0
 # that can be found in the LICENSE file.
 
 """Archives a set of files or directories to an Isolate Server."""
 
 __version__ = '0.8.0'
 
 import errno
@@ skipping 346 matching lines @@
         os.path.basename(path), e)
     return False
   if size != actual_size:
     logging.warning(
         'Found invalid item %s; %d != %d',
         os.path.basename(path), actual_size, size)
     return False
   return True
 
 
+def is_cached(ifile, cache):
+  """Determines if all the isolated file's contents are in the given LocalCache
+
+  """
+  files = ifile.data.get(u'files', {})
+  for f in files.keys():
+    # Can't do for f, props in files.keys() otherwise we see "too many
+    # values to unpack".
+    props = files.get(f, None)
+    if not props:
+      logging.warning('Problem getting info for %s', f)
+      return False
+    digest = props.get('h', None)
+    if not digest:
+      logging.warning('Hash can\'t be empty %s', f)
+      return False
+    if digest not in cache:
+      logging.info('File with digest %s is missing', digest)
+      return False
+  return True
+
+
+def extract(ifile, target_dir, cache):
+  """Extracts the isolated file's contents to target dir.
+
+  It stops if any couldn't be found.
+  """
+  files = ifile.data.get(u'files', {})
+  for f in files.keys():
+    # Can't do for f, props in files.keys() otherwise we see "too many
+    # values to unpack".
+    props = files.get(f, None)
+    if not props:
+      logging.warning('Problem getting info for %s', f)
+      return False
+    file_mode = props.get('m', None)
+    if file_mode:
+      # Ignore all bits apart from the user
+      file_mode &= 0700
+
+    dstpath = os.path.join(target_dir, f)
+    file_path.ensure_tree(os.path.dirname(dstpath))
+    digest = props.get('h', None)
+    if not digest:
+      logging.warning('Hash can\'t be empty %s', f)
+      return False
+    if digest not in cache:
+      logging.info('File with digest %s is missing', digest)
+      return False
+    srcpath = cache.getfileobj(digest).name
+
+    file_path.link_file(unicode(dstpath), unicode(srcpath),
+                        file_path.HARDLINK_WITH_FALLBACK)
+
+    if file_mode is not None:
+      fs.chmod(dstpath, file_mode)
+  return True
+
+
 class FileItem(Item):
   """A file to push to Storage.
 
   Its digest and size may be provided in advance, if known. Otherwise they will
   be derived from the file content.
   """
 
   def __init__(self, path, digest=None, size=None, high_priority=False):
     super(FileItem, self).__init__(
         digest,
@@ skipping 1335 matching lines @@
             logging.info(msg)
             last_update = time.time()
 
   # Cache could evict some items we just tried to fetch, it's a fatal error.
   if not fetch_queue.verify_all_cached():
     raise isolated_format.MappingError(
         'Cache is too small to hold all requested files')
   return bundle
 
 
-def directory_to_metadata(root, algo, blacklist):
+def directory_to_metadata(root, algo, blacklist, collapse_symlinks):
   """Returns the FileItem list and .isolated metadata for a directory."""
   root = file_path.get_native_path_case(root)
   paths = isolated_format.expand_directory_and_symlink(
       root, '.' + os.path.sep, blacklist, sys.platform != 'win32')
   metadata = {
     relpath: isolated_format.file_to_metadata(
-        os.path.join(root, relpath), {}, 0, algo, False)
+        os.path.join(root, relpath), {}, 0, algo, collapse_symlinks)
     for relpath in paths
   }
   for v in metadata.itervalues():
     v.pop('t')
   items = [
       FileItem(
           path=os.path.join(root, relpath),
           digest=meta['h'],
           size=meta['s'],
           high_priority=relpath.endswith('.isolated'))
@@ skipping 25 matching lines @@
   tempdir = None
   try:
     # TODO(maruel): Yield the files to a worker thread.
     items_to_upload = []
     for f in files:
       try:
         filepath = os.path.abspath(f)
         if fs.isdir(filepath):
           # Uploading a whole directory.
           items, metadata = directory_to_metadata(
-              filepath, storage.hash_algo, blacklist)
+              filepath, storage.hash_algo, blacklist, False)
 
           # Create the .isolated file.
           if not tempdir:
             tempdir = tempfile.mkdtemp(prefix=u'isolateserver')
           handle, isolated = tempfile.mkstemp(dir=tempdir, suffix=u'.isolated')
           os.close(handle)
           data = {
               'algo':
                   isolated_format.SUPPORTED_ALGOS_REVERSE[storage.hash_algo],
               'files': metadata,
@@ skipping 260 matching lines @@
   return dispatcher.execute(OptionParserIsolateServer(), args)
 
 
 if __name__ == '__main__':
   subprocess42.inhibit_os_error_reporting()
   fix_encoding.fix_encoding()
   tools.disable_buffering()
   colorama.init()
   file_path.enable_symlink()
   sys.exit(main(sys.argv[1:]))