Make isolate cache resilient to unexpectedly deleted items.

client/isolateserver.py

 #!/usr/bin/env python
 # Copyright 2013 The LUCI Authors. All rights reserved.
 # Use of this source code is governed under the Apache License, Version 2.0
 # that can be found in the LICENSE file.
 
 """Archives a set of files or directories to an Isolate Server."""
 
 __version__ = '0.8.0'
 
-import base64
 import errno
 import functools
 import io
 import logging
 import optparse
 import os
 import re
 import signal
 import stat
 import sys
 import tarfile
 import tempfile
-import threading
 import time
-import types
 import zlib
 
 from third_party import colorama
 from third_party.depot_tools import fix_encoding
 from third_party.depot_tools import subcommand
 
 from libs import arfile
 from utils import file_path
 from utils import fs
 from utils import logging_utils
@@ skipping 305 matching lines @@
       os.symlink(properties['l'], outfile)  # pylint: disable=E1101
     except OSError as e:
       if e.errno == errno.EEXIST:
         raise AlreadyExists('File %s already exists.' % outfile)
       raise
 
 
 def is_valid_file(path, size):
   """Determines if the given files appears valid.
 
-  Currently it just checks the file's size.
+  Currently it just checks the file exists and its size matches the expectation.
   """
   if size == UNKNOWN_FILE_SIZE:
     return fs.isfile(path)
-  actual_size = fs.stat(path).st_size
+  try:
+    actual_size = fs.stat(path).st_size
+  except OSError as e:
+    logging.warning(
+        'Can\'t read item %s, assuming it\'s invalid: %s',
+        os.path.basename(path), e)
+    return False
   if size != actual_size:
     logging.warning(
         'Found invalid item %s; %d != %d',
         os.path.basename(path), actual_size, size)
     return False
   return True
 
 
 class FileItem(Item):
   """A file to push to Storage.
@@ skipping 797 matching lines @@
     # call are manually verified.
     #
     #with self._lock:
     #  for digest in self._lru:
     #    if not isolated_format.is_valid_hash(
     #        self._path(digest), self.hash_algo):
     #      self.evict(digest)
     #      logging.info('Deleted corrupted item: %s', digest)
 
   def touch(self, digest, size):
-    """Verifies an actual file is valid.
+    """Verifies an actual file is valid and bumps its LRU position.
+
+    Returns False if the file is missing or invalid. Doesn't kick it from LRU
+    though (call 'evict' explicitly).
 
     Note that is doesn't compute the hash so it could still be corrupted if the
     file size didn't change.
 
     TODO(maruel): More stringent verification while keeping the check fast.
     """
     # Do the check outside the lock.
     if not is_valid_file(self._path(digest), size):
       return False
 
@@ skipping 152 matching lines @@
   def _path(self, digest):
     """Returns the path to one item."""
     return os.path.join(self.cache_dir, digest)
 
   def _remove_lru_file(self, allow_protected):
     """Removes the lastest recently used file and returns its size."""
     self._lock.assert_locked()
     try:
       digest, (size, _) = self._lru.get_oldest()
       if not allow_protected and digest == self._protected:
-        raise Error('Not enough space to map the whole isolated tree')
+        raise Error('Not enough space to fetch the whole isolated tree')
     except KeyError:
       raise Error('Nothing to remove')
     digest, (size, _) = self._lru.pop_oldest()
-    logging.debug("Removing LRU file %s", digest)
+    logging.debug('Removing LRU file %s', digest)
     self._delete_file(digest, size)
     return size
 
   def _add(self, digest, size=UNKNOWN_FILE_SIZE):
     """Adds an item into LRU cache marking it as a newest one."""
     self._lock.assert_locked()
     if size == UNKNOWN_FILE_SIZE:
       size = fs.stat(self._path(digest)).st_size
     self._added.append(size)
     self._lru.add(digest, size)
     self._free_disk -= size
     # Do a quicker version of self._trim(). It only enforces free disk space,
     # not cache size limits. It doesn't actually look at real free disk space,
     # only uses its cache values. self._trim() will be called later to enforce
     # real trimming but doing this quick version here makes it possible to map
     # an isolated that is larger than the current amount of free disk space when
     # the cache size is already large.
     while (
         self.policies.min_free_space and
         self._lru and
         self._free_disk < self.policies.min_free_space):
       self._remove_lru_file(False)
 
   def _delete_file(self, digest, size=UNKNOWN_FILE_SIZE):
     """Deletes cache file from the file system."""
     self._lock.assert_locked()
     try:
       if size == UNKNOWN_FILE_SIZE:
-        size = fs.stat(self._path(digest)).st_size
+        try:
+          size = fs.stat(self._path(digest)).st_size
+        except OSError:
+          size = 0
       file_path.try_remove(self._path(digest))
       self._evicted.append(size)
       self._free_disk += size
     except OSError as e:
-      logging.error('Error attempting to delete a file %s:\n%s' % (digest, e))
+      if e.errno != errno.ENOENT:
+        logging.error('Error attempting to delete a file %s:\n%s' % (digest, e))
 
 
 class IsolatedBundle(object):
   """Fetched and parsed .isolated file with all dependencies."""
 
   def __init__(self):
     self.command = []
     self.files = {}
     self.read_only = None
     self.relative_cwd = None
@@ skipping 623 matching lines @@
   return dispatcher.execute(OptionParserIsolateServer(), args)
 
 
 if __name__ == '__main__':
   subprocess42.inhibit_os_error_reporting()
   fix_encoding.fix_encoding()
   tools.disable_buffering()
   colorama.init()
   file_path.enable_symlink()
   sys.exit(main(sys.argv[1:]))