Mark removable Drive caches for cryptohome.

components/drive/chromeos/file_cache.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "components/drive/chromeos/file_cache.h"
 
-#include <unistd.h>
+#include <linux/fs.h>
+#include <sys/ioctl.h>
+#include <sys/xattr.h>
 
 #include <queue>
 #include <vector>
 
 #include "base/bind.h"
 #include "base/bind_helpers.h"
 #include "base/callback_helpers.h"
+#include "base/files/file.h"
 #include "base/files/file_enumerator.h"
 #include "base/files/file_util.h"
 #include "base/location.h"
 #include "base/logging.h"
 #include "base/metrics/histogram.h"
+#include "base/stl_util.h"
 #include "base/strings/string_util.h"
 #include "base/strings/stringprintf.h"
 #include "base/sys_info.h"
 #include "build/build_config.h"
 #include "components/drive/drive.pb.h"
 #include "components/drive/drive_api_util.h"
 #include "components/drive/file_system_core_util.h"
 #include "components/drive/resource_metadata_storage.h"
 #include "google_apis/drive/task_util.h"
 #include "net/base/filename_util.h"
 #include "net/base/mime_sniffer.h"
 #include "net/base/mime_util.h"
 
 namespace drive {
 namespace internal {
 namespace {
 
 // Returns ID extracted from the path.
 std::string GetIdFromPath(const base::FilePath& path) {
   return util::UnescapeCacheFileName(path.BaseName().AsUTF8Unsafe());
 }
 
 base::FilePath GetPathForId(const base::FilePath& cache_directory,
                             const std::string& id) {
   return cache_directory.Append(
       base::FilePath::FromUTF8Unsafe(util::EscapeCacheFileName(id)));
 }
 
+// Sets extended file attribute as |name| |value| pair.
+bool SetExtendedFileAttributes(const base::FilePath& path,
+    const std::string& name, const std::string& value) {
+  return setxattr(path.value().c_str(), name.c_str(), value.c_str(),
+      value.size() + 1, 0) == 0;
+}
+
+// Changes attributes of the file with |flags|, e.g. FS_NODUMP_FL (cryptohome
+// will remove Drive caches with this attribute).
+// See linux/fs.h for available flags, and chattr(1) which does similar thing.
+// Returns whether operation succeeded.
+bool SetFileAttributes(const base::FilePath& path, int64_t flags) {
+  base::File file(path, base::File::FLAG_OPEN | base::File::FLAG_READ);
+  if (!file.IsValid()) {
+    PLOG(ERROR) << "Failed to open file: " << path.value();
+    return false;
+  }
+  long f = static_cast<long>(flags);  // NOLINT long
+  if (ioctl(file.GetPlatformFile(), FS_IOC_SETFLAGS, &f) < 0) {
+    PLOG(ERROR) << "ioctl: " << path.value();
+    return false;
+  }
+  return true;
+}
+
+// Gets file attributes similarly to lsattr(1). Returns flags or -1 on error.
+// See linux/fs.h for the definition of the returned flags e.g. FS_NODUMP_FL.
+int64_t GetFileAttributes(const base::FilePath& path) {
+  base::File file(path, base::File::FLAG_OPEN | base::File::FLAG_READ);
+  if (!file.IsValid()) {
+    PLOG(ERROR) << "Failed to open file: " << path.value();
+    return -1;
+  }
+  long flags = 0;  // NOLINT long
+  if (ioctl(file.GetPlatformFile(), FS_IOC_GETFLAGS, &flags) < 0) {
+    PLOG(ERROR) << "ioctl: " << path.value();
+    return -1;
+  }
+  return static_cast<int64_t>(flags);
+}
+
+// Marks the cache file to be removable by cryptohome.
+bool SetRemovable(const base::FilePath& path) {
+  int64_t flags = GetFileAttributes(path);
+  if (flags < 0) return false;
+  if ((flags & FS_NODUMP_FL) == FS_NODUMP_FL) return true;
+
+  return SetFileAttributes(path, flags | FS_NODUMP_FL);
+}
+
+// Marks the cache file to be unremovable by cryptohome.
+bool UnsetRemovable(const base::FilePath& path) {
+  int64_t flags = GetFileAttributes(path);
+  if (flags < 0) return false;
+  if ((flags & FS_NODUMP_FL) == 0) return true;
+
+  return SetFileAttributes(path, flags & ~(static_cast<int64_t>(FS_NODUMP_FL)));
+}
+
+// Marks |path| as drive cache dir.
+// Returns if the operation succeeded.
+bool MarkAsDriveCacheDir(const base::FilePath& path) {
+  return SetRemovable(path)
+      && SetExtendedFileAttributes(path, FileCache::kGCacheFilesAttribute, "");
+}
+
 typedef std::pair<base::File::Info, ResourceEntry> CacheInfo;
 
 class CacheInfoLatestCompare {
  public:
   bool operator()(const CacheInfo& info_a, const CacheInfo& info_b) {
     return info_a.first.last_accessed < info_b.first.last_accessed;
   }
 };
 
 const size_t kMaxNumOfEvictedCacheFiles = 30000;
 
 }  // namespace
 
+// static
+const char FileCache::kGCacheFilesAttribute[] = "user.GCacheFiles";
+
 FileCache::FileCache(ResourceMetadataStorage* storage,
                      const base::FilePath& cache_file_directory,
                      base::SequencedTaskRunner* blocking_task_runner,
                      FreeDiskSpaceGetterInterface* free_disk_space_getter)
     : cache_file_directory_(cache_file_directory),
       blocking_task_runner_(blocking_task_runner),
       storage_(storage),
       free_disk_space_getter_(free_disk_space_getter),
       max_num_of_evicted_cache_files_(kMaxNumOfEvictedCacheFiles),
       weak_ptr_factory_(this) {
@@ skipping 204 matching lines @@
     return FILE_ERROR_FAILED;
   }
 
   // Now that file operations have completed, update metadata.
   FileCacheEntry* cache_state =
       entry.mutable_file_specific_info()->mutable_cache_state();
   cache_state->set_md5(md5);
   cache_state->set_is_present(true);
   if (md5.empty())
     cache_state->set_is_dirty(true);
+
+  if (!cache_state->is_pinned() && !cache_state->is_dirty()) {
+    if (!SetRemovable(dest_path))
+      return FILE_ERROR_FAILED;
+  } else {
+    if (!UnsetRemovable(dest_path))
+      return FILE_ERROR_FAILED;
+  }
+
   return storage_->PutEntry(entry);
 }
 
 FileError FileCache::Pin(const std::string& id) {
   AssertOnSequencedWorkerPool();
 
   ResourceEntry entry;
   FileError error = storage_->GetEntry(id, &entry);
   if (error != FILE_ERROR_OK)
     return error;
+
   entry.mutable_file_specific_info()->mutable_cache_state()->set_is_pinned(
       true);
+
+  base::FilePath file_path = GetCacheFilePath(entry.local_id());
+  // Cache file can be created later.
+  if (entry.file_specific_info().cache_state().is_present()) {
+    if (!UnsetRemovable(file_path))
+      return FILE_ERROR_FAILED;
+  }
+
   return storage_->PutEntry(entry);
 }
 
 FileError FileCache::Unpin(const std::string& id) {
   AssertOnSequencedWorkerPool();
 
   // Unpinning a file means its entry must exist in cache.
   ResourceEntry entry;
   FileError error = storage_->GetEntry(id, &entry);
   if (error != FILE_ERROR_OK)
     return error;
 
   // Now that file operations have completed, update metadata.
   if (entry.file_specific_info().cache_state().is_present()) {
     entry.mutable_file_specific_info()->mutable_cache_state()->set_is_pinned(
         false);
+    if (!entry.file_specific_info().cache_state().is_dirty()) {
+      if (!SetRemovable(GetCacheFilePath(entry.local_id())))
+        return FILE_ERROR_FAILED;
+    }
   } else {
     // Remove the existing entry if we are unpinning a non-present file.
     entry.mutable_file_specific_info()->clear_cache_state();
   }
   error = storage_->PutEntry(entry);
   if (error != FILE_ERROR_OK)
     return error;
 
   // Now it's a chance to free up space if needed.
   FreeDiskSpaceIfNeededFor(0);
@@ skipping 46 matching lines @@
   ResourceEntry entry;
   FileError error = storage_->GetEntry(id, &entry);
   if (error != FILE_ERROR_OK)
     return error;
   if (!entry.file_specific_info().cache_state().is_present()) {
     LOG(WARNING) << "Can't mark dirty a file that wasn't cached: " << id;
     return FILE_ERROR_NOT_FOUND;
   }
 
   entry.mutable_file_specific_info()->mutable_cache_state()->set_is_dirty(true);
+  if (!UnsetRemovable(GetCacheFilePath(entry.local_id())))
+    return FILE_ERROR_FAILED;
+
   entry.mutable_file_specific_info()->mutable_cache_state()->clear_md5();
   error = storage_->PutEntry(entry);
   if (error != FILE_ERROR_OK)
     return error;
 
   write_opened_files_[id]++;
   file_closer->reset(new base::ScopedClosureRunner(
       base::Bind(&google_apis::RunTaskWithTaskRunner,
                  blocking_task_runner_,
                  base::Bind(&FileCache::CloseForWrite,
@@ skipping 51 matching lines @@
 
   // If a file is not dirty (it should have been marked dirty via OpenForWrite),
   // clearing its dirty state is an invalid operation.
   if (!entry.file_specific_info().cache_state().is_dirty()) {
     LOG(WARNING) << "Can't clear dirty state of a non-dirty file: " << id;
     return FILE_ERROR_INVALID_OPERATION;
   }
 
   entry.mutable_file_specific_info()->mutable_cache_state()->set_is_dirty(
       false);
+  if (!entry.file_specific_info().cache_state().is_pinned()) {
+    if (!SetRemovable(GetCacheFilePath(entry.local_id())))
+      return FILE_ERROR_FAILED;
+  }
+
   return storage_->PutEntry(entry);
 }
 
 FileError FileCache::Remove(const std::string& id) {
   AssertOnSequencedWorkerPool();
 
   ResourceEntry entry;
 
   // If entry doesn't exist, nothing to do.
   FileError error = storage_->GetEntry(id, &entry);
@@ skipping 46 matching lines @@
           clear_md5();
       if (storage_->PutEntry(new_entry) != FILE_ERROR_OK)
         return false;
     }
   }
   if (it->HasError())
     return false;
 
   if (!RenameCacheFilesToNewFormat())
     return false;
+
+  // Run this every time to resolve inconsistency between metadata
+  // and file attributes which possibly occurs on abrupt power failure.
+  if (!MigrateCacheFilesResolvingInconsistency()) {
+    return false;
+  }
+
   return true;
 }
 
 void FileCache::Destroy() {
   DCHECK(thread_checker_.CalledOnValidThread());
 
   in_shutdown_.Set();
 
   // Destroy myself on the blocking pool.
   // Note that base::DeletePointer<> cannot be used as the destructor of this
@@ skipping 151 matching lines @@
       continue;
     }
     const std::string& id = GetIdFromPath(new_path);
     new_path = GetCacheFilePath(util::CanonicalizeResourceId(id));
     if (new_path != current && !base::Move(current, new_path))
       return false;
   }
   return true;
 }
 
-// static
-bool FileCache::MigrateCacheFiles(const base::FilePath& from,
-                                  const base::FilePath& to_files,
-                                  const base::FilePath& to_links,
-                                  ResourceMetadataStorage* metadata_storage) {
+bool FileCache::MigrateCacheFilesResolvingInconsistency() {
   std::unique_ptr<ResourceMetadataStorage::Iterator> it =
-      metadata_storage->GetIterator();
+      storage_->GetIterator();
+
   for (; !it->IsAtEnd(); it->Advance()) {
-    const ResourceEntry& entry = it->GetValue();
-    if (!entry.file_specific_info().cache_state().is_present()) {
+    ResourceEntry entry = it->GetValue();
+    FileCacheEntry* file_cache_entry =
+        entry.mutable_file_specific_info()->mutable_cache_state();
+
+    const base::FilePath filepath = GetPathForId(cache_file_directory_,
+        entry.local_id());
+    const bool file_is_present = base::PathExists(filepath);
+
+    // Delete file if the file is present but metadata says not.
+    // It happens only on abrupt shutdown.
+    if (file_is_present && !file_cache_entry->is_present()) {
+      LOG(WARNING) << "File is present but metadata's state was inconsistent.";
+
+      if (!base::DeleteFile(filepath, false /* recursive */))

[hashimoto, 2016/05/11 06:29:12]

Sorry for not saying this earlier, but I realized that this code is not needed
as FreeDiskSpaceIfNeededFor() is already deleting files without corresponding
metadata entries.
FreeDiskSpaceIfNeededFor() uses base::FileEnumerator so it also deletes files
which cannot be found with the storage iterator.

[oka, 2016/05/11 09:35:04]

Discussed offline. Keeping this logic will be OK, since 1.
FreeDiskSpaceIfNeededFor do nothing if there's enough space. 2. We do nothing
special to detect this case.

+        return false;
+      continue;
+    }
+    if (!file_is_present) {
+      // Update metatadata if there is no file but metadata says there is.
+      // It happens when cryptohome removed the file.
+      if (file_cache_entry->is_present()) {
+        file_cache_entry->set_is_present(false);
+        file_cache_entry->set_is_pinned(false);
+        file_cache_entry->set_is_dirty(false);
+        file_cache_entry->clear_md5();
+        if (storage_->PutEntry(entry) != FILE_ERROR_OK)
+          return false;
+      }
       continue;
     }
 
-    // Ignore missing cache file case since it never succeeds.
-    // TODO(yawano): handle this case at metadata validation in FileCache.
-    const base::FilePath move_from = GetPathForId(from, entry.local_id());
-    if (!base::PathExists(move_from)) {
-      continue;
-    }
-
-    // Create hard link to cache file if it's pinned or dirty. cryptohome will
-    // not delete a cache file if there is a hard link to it.
-    const FileCacheEntry& file_cache_entry =
-        entry.file_specific_info().cache_state();
-    if (file_cache_entry.is_pinned() || file_cache_entry.is_dirty()) {
-      const base::FilePath link_path = GetPathForId(to_links, entry.local_id());
-      int link_result = link(move_from.AsUTF8Unsafe().c_str(),
-                             link_path.AsUTF8Unsafe().c_str());
-      if (link_result != 0 && errno != EEXIST) {
-        return false;
-      }
-    }
-
-    // Move cache file.
-    const base::FilePath move_to = GetPathForId(to_files, entry.local_id());
-    if (!base::Move(move_from, move_to)) {
-      return false;
+    // Update file attribues for cryptohome.

[hashimoto, 2016/05/11 06:29:12]

As the proposed code is using "continue" in many places, it's a bit difficult to
figure out this line can be reached only when file_is_present &&
file_cache_entry->is_present().

How about structuring the code like this without using "continue"?

if (base::PathExists()) {
  if (file_cache_entry->is_present()) {
    (set attributes)
  } else {
    (remove file)
  }
} else {
  if (file_cache_entry->is_present()) {
    (update metadata)
  }
}

[oka, 2016/05/11 09:35:04]

Done.

+    if (file_cache_entry->is_pinned() || file_cache_entry->is_dirty()) {
+      if (!UnsetRemovable(filepath)) return false;
+    } else {
+      if (!SetRemovable(filepath)) return false;
     }
   }
 
-  return true;
+  return MarkAsDriveCacheDir(cache_file_directory_);
 }
 
 void FileCache::CloseForWrite(const std::string& id) {
   AssertOnSequencedWorkerPool();
 
   std::map<std::string, int>::iterator it = write_opened_files_.find(id);
   if (it == write_opened_files_.end())
     return;
 
   DCHECK_LT(0, it->second);
@@ skipping 20 matching lines @@
 
 bool FileCache::IsEvictable(const std::string& id, const ResourceEntry& entry) {
   return entry.file_specific_info().has_cache_state() &&
          !entry.file_specific_info().cache_state().is_pinned() &&
          !entry.file_specific_info().cache_state().is_dirty() &&
          !mounted_files_.count(id);
 }
 
 }  // namespace internal
 }  // namespace drive