Mark removable Drive caches for cryptohome.

components/drive/chromeos/file_cache.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "components/drive/chromeos/file_cache.h"
 
-#include <unistd.h>
+#include <linux/fs.h>
+#include <sys/ioctl.h>
+#include <sys/xattr.h>
 
 #include <queue>
 #include <vector>
 
 #include "base/bind.h"
 #include "base/bind_helpers.h"
 #include "base/callback_helpers.h"
+#include "base/files/file.h"
 #include "base/files/file_enumerator.h"
 #include "base/files/file_util.h"
 #include "base/location.h"
 #include "base/logging.h"
 #include "base/metrics/histogram.h"
+#include "base/stl_util.h"
 #include "base/strings/string_util.h"
 #include "base/strings/stringprintf.h"
 #include "base/sys_info.h"
 #include "build/build_config.h"
 #include "components/drive/drive.pb.h"
 #include "components/drive/drive_api_util.h"
 #include "components/drive/file_system_core_util.h"
 #include "components/drive/resource_metadata_storage.h"
 #include "google_apis/drive/task_util.h"
 #include "net/base/filename_util.h"
 #include "net/base/mime_sniffer.h"
 #include "net/base/mime_util.h"
 
 namespace drive {
 namespace internal {
 namespace {
 
 // Returns ID extracted from the path.
 std::string GetIdFromPath(const base::FilePath& path) {
   return util::UnescapeCacheFileName(path.BaseName().AsUTF8Unsafe());
 }
 
 base::FilePath GetPathForId(const base::FilePath& cache_directory,
                             const std::string& id) {
   return cache_directory.Append(
       base::FilePath::FromUTF8Unsafe(util::EscapeCacheFileName(id)));
 }
 
+// Sets extended file attribute as |name| |value| pair.
+bool SetExtendedFileAttributes(const base::FilePath& path,
+    const std::string& name, const std::string& value) {
+  return setxattr(path.value().c_str(), name.c_str(), value.c_str(),
+      value.size() + 1, 0) == 0;
+}
+
+// Changes attributes of the file with |flags|, e.g. FS_NODUMP_FL (cryptohome
+// will remove Drive caches with this attribute).
+// See linux/fs.h for available flags, and chattr(1) which does similar thing.
+// Returns whether operation succeeded.
+bool SetFileAttributes(const base::FilePath& path, int64_t flags) {

[hashimoto, 2016/05/10 10:33:48]

Does this code work on a 32-bit device (e.g. arm)?

|flags|' type is always 64-bit here, but the kernel header says it's long which
can be 32-bit on a 32-bit devices.
https://chromium.googlesource.com/chromiumos/third_party/kernel/+/chromeos-3....

[oka, 2016/05/11 04:24:10]

Used long. Same for GetFileAttributes.

[hashimoto, 2016/05/11 06:29:12]

Please also replace int64_t above with long.
The same goes for almost all places which use int64_t.

If you don't want to add NOLINT everywhere, you can avoid it by typedef'ing long
(e.g. typedef long FileAttributes;)

[oka, 2016/05/11 09:35:04]

Done. Used typedef.

+  base::File file(path, base::File::FLAG_OPEN | base::File::FLAG_READ);
+  if (!file.IsValid()) return false;
+  if (ioctl(file.GetPlatformFile(), FS_IOC_SETFLAGS, &flags) < 0) {
+    PLOG(ERROR) << "ioctl: " << path.value();
+    return false;
+  }
+  return true;
+}
+
+// Gets file attributes similarly to lsattr(1). Returns flags or -1 on error.
+// See linux/fs.h for the definition of the returned flags e.g. FS_NODUMP_FL.
+int64_t GetFileAttributes(const base::FilePath& path) {
+  base::File file(path, base::File::FLAG_OPEN | base::File::FLAG_READ);
+  if (!file.IsValid()) {
+    return -1;
+  }
+  int64_t flags = 0;
+  if (ioctl(file.GetPlatformFile(), FS_IOC_GETFLAGS, &flags) < 0) {
+    PLOG(ERROR) << "ioctl: " << path.value();
+    return -1;
+  }
+  return flags;
+}
+
+// Marks the cache file to be removable by cryptohome.
+bool SetRemovable(const base::FilePath& path) {
+  int64_t flags = GetFileAttributes(path);
+  if (flags < 0) return false;
+  if ((flags & FS_NODUMP_FL) == FS_NODUMP_FL) return true;
+
+  return SetFileAttributes(path, flags | FS_NODUMP_FL);
+}
+
+// Marks the cache file to be unremovable by cryptohome.
+bool UnsetRemovable(const base::FilePath& path) {
+  int64_t flags = GetFileAttributes(path);
+  if (flags < 0) return false;
+  if ((flags & FS_NODUMP_FL) == 0) return true;
+
+  return SetFileAttributes(path, flags & ~(static_cast<int64_t>(FS_NODUMP_FL)));
+}
+
+// Marks |path| as drive cache dir.
+// Returns if the operation succeeded.
+bool MarkAsDriveCacheDir(const base::FilePath& path) {
+  return SetRemovable(path)
+      && SetExtendedFileAttributes(path, FileCache::kGCacheFilesAttribute, "");
+}
+
 typedef std::pair<base::File::Info, ResourceEntry> CacheInfo;
 
 class CacheInfoLatestCompare {
  public:
   bool operator()(const CacheInfo& info_a, const CacheInfo& info_b) {
     return info_a.first.last_accessed < info_b.first.last_accessed;
   }
 };
 
 const size_t kMaxNumOfEvictedCacheFiles = 30000;
 
 }  // namespace
 
+// static
+const std::string FileCache::kGCacheFilesAttribute = "user.GCacheFiles";
+
 FileCache::FileCache(ResourceMetadataStorage* storage,
                      const base::FilePath& cache_file_directory,
                      base::SequencedTaskRunner* blocking_task_runner,
                      FreeDiskSpaceGetterInterface* free_disk_space_getter)
     : cache_file_directory_(cache_file_directory),
       blocking_task_runner_(blocking_task_runner),
       storage_(storage),
       free_disk_space_getter_(free_disk_space_getter),
       max_num_of_evicted_cache_files_(kMaxNumOfEvictedCacheFiles),
       weak_ptr_factory_(this) {
@@ skipping 204 matching lines @@
     return FILE_ERROR_FAILED;
   }
 
   // Now that file operations have completed, update metadata.
   FileCacheEntry* cache_state =
       entry.mutable_file_specific_info()->mutable_cache_state();
   cache_state->set_md5(md5);
   cache_state->set_is_present(true);
   if (md5.empty())
     cache_state->set_is_dirty(true);
+
+  if (!cache_state->is_pinned() && !cache_state->is_dirty()) {
+    if (!SetRemovable(dest_path))
+      return FILE_ERROR_FAILED;
+  } else {
+    if (!UnsetRemovable(dest_path))
+      return FILE_ERROR_FAILED;
+  }
+
   return storage_->PutEntry(entry);
 }
 
 FileError FileCache::Pin(const std::string& id) {
   AssertOnSequencedWorkerPool();
 
   ResourceEntry entry;
   FileError error = storage_->GetEntry(id, &entry);
   if (error != FILE_ERROR_OK)
     return error;
+
   entry.mutable_file_specific_info()->mutable_cache_state()->set_is_pinned(
       true);
+
+  base::FilePath file_path = GetCacheFilePath(entry.local_id());
+  // Cache file can be created later.
+  if (base::PathExists(file_path)) {

[hashimoto, 2016/05/10 10:33:48]

Checking is_present() is better than base::PathExists as it's consistent with
other parts of this file?

[oka, 2016/05/11 04:24:10]

Done.

+    if (!UnsetRemovable(file_path))
+      return FILE_ERROR_FAILED;
+  }
+
   return storage_->PutEntry(entry);
 }
 
 FileError FileCache::Unpin(const std::string& id) {
   AssertOnSequencedWorkerPool();
 
   // Unpinning a file means its entry must exist in cache.
   ResourceEntry entry;
   FileError error = storage_->GetEntry(id, &entry);
   if (error != FILE_ERROR_OK)
     return error;
 
   // Now that file operations have completed, update metadata.
   if (entry.file_specific_info().cache_state().is_present()) {
     entry.mutable_file_specific_info()->mutable_cache_state()->set_is_pinned(
         false);
+    if (!entry.file_specific_info().cache_state().is_dirty()) {
+      if (!SetRemovable(GetCacheFilePath(entry.local_id())))
+        return FILE_ERROR_FAILED;
+    }
   } else {
     // Remove the existing entry if we are unpinning a non-present file.
     entry.mutable_file_specific_info()->clear_cache_state();
   }
   error = storage_->PutEntry(entry);
   if (error != FILE_ERROR_OK)
     return error;
 
   // Now it's a chance to free up space if needed.
   FreeDiskSpaceIfNeededFor(0);
@@ skipping 46 matching lines @@
   ResourceEntry entry;
   FileError error = storage_->GetEntry(id, &entry);
   if (error != FILE_ERROR_OK)
     return error;
   if (!entry.file_specific_info().cache_state().is_present()) {
     LOG(WARNING) << "Can't mark dirty a file that wasn't cached: " << id;
     return FILE_ERROR_NOT_FOUND;
   }
 
   entry.mutable_file_specific_info()->mutable_cache_state()->set_is_dirty(true);
+  if (!UnsetRemovable(GetCacheFilePath(entry.local_id())))
+    return FILE_ERROR_FAILED;
+
   entry.mutable_file_specific_info()->mutable_cache_state()->clear_md5();
   error = storage_->PutEntry(entry);
   if (error != FILE_ERROR_OK)
     return error;
 
   write_opened_files_[id]++;
   file_closer->reset(new base::ScopedClosureRunner(
       base::Bind(&google_apis::RunTaskWithTaskRunner,
                  blocking_task_runner_,
                  base::Bind(&FileCache::CloseForWrite,
@@ skipping 51 matching lines @@
 
   // If a file is not dirty (it should have been marked dirty via OpenForWrite),
   // clearing its dirty state is an invalid operation.
   if (!entry.file_specific_info().cache_state().is_dirty()) {
     LOG(WARNING) << "Can't clear dirty state of a non-dirty file: " << id;
     return FILE_ERROR_INVALID_OPERATION;
   }
 
   entry.mutable_file_specific_info()->mutable_cache_state()->set_is_dirty(
       false);
+  if (!entry.file_specific_info().cache_state().is_pinned()) {
+    if (!SetRemovable(GetCacheFilePath(entry.local_id())))
+      return FILE_ERROR_FAILED;
+  }
+
   return storage_->PutEntry(entry);
 }
 
 FileError FileCache::Remove(const std::string& id) {
   AssertOnSequencedWorkerPool();
 
   ResourceEntry entry;
 
   // If entry doesn't exist, nothing to do.
   FileError error = storage_->GetEntry(id, &entry);
@@ skipping 46 matching lines @@
           clear_md5();
       if (storage_->PutEntry(new_entry) != FILE_ERROR_OK)
         return false;
     }
   }
   if (it->HasError())
     return false;
 
   if (!RenameCacheFilesToNewFormat())
     return false;
+
+  // Run this every time to resolve inconsistency between metadata
+  // and file attributes which possibly occurs on abrupt power failure.
+  if (!MigrateCacheFilesResolvingInconsistency()) {
+    return false;
+  }
+
   return true;
 }
 
 void FileCache::Destroy() {
   DCHECK(thread_checker_.CalledOnValidThread());
 
   in_shutdown_.Set();
 
   // Destroy myself on the blocking pool.
   // Note that base::DeletePointer<> cannot be used as the destructor of this
@@ skipping 151 matching lines @@
       continue;
     }
     const std::string& id = GetIdFromPath(new_path);
     new_path = GetCacheFilePath(util::CanonicalizeResourceId(id));
     if (new_path != current && !base::Move(current, new_path))
       return false;
   }
   return true;
 }
 
-// static
-bool FileCache::MigrateCacheFiles(const base::FilePath& from,
-                                  const base::FilePath& to_files,
-                                  const base::FilePath& to_links,
-                                  ResourceMetadataStorage* metadata_storage) {
+bool FileCache::MigrateCacheFilesResolvingInconsistency() {
   std::unique_ptr<ResourceMetadataStorage::Iterator> it =
-      metadata_storage->GetIterator();
+      storage_->GetIterator();
+
   for (; !it->IsAtEnd(); it->Advance()) {
-    const ResourceEntry& entry = it->GetValue();
-    if (!entry.file_specific_info().cache_state().is_present()) {
-      continue;
+    ResourceEntry entry = it->GetValue();
+    const base::FilePath filepath = GetPathForId(cache_file_directory_,
+        entry.local_id());
+    const bool fileIsPresent = base::PathExists(filepath);

[hashimoto, 2016/05/10 10:33:48]

nit: The names of variables and data members should be all lowercase.
https://google.github.io/styleguide/cppguide.html#Variable_Names

[oka, 2016/05/11 04:24:10]

Done.

+
+    // Update metadata on inconsistency.
+    if (fileIsPresent !=
+        entry.file_specific_info().cache_state().is_present()) {
+      if (fileIsPresent) {
+        // This happens only on abrupt shutdown. Opposite is expected
+        // after cryptohome runs.
+        LOG(WARNING) << "File is present but metadata's state is inconsistent.";
+      }
+      entry.mutable_file_specific_info()->mutable_cache_state()->set_is_present(

[hashimoto, 2016/05/10 10:33:48]

!file exists && is_present case:
Shouldn't we clear is_dirty and md5 of the cache entry?

file exists && !is_present case:
Probably we should just remove the file instead of fixing the metadata, as we
have no clue about the state of the found file.
(probably we shouldn't, I'll also think about this...)

[oka, 2016/05/11 04:24:10]

Done.
I think removing the file is fine since this situation only happens when user
tries to remove the file but it failed halfway (due to abrupt shutdown or
something).

+          fileIsPresent);
+      if (storage_->PutEntry(entry) != FILE_ERROR_OK)
+        return false;
     }
 
-    // Ignore missing cache file case since it never succeeds.
-    // TODO(yawano): handle this case at metadata validation in FileCache.
-    const base::FilePath move_from = GetPathForId(from, entry.local_id());
-    if (!base::PathExists(move_from)) {
-      continue;
-    }
+    if (!fileIsPresent) continue;
 
-    // Create hard link to cache file if it's pinned or dirty. cryptohome will
-    // not delete a cache file if there is a hard link to it.
+    // Update file attribues for cryptohome.
     const FileCacheEntry& file_cache_entry =
         entry.file_specific_info().cache_state();
     if (file_cache_entry.is_pinned() || file_cache_entry.is_dirty()) {
-      const base::FilePath link_path = GetPathForId(to_links, entry.local_id());
-      int link_result = link(move_from.AsUTF8Unsafe().c_str(),
-                             link_path.AsUTF8Unsafe().c_str());
-      if (link_result != 0 && errno != EEXIST) {
-        return false;
-      }
-    }
-
-    // Move cache file.
-    const base::FilePath move_to = GetPathForId(to_files, entry.local_id());
-    if (!base::Move(move_from, move_to)) {
-      return false;
+      if (!UnsetRemovable(filepath)) return false;
+    } else {
+      if (!SetRemovable(filepath)) return false;
     }
   }
 
-  return true;
+  return MarkAsDriveCacheDir(cache_file_directory_);
 }
 
 void FileCache::CloseForWrite(const std::string& id) {
   AssertOnSequencedWorkerPool();
 
   std::map<std::string, int>::iterator it = write_opened_files_.find(id);
   if (it == write_opened_files_.end())
     return;
 
   DCHECK_LT(0, it->second);
@@ skipping 20 matching lines @@
 
 bool FileCache::IsEvictable(const std::string& id, const ResourceEntry& entry) {
   return entry.file_specific_info().has_cache_state() &&
          !entry.file_specific_info().cache_state().is_pinned() &&
          !entry.file_specific_info().cache_state().is_dirty() &&
          !mounted_files_.count(id);
 }
 
 }  // namespace internal
 }  // namespace drive