gdata: Support mounting archive files in GData cache.

chrome/browser/chromeos/gdata/gdata_file_system.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "chrome/browser/chromeos/gdata/gdata_file_system.h"
 
 #include <errno.h>
 #include <sys/stat.h>
 
 #include <set>
@@ skipping 40 matching lines @@
 namespace gdata {
 namespace {
 
 const char kMimeTypeJson[] = "application/json";
 const char kMimeTypeOctetStream[] = "application/octet-stream";
 
 const FilePath::CharType kGDataRootDirectory[] = FILE_PATH_LITERAL("gdata");
 const char kFeedField[] = "feed";
 const char kWildCard[] = "*";
 const char kLocallyModifiedFileExtension[] = "local";
+const char kMountedArchiveFileExtension[] = "mounted";
 
 const FilePath::CharType kGDataCacheVersionDir[] = FILE_PATH_LITERAL("v1");
 const FilePath::CharType kGDataCacheMetaDir[] = FILE_PATH_LITERAL("meta");
 const FilePath::CharType kGDataCachePinnedDir[] = FILE_PATH_LITERAL("pinned");
 const FilePath::CharType kGDataCacheOutgoingDir[] =
     FILE_PATH_LITERAL("outgoing");
 const FilePath::CharType kGDataCachePersistentDir[] =
     FILE_PATH_LITERAL("persistent");
 const FilePath::CharType kGDataCacheTmpDir[] = FILE_PATH_LITERAL("tmp");
 const FilePath::CharType kGDataCacheTmpDownloadsDir[] =
@@ skipping 431 matching lines @@
     const std::string& md5,
     const FilePath& gdata_file_path,
     FilePath* cache_file_path) {
   DCHECK(error);
   DCHECK(cache_file_path);
 
   if (!callback.is_null())
     callback.Run(*error, resource_id, md5, gdata_file_path, *cache_file_path);
 }
 
+// Ditto for SetMountedStateCallback
+void RunSetMountedStateCallbackHelper(
+    const SetMountedStateCallback& callback,
+    base::PlatformFileError* error,
+    FilePath* cache_file_path) {
+  DCHECK(error);
+  DCHECK(cache_file_path);
+
+  if (!callback.is_null())
+    callback.Run(*error, *cache_file_path);
+}
+
 void RunGetCacheStateCallbackHelper(
     const GetCacheStateCallback& callback,
     base::PlatformFileError* error,
     int* cache_state) {
   DCHECK(error);
   DCHECK(cache_state);
 
   if (!callback.is_null())
     callback.Run(*error, *cache_state);
 }
@@ skipping 1410 matching lines @@
   if (regular_file) {
     properties->file_md5 = regular_file->file_md5();
     properties->mime_type = regular_file->content_mime_type();
     properties->content_url = regular_file->content_url();
     properties->alternate_url = regular_file->alternate_url();
     properties->is_hosted_document = regular_file->is_hosted_document();
   }
   return true;
 }
 
+bool GDataFileSystem::IsUnderGDataCacheDirectory(const FilePath& path) const {
+  return gdata_cache_path_ == path || gdata_cache_path_.IsParent(path);
+}
+
 FilePath GDataFileSystem::GetGDataCacheTmpDirectory() const {
   return cache_paths_[GDataRootDirectory::CACHE_TYPE_TMP];
 }
 
 FilePath GDataFileSystem::GetGDataTempDownloadFolderPath() const {
   return cache_paths_[GDataRootDirectory::CACHE_TYPE_TMP_DOWNLOADS];
 }
 
 FilePath GDataFileSystem::GetGDataTempDocumentFolderPath() const {
   return cache_paths_[GDataRootDirectory::CACHE_TYPE_TMP_DOCUMENTS];
@@ skipping 128 matching lines @@
                                 GetWeakPtrForCurrentThread(),
                                 callback);
   }
 
   if (to_pin)
     Pin(resource_id, md5, cache_callback);
   else
     Unpin(resource_id, md5, cache_callback);
 }
 
+void GDataFileSystem::SetMountedState(const FilePath& file_path, bool to_mount,
+                                      const SetMountedStateCallback& callback) {
+  InitializeCacheIfNecessary();
+
+  base::PlatformFileError* error =
+      new base::PlatformFileError(base::PLATFORM_FILE_OK);
+  FilePath* cache_file_path = new FilePath;
+  PostBlockingPoolSequencedTaskAndReply(
+      kGDataFileSystemToken,
+      FROM_HERE,
+      base::Bind(&GDataFileSystem::SetMountedStateOnIOThreadPool,
+                 base::Unretained(this),
+                 file_path,
+                 to_mount,
+                 error,
+                 cache_file_path),
+      base::Bind(&RunSetMountedStateCallbackHelper,
+                 callback,
+                 base::Owned(error),
+                 base::Owned(cache_file_path)));
+}
+
+void GDataFileSystem::SetMountedStateOnIOThreadPool(
+    const FilePath& file_path,
+    bool to_mount,
+    base::PlatformFileError *error,
+    FilePath* cache_file_path) {
+  DCHECK(error);
+  DCHECK(cache_file_path);
+
+  // Lock to access cache map.
+  base::AutoLock lock(lock_);
+
+  FilePath base_name = file_path.BaseName();
+  FilePath::StringType resource_id = base_name.RemoveExtension().value();
+  FilePath::StringType extension = base_name.Extension();
+  FilePath::StringType md5 = FilePath::StringType();

[satorux1, 2012/04/20 20:11:25]

nit: the following is sufficient

FilePath::StringType md5;

+  if (!extension.empty()) {
+    // FilePath::Extension returns ".", so strip it.
+    md5 = GDataEntry::UnescapeUtf8FileName(extension.substr(1));
+    if (md5 == kMountedArchiveFileExtension) {
+      DCHECK(!to_mount);
+      // Files that are already mounted have filenames in format of
+      // resource_id.md5.mounted, i.e. ".mounted" is the extension, so
+      // strip it and get the actual resource_id and md5.
+      base_name = base_name.RemoveExtension();
+      resource_id = GDataEntry::UnescapeUtf8FileName(
+          base_name.RemoveExtension().value());
+      extension = base_name.Extension();
+      md5 = !extension.empty() ?
+          GDataEntry::UnescapeUtf8FileName(extension.substr(1)) :
+          FilePath::StringType();
+    }
+  }

[satorux1, 2012/04/20 20:11:25]

This function is already large. can you factor out this part into a separate
function, like:

if (ParseCacheFileName(file_path, &resource_id, &md5, &extra_suffix)) {
  ...
}


where extra_suffix returns "mounted", so you can do something like

DCHECK(!to_mount || extra_suffix == "mounted");

You can put this function gdata_util.cc and write a unit test for it.

+  GDataRootDirectory::CacheEntry* entry = root_->GetCacheEntry(resource_id,
+                                                               md5);
+  if (!entry) {
+    *error = base::PLATFORM_FILE_ERROR_NOT_FOUND;
+    return;
+  }
+  if (to_mount == entry->IsMounted()) {
+    *error = base::PLATFORM_FILE_ERROR_INVALID_OPERATION;
+    return;
+  }
+
+  md5 = entry->md5;
+  // Get subdir types for the unmounted and mounted state.
+  GDataRootDirectory::CacheSubDirectoryType unmounted_subdir =
+      entry->IsPinned() ? GDataRootDirectory::CACHE_TYPE_PERSISTENT :
+                          GDataRootDirectory::CACHE_TYPE_TMP;
+  GDataRootDirectory::CacheSubDirectoryType mounted_subdir =
+      GDataRootDirectory::CACHE_TYPE_PERSISTENT;
+  // Gets path of the file if it were to be unmounted.
+  FilePath unmounted_path = GetCacheFilePath(resource_id, md5, unmounted_subdir,
+                                             CACHED_FILE_FROM_SERVER);
+  // Gets path of the file if it were to be mounted.
+  FilePath mounted_path = GetCacheFilePath(resource_id, md5, mounted_subdir,
+                                           CACHED_FILE_MOUNTED);
+
+  // Determine the source and destination paths for moving the cache blob.
+  FilePath source_path;
+  GDataRootDirectory::CacheSubDirectoryType dest_subdir;
+  int cache_state = entry->cache_state;
+  if (to_mount) {
+    source_path = unmounted_path;
+    *cache_file_path = mounted_path;
+    dest_subdir = mounted_subdir;
+    cache_state = GDataFile::SetCacheMounted(cache_state);
+  } else {
+    source_path = mounted_path;
+    *cache_file_path = unmounted_path;
+    dest_subdir = unmounted_subdir;
+    cache_state = GDataFile::ClearCacheMounted(cache_state);
+  }
+
+  // Move cache blob from source path to destination path.
+  *error = ModifyCacheState(source_path, *cache_file_path,
+                            GDataFileSystem::FILE_OPERATION_MOVE,
+                            FilePath(), false);
+  if (*error == base::PLATFORM_FILE_OK) {
+    // Now that cache operation is complete, update cache map
+    root_->UpdateCacheMap(resource_id, md5, dest_subdir, cache_state);
+  }
+}
+
 void GDataFileSystem::OnSetPinStateCompleted(
     const FileOperationCallback& callback,
     base::PlatformFileError error,
     const std::string& resource_id,
     const std::string& md5) {
   callback.Run(error);
 }
 
 void GDataFileSystem::OnGetAvailableSpace(
     const GetAvailableSpaceCallback& callback,
@@ skipping 1161 matching lines @@
   // name and md5 is the extension.
   std::string base_name = GDataEntry::EscapeUtf8FileName(resource_id);
   if (file_origin == CACHED_FILE_LOCALLY_MODIFIED) {
     DCHECK(sub_dir_type == GDataRootDirectory::CACHE_TYPE_PERSISTENT);
     base_name += FilePath::kExtensionSeparator;
     base_name += kLocallyModifiedFileExtension;
   } else if (!md5.empty()) {
     base_name += FilePath::kExtensionSeparator;
     base_name += GDataEntry::EscapeUtf8FileName(md5);
   }
+  // For mounted archives the filename is formatted as resource_id.md5.mounted,
+  // i.e. resource_id.md5 is the base name and ".mounted" is the extension
+  if (file_origin == CACHED_FILE_MOUNTED) {
+     DCHECK(sub_dir_type == GDataRootDirectory::CACHE_TYPE_PERSISTENT);
+     base_name += FilePath::kExtensionSeparator;
+     base_name += kMountedArchiveFileExtension;
+  }
   return cache_paths_[sub_dir_type].Append(base_name);
 }
 
 void GDataFileSystem::GetFromCache(const std::string& resource_id,
                                    const std::string& md5,
                                    const GetFromCacheCallback& callback) {
   GetFromCacheInternal(resource_id, md5, FilePath(), callback);
 }
 
 void GDataFileSystem::StoreToCache(const std::string& resource_id,
@@ skipping 218 matching lines @@
     FilePath* cache_file_path) {
   DCHECK(error);
   DCHECK(cache_file_path);
 
   // Lock to access cache map.
   base::AutoLock lock(lock_);
 
   GDataRootDirectory::CacheEntry* entry = root_->GetCacheEntry(resource_id,
                                                                md5);
   if (entry && entry->IsPresent()) {
+    CachedFileOrigin file_origin;
+    if (entry->IsMounted()) {
+      file_origin = CACHED_FILE_MOUNTED;
+    } else if (entry->IsDirty()) {
+      file_origin = CACHED_FILE_LOCALLY_MODIFIED;
+    } else {
+      file_origin = CACHED_FILE_FROM_SERVER;
+    }
     *cache_file_path = GetCacheFilePath(
         resource_id,
         md5,
         entry->sub_dir_type,
-        entry->IsDirty() ? CACHED_FILE_LOCALLY_MODIFIED :
-                           CACHED_FILE_FROM_SERVER);
+        file_origin);
     *error = base::PLATFORM_FILE_OK;
   } else {
     *error = base::PLATFORM_FILE_ERROR_NOT_FOUND;
   }
 }
 
 void GDataFileSystem::GetCacheStateOnIOThreadPool(
     const std::string& resource_id,
     const std::string& md5,
     base::PlatformFileError* error,
@@ skipping 36 matching lines @@
   int cache_state = GDataFile::CACHE_STATE_PRESENT;
   GDataRootDirectory::CacheSubDirectoryType sub_dir_type =
       GDataRootDirectory::CACHE_TYPE_TMP;
 
   GDataRootDirectory::CacheEntry* entry = root_->GetCacheEntry(
       resource_id, md5);
 
   // If file was previously pinned, store it in persistent dir and create
   // symlink in pinned dir.
   if (entry) {  // File exists in cache.
-    // If file is dirty, return error.
-    if (entry->IsDirty()) {
-      LOG(WARNING) << "Can't store a file to replace a dirty file: res_id="
-                   << resource_id
+    // If file is dirty or mounted, return error.
+    if (entry->IsDirty() || entry->IsMounted()) {
+      LOG(WARNING) << "Can't store a file to replace a "
+                   << (entry->IsDirty() ? "dirty" : "mounted")
+                   << " file: res_id=" << resource_id
                    << ", md5=" << md5;
       *error = base::PLATFORM_FILE_ERROR_IN_USE;
       return;
     }
 
     cache_state |= entry->cache_state;
 
     // If file is pinned, determines destination path.
     if (entry->IsPinned()) {
       sub_dir_type = GDataRootDirectory::CACHE_TYPE_PERSISTENT;
@@ skipping 78 matching lines @@
 
     // Set sub_dir_type to PINNED to indicate that the file doesn't exist.
     // When the file is finally downloaded and StoreToCache called, it will be
     // moved to persistent directory.
     sub_dir_type = GDataRootDirectory::CACHE_TYPE_PINNED;
   } else {  // File exists in cache, determines destination path.
     cache_state |= entry->cache_state;
 
     // Determine source and destination paths.
 
-    // If file is dirty, don't move it, so determine |dest_path| and set
-    // |source_path| the same, because ModifyCacheState only moves files if
+    // If file is dirty or mounted, don't move it, so determine |dest_path| and
+    // set |source_path| the same, because ModifyCacheState only moves files if
     // source and destination are different.
-    if (entry->IsDirty()) {
+    if (entry->IsDirty() || entry->IsMounted()) {
       DCHECK_EQ(GDataRootDirectory::CACHE_TYPE_PERSISTENT, entry->sub_dir_type);
       dest_path = GetCacheFilePath(resource_id,
                                    md5,
                                    entry->sub_dir_type,
                                    CACHED_FILE_LOCALLY_MODIFIED);
       source_path = dest_path;
     } else {
       // Gets the current path of the file in cache.
       source_path = GetCacheFilePath(resource_id,
                                      md5,
@@ skipping 59 matching lines @@
     return;
   }
 
   // Entry exists in cache, determines source and destination paths.
 
   FilePath source_path;
   FilePath dest_path;
   GDataRootDirectory::CacheSubDirectoryType sub_dir_type =
       GDataRootDirectory::CACHE_TYPE_TMP;
 
-  // If file is dirty, don't move it, so determine |dest_path| and set
-  // |source_path| the same, because ModifyCacheState moves files if source
+  // If file is dirty or mounted, don't move it, so determine |dest_path| and
+  // set |source_path| the same, because ModifyCacheState moves files if source
   // and destination are different.
-  if (entry->IsDirty()) {
+  if (entry->IsDirty() || entry->IsMounted()) {
     sub_dir_type = GDataRootDirectory::CACHE_TYPE_PERSISTENT;
     DCHECK_EQ(sub_dir_type, entry->sub_dir_type);
     dest_path = GetCacheFilePath(resource_id,
                                  md5,
                                  entry->sub_dir_type,
                                  CACHED_FILE_LOCALLY_MODIFIED);
     source_path = dest_path;
   } else {
     // Gets the current path of the file in cache.
     source_path = GetCacheFilePath(resource_id,
@@ skipping 311 matching lines @@
   // Lock to access cache map.
   base::AutoLock lock(lock_);
 
   // MD5 is not passed into RemoveFromCache and hence
   // RemoveFromCacheOnIOThreadPool, because we would delete all cache files
   // corresponding to <resource_id> regardless of the md5.
   // So, search for entry in cache without taking md5 into account.
   GDataRootDirectory::CacheEntry* entry = root_->GetCacheEntry(
       resource_id, std::string());
 
-  // If entry doesn't exist or is dirty in cache, nothing to do.
-  if (!entry || entry->IsDirty()) {
-    DVLOG(1) << "Entry " << (entry ? "is dirty" : "doesn't exist")
+  // If entry doesn't exist or is dirty or mounted in cache, nothing to do.
+  if (!entry || entry->IsDirty() || entry->IsMounted()) {
+    DVLOG(1) << "Entry is "
+             << (entry ? (entry->IsDirty() ? "dirty" : "mounted") :
+                         "non-existent")
              << " in cache, not removing";
     *error = base::PLATFORM_FILE_OK;
     return;
   }
 
   // Determine paths to delete all cache versions of |resource_id| in
   // persistent, tmp and pinned directories.
   std::vector<FilePath> paths_to_delete;
 
   // For files in persistent and tmp dirs, delete files that match
@@ skipping 129 matching lines @@
     // Determine cache state.
     int cache_state = GDataFile::CACHE_STATE_NONE;
     // If we're scanning pinned directory and if entry already exists, just
     // update its pinned state.
     if (sub_dir_type == GDataRootDirectory::CACHE_TYPE_PINNED) {
       GDataRootDirectory::CacheMap::iterator iter =
           cache_map->find(resource_id);
       if (iter != cache_map->end()) {  // Entry exists, update pinned state.
         GDataRootDirectory::CacheEntry* entry = iter->second;
         entry->cache_state = GDataFile::SetCachePinned(entry->cache_state);
+        // Check that the file exists in the persistent directory
+        FilePath expected_path = GetCacheFilePath(
+            resource_id, entry->md5,
+            GDataRootDirectory::CACHE_TYPE_PERSISTENT,
+            CACHED_FILE_FROM_SERVER);
+        if (!file_util::PathExists(expected_path))
+        {

[satorux1, 2012/04/20 20:11:25]

nit: move { to the previous line.

+          // If the pinned file is a mounted archive, it may be stripped of the
+          // suffix ".mounted" and moved to tmp. Move it again to persistent.

[satorux1, 2012/04/20 20:11:25]

I'm confused. when can it happen?

[hshi, 2012/04/20 20:20:30]

This happens when a zip file is pinned and also mounted at the last
signout/shutdown. The next time chrome starts, there will be a mounted blob at
persistent/resource_id.md5.mounted and a symlink under pinned/, but
ScanCacheDirectory is called first on the TMP/PERSISTENT subdirs and then on the
OUTGOING/PINNED. For the call on PERSISTENT, the code block I just added below
will strip the ".mounted" suffix and move the blob to tmp/ (because there is no
way to tell that it is a pinned file until we scan the pinned subdir). Later
when we arrive here, we will realize that the file got moved to the wrong
directory, hence moving it back to persistent).

[satorux1, 2012/04/20 21:46:10]

Oh I see. This is fairly tricky and hard to test... can we make it simpler?

I'd suggest to just remove the .mounted file. 

After all, this situation (shut down before unmounting) rarely happens, and we
can recover this by fetch the file again. If the file is pinned, GDataSyncClient
will fetch it automatically if not cached locally.

+          FilePath tmp_path = GetCacheFilePath(
+              resource_id, entry->md5,
+              GDataRootDirectory::CACHE_TYPE_TMP,
+              CACHED_FILE_FROM_SERVER);
+          DCHECK(file_util::PathExists(tmp_path));
+          file_util::Move(tmp_path, expected_path);
+        }
         continue;
       }
       // Entry doesn't exist, this is a special symlink that refers to
       // /dev/null; follow through to create an entry with the PINNED but not
       // PRESENT state.
       cache_state = GDataFile::SetCachePinned(cache_state);
     } else if (sub_dir_type == GDataRootDirectory::CACHE_TYPE_OUTGOING) {
       // If we're scanning outgoing directory, entry must exist, update its
       // dirty state.
       // If entry doesn't exist, it's a logic error from previous execution,
       // ignore this outgoing symlink and move on.
       GDataRootDirectory::CacheMap::iterator iter =
           cache_map->find(resource_id);
       if (iter != cache_map->end()) {  // Entry exists, update dirty state.
         GDataRootDirectory::CacheEntry* entry = iter->second;
         entry->cache_state = GDataFile::SetCacheDirty(entry->cache_state);
       } else {
         NOTREACHED() << "Dirty cache file MUST have actual file blob";
       }
       continue;
     } else {
       // Scanning other directories means that cache file is actually present.
       cache_state = GDataFile::SetCachePresent(cache_state);
     }
 
+    // Mounted archives in cache should be unmounted upon logout/shutdown.
+    // But if we encounter a mounted file at start, move it back to tmp.
+    if (md5 == kMountedArchiveFileExtension) {
+      DCHECK(sub_dir_type == GDataRootDirectory::CACHE_TYPE_PERSISTENT);
+      sub_dir_type = GDataRootDirectory::CACHE_TYPE_TMP;
+      // Files that are already mounted have filenames in format of
+      // resource_id.md5.mounted, i.e. ".mounted" is the extension, so strip
+      // it and get the actual resource_id and md5
+      base_name = base_name.RemoveExtension();
+      resource_id = GDataEntry::UnescapeUtf8FileName(
+          base_name.RemoveExtension().value());
+      FilePath::StringType extension = base_name.Extension();
+      md5 = !extension.empty() ?
+          GDataEntry::UnescapeUtf8FileName(extension.substr(1)) :
+          std::string();
+      file_util::Move(current, GetCacheFilePath(resource_id, md5, sub_dir_type,
+                                                CACHED_FILE_FROM_SERVER));
+    }
+
     // Create and insert new entry into cache map.
     GDataRootDirectory::CacheEntry* entry = new GDataRootDirectory::CacheEntry(
         md5, sub_dir_type, cache_state);
     cache_map->insert(std::make_pair(resource_id, entry));
   }
 }
 
 void GDataFileSystem::GetFromCacheInternal(
     const std::string& resource_id,
     const std::string& md5,
@@ skipping 84 matching lines @@
   pref_registrar_->Init(profile_->GetPrefs());
   pref_registrar_->Add(prefs::kDisableGDataHostedFiles, this);
 }
 
 void SetFreeDiskSpaceGetterForTesting(FreeDiskSpaceGetterInterface* getter) {
   delete global_free_disk_getter_for_testing;  // Safe to delete NULL;
   global_free_disk_getter_for_testing = getter;
 }
 
 }  // namespace gdata