Database support for GDataDirectoryService.

chrome/browser/chromeos/gdata/gdata_files.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "chrome/browser/chromeos/gdata/gdata_files.h"
 
+#include <leveldb/db.h>
 #include <vector>
 
 #include "base/message_loop_proxy.h"
 #include "base/platform_file.h"
+#include "base/string_number_conversions.h"
 #include "base/string_util.h"
 #include "base/stringprintf.h"
+#include "base/sequenced_task_runner.h"
 #include "base/tracked_objects.h"
 #include "base/utf_string_conversions.h"
 #include "chrome/browser/chromeos/gdata/gdata.pb.h"
 #include "chrome/browser/chromeos/gdata/gdata_util.h"
 #include "chrome/browser/chromeos/gdata/gdata_wapi_parser.h"
+#include "content/public/browser/browser_thread.h"
 #include "net/base/escape.h"
 
+using content::BrowserThread;
+
 namespace gdata {
 namespace {
 
 const char kSlash[] = "/";
 const char kEscapedSlash[] = "\xE2\x88\x95";
+const char kDBKeyLargestChangestamp[] = "m:largest_changestamp";

[satorux1, 2012/08/03 20:19:16]

Maybe add

// m: prefix is used for global metadata of the file system
// r: prefix is used for resource id to entry proto pairs

[achuithb, 2012/08/03 21:41:04]

Done.

+const char kDBKeyVersion[] = "m:version";
+const char kDBKeyResourceIdPrefix[] = "r:";
 
 // Extracts resource_id out of edit url.
 std::string ExtractResourceId(const GURL& url) {
   return net::UnescapeURLComponent(url.ExtractFileName(),
                                    net::UnescapeRule::URL_SPECIAL_CHARS);
 }
 
 // Returns true if |proto| is a valid proto as the root directory.
 // Used to reject incompatible proto.
 bool IsValidRootDirectoryProto(const GDataDirectoryProto& proto) {
@@ skipping 381 matching lines @@
     GDataDirectory* dir = iter->second;
     // Remove directories recursively.
     dir->RemoveChildren();
     if (directory_service_)
       directory_service_->RemoveEntryFromResourceMap(dir);
   }
   STLDeleteValues(&child_directories_);
   child_directories_.clear();
 }
 
+// ResourceMetadataDB implementation.
+
+// Params for GDatadirectoryServiceDB::Create.
+struct CreateDBParams {
+  CreateDBParams(const FilePath& db_path,
+                 base::SequencedTaskRunner* blocking_task_runner)
+                 : db_path(db_path),
+                   blocking_task_runner(blocking_task_runner) {
+  }
+
+  FilePath db_path;
+  scoped_refptr<base::SequencedTaskRunner> blocking_task_runner;
+  scoped_ptr<ResourceMetadataDB> db;
+  GDataDirectoryService::SerializedMap serialized_resources;
+};
+
+// Wrapper for level db. All methods must be called on blocking thread.
+class ResourceMetadataDB {
+ public:
+  ResourceMetadataDB(const FilePath& db_path,
+                          base::SequencedTaskRunner* blocking_task_runner);
+
+  // Initializes the database.
+  void Init();
+
+  // Reads the database into |serialized_resources|.
+  void Read(GDataDirectoryService::SerializedMap* serialized_resources);
+
+  // Saves |serialized_resources| to the database.
+  void Save(const GDataDirectoryService::SerializedMap& serialized_resources);
+
+ private:
+  // Clears the database.
+  void Clear();
+
+  scoped_refptr<base::SequencedTaskRunner> blocking_task_runner_;
+  scoped_ptr<leveldb::DB> level_db_;
+  FilePath db_path_;
+};
+
+ResourceMetadataDB::ResourceMetadataDB(const FilePath& db_path,
+    base::SequencedTaskRunner* blocking_task_runner)
+  : blocking_task_runner_(blocking_task_runner),
+    db_path_(db_path) {
+  DCHECK(blocking_task_runner_->RunsTasksOnCurrentThread());
+}
+
+// Creates, initializes and reads from the database.
+// This must be defined after ResourceMetadataDB and CreateDBParams.
+static void CreateResourceMetadataDBOnBlockingPool(
+    CreateDBParams* params) {
+  DCHECK(params->blocking_task_runner->RunsTasksOnCurrentThread());
+  DCHECK(!params->db_path.empty());
+
+  params->db.reset(new ResourceMetadataDB(params->db_path,
+                                               params->blocking_task_runner));
+  params->db->Init();
+  params->db->Read(&params->serialized_resources);
+}
+
+void ResourceMetadataDB::Init() {
+  DCHECK(blocking_task_runner_->RunsTasksOnCurrentThread());
+  DCHECK(!db_path_.empty());
+
+  DVLOG(1) << "Init " << db_path_.value();
+
+  leveldb::DB* level_db = NULL;
+  leveldb::Options options;
+  options.create_if_missing = true;
+  leveldb::Status db_status = leveldb::DB::Open(options, db_path_.value(),
+                                                &level_db);
+  DCHECK(level_db);
+  DCHECK(db_status.ok());
+  level_db_.reset(level_db);
+}
+
+void ResourceMetadataDB::Read(
+  GDataDirectoryService::SerializedMap* serialized_resources) {
+  DCHECK(blocking_task_runner_->RunsTasksOnCurrentThread());
+  DCHECK(serialized_resources);
+  DVLOG(1) << "Read " << db_path_.value();
+
+  scoped_ptr<leveldb::Iterator> iter(level_db_->NewIterator(
+        leveldb::ReadOptions()));
+  for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
+    DVLOG(1) << "Read, resource " << iter->key().ToString();
+    serialized_resources->insert(std::make_pair(iter->key().ToString(),
+                                                iter->value().ToString()));
+  }
+}
+
+void ResourceMetadataDB::Save(
+    const GDataDirectoryService::SerializedMap& serialized_resources) {
+  DCHECK(blocking_task_runner_->RunsTasksOnCurrentThread());
+
+  Clear();
+  for (GDataDirectoryService::SerializedMap::const_iterator iter =
+      serialized_resources.begin();
+      iter != serialized_resources.end(); ++iter) {
+    DVLOG(1) << "Saving resource " << iter->first << " to db";
+    leveldb::Status status = level_db_->Put(leveldb::WriteOptions(),
+                                            leveldb::Slice(iter->first),
+                                            leveldb::Slice(iter->second));
+    if (!status.ok()) {
+      LOG(ERROR) << "leveldb Put failed of " << iter->first
+                 << ", with " << status.ToString();
+      NOTREACHED();
+    }
+  }
+}
+
+void ResourceMetadataDB::Clear() {
+  level_db_.reset();
+  leveldb::DestroyDB(db_path_.value(), leveldb::Options());
+  Init();
+}
+
 // GDataDirectoryService class implementation.
 
 GDataDirectoryService::GDataDirectoryService()
-    : serialized_size_(0),
+    : blocking_task_runner_(NULL),
+      serialized_size_(0),
       largest_changestamp_(0),
-      origin_(UNINITIALIZED) {
+      origin_(UNINITIALIZED),
+      weak_ptr_factory_(ALLOW_THIS_IN_INITIALIZER_LIST(this)) {
   root_.reset(new GDataDirectory(NULL, this));
   root_->set_title(kGDataRootDirectory);
   root_->SetBaseNameFromTitle();
   root_->set_resource_id(kGDataRootDirectoryResourceId);
   AddEntryToResourceMap(root_.get());
 }
 
 GDataDirectoryService::~GDataDirectoryService() {
+  ClearRoot();
+
+  // Ensure db is closed on the blocking pool.
+  if (blocking_task_runner_ && directory_service_db_.get())
+    blocking_task_runner_->DeleteSoon(FROM_HERE,
+                                      directory_service_db_.release());
+}
+
+void GDataDirectoryService::ClearRoot() {
   // Note that children have a reference to root_,
   // so we need to delete them here.
   root_->RemoveChildren();
   RemoveEntryFromResourceMap(root_.get());
   DCHECK(resource_map_.empty());
   resource_map_.clear();
+  root_.reset();
 }
 
 void GDataDirectoryService::AddEntryToDirectory(
     const FilePath& directory_path,
     GDataEntry* entry,
     const FileOperationCallback& callback) {
   GDataEntry* destination = FindEntryByPathSync(directory_path);
   GDataFileError error = GDATA_FILE_ERROR_FAILED;
   if (!destination) {
     error = GDATA_FILE_ERROR_NOT_FOUND;
@@ skipping 98 matching lines @@
   GDataDirectory* entry_parent = old_entry ? old_entry->parent() : NULL;
   if (entry_parent) {
     DCHECK_EQ(fresh_file->resource_id(), old_entry->resource_id());
     DCHECK(old_entry->AsGDataFile());
 
     entry_parent->RemoveEntry(old_entry);
     entry_parent->AddEntry(fresh_file.release());
   }
 }
 
+void GDataDirectoryService::InitFromDB(
+    const FilePath& db_path,
+    base::SequencedTaskRunner* blocking_task_runner,
+    LoadRootFeedParams* load_params,
+    const base::Closure& callback) {
+  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
+  DCHECK(!db_path.empty());
+  DCHECK(blocking_task_runner);
+
+  if (directory_service_db_.get()) {
+    NOTREACHED();
+    if (!callback.is_null())
+      callback.Run();
+  }
+
+  blocking_task_runner_ = blocking_task_runner;
+
+  DVLOG(1) << "InitFromDB " << db_path.value();
+
+  CreateDBParams* create_params =
+      new CreateDBParams(db_path, blocking_task_runner);
+  blocking_task_runner_->PostTaskAndReply(
+      FROM_HERE,
+      base::Bind(&CreateResourceMetadataDBOnBlockingPool,
+                 create_params),
+      base::Bind(&GDataDirectoryService::InitResourceMap,
+                 weak_ptr_factory_.GetWeakPtr(),
+                 base::Owned(create_params),
+                 load_params,
+                 callback));
+}
+
+void GDataDirectoryService::InitResourceMap(
+    CreateDBParams* create_params,
+    LoadRootFeedParams* load_params,
+    const base::Closure& callback) {
+  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
+  DCHECK(create_params);
+  // Ensure this callback is run before we return.
+  base::ScopedClosureRunner callback_runner(callback);
+
+  SerializedMap* serialized_resources = &create_params->serialized_resources;
+  directory_service_db_ = create_params->db.Pass();
+  if (serialized_resources->empty()) {
+    origin_ = INITIALIZING;
+    if (load_params)
+      load_params->load_error = GDATA_FILE_ERROR_NOT_FOUND;
+    return;
+  }
+
+  ClearRoot();
+
+  // Version check.
+  int32 version = 0;
+  SerializedMap::iterator iter = serialized_resources->find(kDBKeyVersion);
+  if (iter == serialized_resources->end() ||
+      !base::StringToInt(iter->second, &version) ||
+      version != kProtoVersion) {
+    if (load_params)

[satorux1, 2012/08/03 20:19:16]

can it be NULL? Can we add DCHECK(load_params) at the beginning of the function
instead?

[achuithb, 2012/08/03 21:41:04]

It can be NULL for tests. 

I'm going to change this, so it's not supposed to be NULL, and add a DCHECK.
Also, we don't really need all the LoadRootParams, just the error, so I'm only
going to pass this in.

+      load_params->load_error = GDATA_FILE_ERROR_NOT_FOUND;

[satorux1, 2012/08/03 20:19:16]

maybe GDATA_FILE_ERROR_FAILED?

[achuithb, 2012/08/03 21:41:04]

Done.

+    return;
+  }
+  serialized_resources->erase(iter);
+
+  // Get the largest changestamp.
+  iter = serialized_resources->find(kDBKeyLargestChangestamp);
+  if (iter != serialized_resources->end()) {
+    base::StringToInt(iter->second, &largest_changestamp_);
+    DVLOG(1) << "InitResourceMap largest_changestamp_"
+             << largest_changestamp_;
+    serialized_resources->erase(iter);
+  } else {
+    NOTREACHED() << "Could not find largest_changestamp";

[satorux1, 2012/08/03 20:19:16]

Add this?

load_params->load_error = GDATA_FILE_ERROR_FAILED;
return;

[achuithb, 2012/08/03 21:41:04]

Done.

+  }
+
+  ResourceMap resource_map;
+  for (SerializedMap::const_iterator iter = serialized_resources->begin();
+      iter != serialized_resources->end(); ++iter) {
+    if (iter->first.find(kDBKeyResourceIdPrefix) != 0) {
+      NOTREACHED() << "Incorrect prefix for db key " << iter->first;
+      continue;
+    }
+
+    const std::string resource_id =
+        iter->first.substr(strlen(kDBKeyResourceIdPrefix));
+    scoped_ptr<GDataEntry> entry = FromProtoString(iter->second);
+    if (entry.get()) {
+      DVLOG(1) << "Inserting resource " << resource_id
+               << " into resource_map";
+      resource_map.insert(std::make_pair(resource_id, entry.release()));
+    } else {
+      NOTREACHED() << "Failed to parse GDataEntry for resource " << resource_id;
+    }
+  }
+
+  // Fix up parent-child relations.
+  for (ResourceMap::iterator iter = resource_map.begin();
+      iter != resource_map.end(); ++iter) {
+    GDataEntry* entry = iter->second;
+    ResourceMap::iterator parent_it =
+        resource_map.find(entry->parent_resource_id());
+    if (parent_it != resource_map.end()) {
+      GDataDirectory* parent = parent_it->second->AsGDataDirectory();
+      if (parent) {
+        DVLOG(1) << "Adding " << entry->resource_id()
+                 << " as a child of " << parent->resource_id();
+        parent->AddEntry(entry);
+      } else {
+        NOTREACHED() << "Parent is not a directory " << parent->resource_id();
+      }
+    } else if (entry->resource_id() == kGDataRootDirectoryResourceId) {
+      root_.reset(entry->AsGDataDirectory());
+      DCHECK(root_.get());
+      AddEntryToResourceMap(root_.get());
+    } else {
+      NOTREACHED() << "Missing parent id " << entry->parent_resource_id()
+                   << " for resource " << entry->resource_id();
+    }
+  }
+
+  DCHECK(root_.get());
+  DCHECK_EQ(resource_map.size(), resource_map_.size());
+  DCHECK_EQ(resource_map.size(), serialized_resources->size());
+
+  origin_ = FROM_CACHE;
+}
+
+void GDataDirectoryService::SaveToDB() {
+  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
+
+  if (!blocking_task_runner_ || !directory_service_db_.get()) {
+    NOTREACHED();
+    return;
+  }
+
+  size_t serialized_size = 0;
+  SerializedMap serialized_resources;
+  for (ResourceMap::const_iterator iter = resource_map_.begin();
+      iter != resource_map_.end(); ++iter) {
+    GDataEntryProto proto;
+    iter->second->ToProtoFull(&proto);
+    std::string serialized_string;
+    const bool ok = proto.SerializeToString(&serialized_string);
+    DCHECK(ok);
+    if (ok) {
+      serialized_resources.insert(
+          std::make_pair(std::string(kDBKeyResourceIdPrefix) + iter->first,
+                         serialized_string));
+      serialized_size += serialized_string.size();
+    }
+  }
+
+  serialized_resources.insert(std::make_pair(kDBKeyVersion,
+      base::IntToString(kProtoVersion)));
+  serialized_resources.insert(std::make_pair(kDBKeyLargestChangestamp,
+      base::IntToString(largest_changestamp_)));
+  set_last_serialized(base::Time::Now());
+  set_serialized_size(serialized_size);
+
+  blocking_task_runner_->PostTask(
+      FROM_HERE,
+      base::Bind(&ResourceMetadataDB::Save,
+                 base::Unretained(directory_service_db_.get()),
+                 serialized_resources));
+}
+
 // Convert to/from proto.
 
 // static
 void GDataEntry::ConvertProtoToPlatformFileInfo(
     const PlatformFileInfoProto& proto,
     base::PlatformFileInfo* file_info) {
   file_info->size = proto.size();
   file_info->is_directory = proto.is_directory();
   file_info->is_symbolic_link = proto.is_symbolic_link();
   file_info->last_modified = base::Time::FromInternalValue(
@@ skipping 214 matching lines @@
 
   if (!root_->FromProto(proto.gdata_directory()))
     return false;
 
   origin_ = FROM_CACHE;
   largest_changestamp_ = proto.largest_changestamp();
 
   return true;
 }
 
+scoped_ptr<GDataEntry> GDataDirectoryService::FromProtoString(
+    const std::string& serialized_proto) {
+  GDataEntryProto entry_proto;
+  if (!entry_proto.ParseFromString(serialized_proto))
+    return scoped_ptr<GDataEntry>();
+
+  scoped_ptr<GDataEntry> entry;
+  if (entry_proto.file_info().is_directory()) {
+    entry.reset(new GDataDirectory(NULL, this));
+    // Call GDataEntry::FromProto instead of GDataDirectory::FromProto because
+    // the proto does not include children.
+    if (!entry->FromProto(entry_proto)) {
+      NOTREACHED() << "FromProto (directory) failed";
+      entry.reset();
+    }
+  } else {
+    scoped_ptr<GDataFile> file(new GDataFile(NULL, this));
+    // Call GDataFile::FromProto.
+    if (file->FromProto(entry_proto)) {
+      entry.reset(file.release());
+    } else {
+      NOTREACHED() << "FromProto (file) failed";
+    }
+  }
+  return entry.Pass();
+}
+
 }  // namespace gdata