LeveldbValueStore: Deleting db when open fails due to corruption.

extensions/browser/value_store/leveldb_value_store.cc

 // Copyright 2014 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "extensions/browser/value_store/leveldb_value_store.h"
 
 #include "base/files/file_util.h"
 #include "base/json/json_reader.h"
 #include "base/json/json_writer.h"
 #include "base/logging.h"
@@ skipping 11 matching lines @@
 #include "third_party/leveldatabase/src/include/leveldb/write_batch.h"
 
 namespace util = value_store_util;
 using content::BrowserThread;
 
 namespace {
 
 const char kInvalidJson[] = "Invalid JSON";
 const char kCannotSerialize[] = "Cannot serialize value to JSON";
 
+// UMA values used when recovering from a corrupted leveldb.
+// Do not change/delete these values as you will break reporting for older
+// copies of Chrome. Only add new values to the end.
+enum LevelDBCorruptionRecoveryValue {
+  LEVELDB_RESTORE_DELETE_SUCCESS = 0,
+  LEVELDB_RESTORE_DELETE_FAILURE,
+  LEVELDB_RESTORE_REPAIR_SUCCESS,
+  LEVELDB_RESTORE_MAX
+};
+
 // Scoped leveldb snapshot which releases the snapshot on destruction.
 class ScopedSnapshot {
  public:
   explicit ScopedSnapshot(leveldb::DB* db)
       : db_(db), snapshot_(db->GetSnapshot()) {}
 
   ~ScopedSnapshot() {
     db_->ReleaseSnapshot(snapshot_);
   }
 
   const leveldb::Snapshot* get() {
     return snapshot_;
   }
 
  private:
   leveldb::DB* db_;
   const leveldb::Snapshot* snapshot_;
 
   DISALLOW_COPY_AND_ASSIGN(ScopedSnapshot);
 };
 
 }  // namespace
 
 LeveldbValueStore::LeveldbValueStore(const std::string& uma_client_name,
                                      const base::FilePath& db_path)
-    : db_path_(db_path), open_histogram_(nullptr) {
+    : db_path_(db_path), open_histogram_(nullptr), restore_histogram_(nullptr) {
   DCHECK_CURRENTLY_ON(BrowserThread::FILE);
 
   // Used in lieu of UMA_HISTOGRAM_ENUMERATION because the histogram name is
   // not a constant.
   open_histogram_ = base::LinearHistogram::FactoryGet(
       "Extensions.Database.Open." + uma_client_name, 1,
       leveldb_env::LEVELDB_STATUS_MAX, leveldb_env::LEVELDB_STATUS_MAX + 1,
       base::Histogram::kUmaTargetedHistogramFlag);
+  restore_histogram_ = base::LinearHistogram::FactoryGet(

[Devlin, 2015/11/06 19:54:46]

This is histogram black magic to me, so I'll trust the histogram owner to verify
it. :)

[cmumford, 2015/11/09 23:04:21]

I manually hack-ed in a call to
"restore_histogram_->Add(LEVELDB_RESTORE_REPAIR_SUCCESS);" and verified it was
in about:histograms.

+      "Extensions.Database.Restore." + uma_client_name, 1, LEVELDB_RESTORE_MAX,
+      LEVELDB_RESTORE_MAX + 1, base::Histogram::kUmaTargetedHistogramFlag);
   base::trace_event::MemoryDumpManager::GetInstance()->RegisterDumpProvider(
       this, "LeveldbValueStore", base::ThreadTaskRunnerHandle::Get());
 }
 
 LeveldbValueStore::~LeveldbValueStore() {
   DCHECK_CURRENTLY_ON(BrowserThread::FILE);
   base::trace_event::MemoryDumpManager::GetInstance()->UnregisterDumpProvider(
       this);
 
   // Delete the database from disk if it's empty (but only if we managed to
@@ skipping 204 matching lines @@
         ValueStoreChange(next_key, next_value.release(), NULL));
   }
 
   DeleteDbFile();
   return MakeWriteResult(changes.Pass());
 }
 
 bool LeveldbValueStore::Restore() {
   DCHECK_CURRENTLY_ON(BrowserThread::FILE);
 
-  ReadResult result = Get();
-  std::string previous_key;
-  while (result->IsCorrupted()) {
-    // If we don't have a specific corrupted key, or we've tried and failed to
-    // clear this specific key, or we fail to restore the key, then wipe the
-    // whole database.
-    if (!result->error().key.get() || *result->error().key == previous_key ||
-        !RestoreKey(*result->error().key)) {
-      DeleteDbFile();
-      result = Get();
-      break;
-    }
+  // Possible to have a corrupted open database, so first close it.
+  db_.reset();
 
-    // Otherwise, re-Get() the database to check if there is still any
-    // corruption.
-    previous_key = *result->error().key;
-    result = Get();
+  leveldb::Options options;
+  options.create_if_missing = true;
+
+  // Repair can drop an unbounded number of leveldb tables (key/value sets)
+  leveldb::Status status = leveldb::RepairDB(db_path_.AsUTF8Unsafe(), options);
+  if (status.ok()) {
+    restore_histogram_->Add(LEVELDB_RESTORE_REPAIR_SUCCESS);
+    return true;
   }
 
-  // If we still have an error, it means we've tried deleting the database file,
-  // and failed. There's nothing more we can do.
-  return !result->IsCorrupted();
+  if (DeleteDbFile()) {
+    restore_histogram_->Add(LEVELDB_RESTORE_DELETE_SUCCESS);
+    return true;
+  }
+
+  restore_histogram_->Add(LEVELDB_RESTORE_DELETE_FAILURE);
+  return false;
 }
 
 bool LeveldbValueStore::RestoreKey(const std::string& key) {
   DCHECK_CURRENTLY_ON(BrowserThread::FILE);
 
   ReadResult result = Get(key);
   if (result->IsCorrupted()) {
     leveldb::WriteBatch batch;
     batch.Delete(key);
     scoped_ptr<ValueStore::Error> error = WriteToDb(&batch);
@@ skipping 52 matching lines @@
   leveldb::Options options;
   options.max_open_files = 0;  // Use minimum.
   options.create_if_missing = true;
   options.reuse_logs = leveldb_env::kDefaultLogReuseOptionValue;
 
   leveldb::DB* db = NULL;
   leveldb::Status status =
       leveldb::DB::Open(options, db_path_.AsUTF8Unsafe(), &db);
   if (open_histogram_)
     open_histogram_->Add(leveldb_env::GetLevelDBStatusUMAValue(status));
+  if (status.IsCorruption()) {
+    // Returning a corruption error should result in Restore() being called.
+    // However, since once a leveldb becomes corrupt it's unusable without
+    // some kind of repair or delete, so do that right now.
+    Restore();
+  }
   if (!status.ok())
     return ToValueStoreError(status, util::NoKey());
 
   CHECK(db);
   db_.reset(db);
   return util::NoError();
 }
 
 scoped_ptr<ValueStore::Error> LeveldbValueStore::ReadFromDb(
     leveldb::ReadOptions options,
@@ skipping 67 matching lines @@
 
   it->SeekToFirst();
   bool is_empty = !it->Valid();
   if (!it->status().ok()) {
     LOG(ERROR) << "Checking DB emptiness failed: " << it->status().ToString();
     return false;
   }
   return is_empty;
 }
 
-void LeveldbValueStore::DeleteDbFile() {
+bool LeveldbValueStore::DeleteDbFile() {
   db_.reset();  // release any lock on the directory
   if (!base::DeleteFile(db_path_, true /* recursive */)) {
     LOG(WARNING) << "Failed to delete LeveldbValueStore database at " <<
         db_path_.value();
+    return false;
   }
+  return true;
 }
 
 scoped_ptr<ValueStore::Error> LeveldbValueStore::ToValueStoreError(
     const leveldb::Status& status,
     scoped_ptr<std::string> key) {
   CHECK(!status.ok());
   CHECK(!status.IsNotFound());  // not an error
 
   std::string message = status.ToString();
   // The message may contain |db_path_|, which may be considered sensitive
   // data, and those strings are passed to the extension, so strip it out.
   base::ReplaceSubstringsAfterOffset(
       &message, 0u, db_path_.AsUTF8Unsafe(), "...");
 
   return Error::Create(CORRUPTION, message, key.Pass());
 }