Handling LevelDB errors encountered after open.

content/browser/indexed_db/indexed_db_backing_store.cc

Index: content/browser/indexed_db/indexed_db_backing_store.cc
diff --git a/content/browser/indexed_db/indexed_db_backing_store.cc b/content/browser/indexed_db/indexed_db_backing_store.cc
index 2aebb3ea0ae2ab8b4cc7bcfbe92bfc0e67d0939e..19ea5e4a28e9754f0e5454233483ebc167c7dfef 100644
--- a/content/browser/indexed_db/indexed_db_backing_store.cc
+++ b/content/browser/indexed_db/indexed_db_backing_store.cc
@@ -5,11 +5,15 @@
 #include "content/browser/indexed_db/indexed_db_backing_store.h"
 
 #include "base/file_util.h"
+#include "base/json/json_reader.h"
+#include "base/json/json_writer.h"
 #include "base/logging.h"
 #include "base/metrics/histogram.h"
+#include "base/platform_file.h"
 #include "base/strings/string_piece.h"
 #include "base/strings/string_util.h"
 #include "base/strings/utf_string_conversions.h"
+#include "content/browser/indexed_db/indexed_db_database_error.h"
 #include "content/browser/indexed_db/indexed_db_leveldb_coding.h"
 #include "content/browser/indexed_db/indexed_db_metadata.h"
 #include "content/browser/indexed_db/indexed_db_tracing.h"
@@ -42,6 +46,10 @@ static base::FilePath ComputeFileName(const GURL& origin_url) {
       .AddExtension(FILE_PATH_LITERAL(".indexeddb.leveldb"));
 }
 
+static base::FilePath ComputeCorruptionFileName(const GURL& origin_url) {
+  return ComputeFileName(origin_url).Append("corruption_info.json");

[jsbell, 2014/03/26 18:16:58]

Doesn't this put the file in the directory, which will end up being deleted? 

[cmumford, 2014/03/26 21:40:36]

No, we call DestroyDB when the db is determined to be corrupt. This LevelDB
function only deletes the LevelDB files (*.ldb, etc.). It will call
env::DeleteDir, but that will fail if other files (like this one) are present.
Had that been the case then the corrupted_open_db_recovery.html test would fail.

+}
+
 }  // namespace
 
 static const int64 kKeyGeneratorInitialNumber =
@@ -85,13 +93,12 @@ static void RecordInternalError(const char* type,
       ->Add(location);
 }
 
-// Use to signal conditions that usually indicate developer error, but
-// could be caused by data corruption.  A macro is used instead of an
-// inline function so that the assert and log report the line number.
+// Use to signal conditions caused by data corruption.
+// A macro is used instead of an inline function so that the assert and log
+// report the line number.
 #define REPORT_ERROR(type, location)                      \
   do {                                                    \
     LOG(ERROR) << "IndexedDB " type " Error: " #location; \
-    NOTREACHED();                                         \
     RecordInternalError(type, location);                  \
   } while (0)
 
@@ -100,6 +107,23 @@ static void RecordInternalError(const char* type,
   REPORT_ERROR("Consistency", location)
 #define INTERNAL_WRITE_ERROR(location) REPORT_ERROR("Write", location)
 
+// Use to signal conditions that usually indicate developer error, but
+// could be caused by data corruption.  A macro is used instead of an

[jsbell, 2014/03/26 18:16:58]

Maybe add to the comment that anything that could be caused by data corruption
should really be handled, i.e. we should eliminate all uses of
REPORT_ERROR_UNTESTED over time.

[cmumford, 2014/03/26 21:40:36]

Done.

+// inline function so that the assert and log report the line number.
+#define REPORT_ERROR_UNTESTED(type, location)             \
+  do {                                                    \
+    LOG(ERROR) << "IndexedDB " type " Error: " #location; \
+    NOTREACHED();                                         \
+    RecordInternalError(type, location);                  \
+  } while (0)
+
+#define INTERNAL_READ_ERROR_UNTESTED(location) \
+  REPORT_ERROR_UNTESTED("Read", location)
+#define INTERNAL_CONSISTENCY_ERROR_UNTESTED(location) \
+  REPORT_ERROR_UNTESTED("Consistency", location)
+#define INTERNAL_WRITE_ERROR_UNTESTED(location) \
+  REPORT_ERROR_UNTESTED("Write", location)
+
 static void PutBool(LevelDBTransaction* transaction,
                     const StringPiece& key,
                     bool value) {
@@ -277,7 +301,7 @@ WARN_UNUSED_RESULT static bool SetUpMetadata(
   leveldb::Status s =
       GetInt(transaction.get(), schema_version_key, &db_schema_version, &found);
   if (!s.ok()) {
-    INTERNAL_READ_ERROR(SET_UP_METADATA);
+    INTERNAL_READ_ERROR_UNTESTED(SET_UP_METADATA);
     return false;
   }
   if (!found) {
@@ -304,11 +328,11 @@ WARN_UNUSED_RESULT static bool SetUpMetadata(
         found = false;
         s = GetInt(transaction.get(), it->Key(), &database_id, &found);
         if (!s.ok()) {
-          INTERNAL_READ_ERROR(SET_UP_METADATA);
+          INTERNAL_READ_ERROR_UNTESTED(SET_UP_METADATA);
           return false;
         }
         if (!found) {
-          INTERNAL_CONSISTENCY_ERROR(SET_UP_METADATA);
+          INTERNAL_CONSISTENCY_ERROR_UNTESTED(SET_UP_METADATA);
           return false;
         }
         std::string int_version_key = DatabaseMetaDataKey::Encode(
@@ -330,11 +354,11 @@ WARN_UNUSED_RESULT static bool SetUpMetadata(
   found = false;
   s = GetInt(transaction.get(), data_version_key, &db_data_version, &found);
   if (!s.ok()) {
-    INTERNAL_READ_ERROR(SET_UP_METADATA);
+    INTERNAL_READ_ERROR_UNTESTED(SET_UP_METADATA);
     return false;
   }
   if (!found) {
-    INTERNAL_CONSISTENCY_ERROR(SET_UP_METADATA);
+    INTERNAL_CONSISTENCY_ERROR_UNTESTED(SET_UP_METADATA);
     return false;
   }
   if (db_data_version < latest_known_data_version) {
@@ -347,7 +371,7 @@ WARN_UNUSED_RESULT static bool SetUpMetadata(
 
   s = transaction->Commit();
   if (!s.ok()) {
-    INTERNAL_WRITE_ERROR(SET_UP_METADATA);
+    INTERNAL_WRITE_ERROR_UNTESTED(SET_UP_METADATA);
     return false;
   }
   return true;
@@ -514,6 +538,91 @@ static bool IsPathTooLong(const base::FilePath& leveldb_dir) {
   return false;
 }
 
+// Assumes caller has already closed the backing store.

[jsbell, 2014/03/26 18:16:58]

Move this comment to the header file?

[cmumford, 2014/03/26 21:40:36]

Done.

+leveldb::Status IndexedDBBackingStore::DestroyBackingStore(
+    const base::FilePath& path_base,
+    const GURL& origin_url) {
+  const base::FilePath file_path =
+      path_base.Append(ComputeFileName(origin_url));
+  DefaultLevelDBFactory leveldb_factory;
+  return leveldb_factory.DestroyLevelDB(file_path);
+}
+
+bool IndexedDBBackingStore::ReadCorruptionInfo(const base::FilePath& path_base,
+                                               const GURL& origin_url,
+                                               std::string& message) {
+
+  const base::FilePath info_path =
+      path_base.Append(ComputeCorruptionFileName(origin_url));
+
+  if (IsPathTooLong(info_path))
+    return false;
+
+  const int64 max_json_len = 4096;
+  int64 file_size(0);
+  if (!GetFileSize(info_path, &file_size) || file_size > max_json_len)
+    return false;
+
+  bool created(false);
+  base::PlatformFileError error(base::PLATFORM_FILE_OK);
+  base::PlatformFile file = base::CreatePlatformFile(
+      info_path,
+      base::PLATFORM_FILE_OPEN | base::PLATFORM_FILE_READ,
+      &created,
+      &error);
+  bool success = false;
+  if (file) {
+    std::vector<char> bytes(file_size);
+    if (file_size == base::ReadPlatformFile(file, 0, &bytes[0], file_size)) {

[jsbell, 2014/03/26 18:16:58]

&bytes[0] is undefined behavior if the vector length is 0. You should probably
guard against 0-length files anyway (i.e. no need to open them)

[cmumford, 2014/03/26 21:40:36]

Done.

+      std::string input_js(&bytes[0], file_size);
+      base::JSONReader reader;
+      scoped_ptr<base::Value> val(reader.ReadToValue(input_js));
+      if (val) {

[jsbell, 2014/03/26 18:16:58]

Add: && val->GetType() == base::Value::TYPE_DICTIONARY

[cmumford, 2014/03/26 21:40:36]

Done.

+        base::DictionaryValue* dict_val =
+            static_cast<base::DictionaryValue*>(val.get());
+        success = dict_val->GetString("message", &message);
+      }
+    }
+    base::ClosePlatformFile(file);
+  }
+
+  base::DeleteFile(info_path, false);
+
+  return success;
+}
+
+bool IndexedDBBackingStore::RecordCorruptionInfo(
+    const base::FilePath& path_base,
+    const GURL& origin_url,
+    const IndexedDBDatabaseError& dberror) {

[jsbell, 2014/03/26 18:16:58]

Since this only serializes the message, and since ReadCorruptionInfo only
returns the message, what do you think about having this method only take a
string rather than an error object?

[cmumford, 2014/03/26 21:40:36]

OK. Either way those two functions should be symmetrical, and they are not.

[jsbell, 2014/03/26 22:49:00]

Yep - having it store all the error fields would also be fine.

+  const base::FilePath info_path =
+      path_base.Append(ComputeCorruptionFileName(origin_url));
+  if (IsPathTooLong(info_path))
+    return false;
+
+  base::DictionaryValue root_dict;
+  root_dict.SetString("message", dberror.message());
+  std::string output_js;
+  base::JSONWriter::Write(&root_dict, &output_js);
+
+  bool created(false);
+  base::PlatformFileError error(base::PLATFORM_FILE_OK);
+  base::PlatformFile file = base::CreatePlatformFile(
+      info_path,
+      base::PLATFORM_FILE_CREATE_ALWAYS | base::PLATFORM_FILE_WRITE,
+      &created,
+      &error);
+  bool success = false;
+  if (file) {

[jsbell, 2014/03/26 18:16:58]

Early exit instead, which would let you eliminate 'success' ?

[cmumford, 2014/03/26 21:40:36]

Done.

+    int written =
+        base::WritePlatformFile(file, 0, output_js.c_str(), output_js.length());
+    base::ClosePlatformFile(file);
+    success = (size_t(written) == output_js.length());
+  }
+
+  return success;
+}
+
 // static
 scoped_refptr<IndexedDBBackingStore> IndexedDBBackingStore::Open(
     const GURL& origin_url,
@@ -567,24 +676,37 @@ scoped_refptr<IndexedDBBackingStore> IndexedDBBackingStore::Open(
 
   bool is_schema_known = false;
   if (db) {
-    bool ok = IsSchemaKnown(db.get(), &is_schema_known);
-    if (!ok) {
-      LOG(ERROR) << "IndexedDB had IO error checking schema, treating it as "
-                    "failure to open";
+    std::string corruption_message;
+    if (ReadCorruptionInfo(path_base, origin_url, corruption_message)) {
+      LOG(ERROR) << "IndexedDB recovering from a corrupted (and deleted) "
+                    "database.";
       HistogramOpenStatus(
           INDEXED_DB_BACKING_STORE_OPEN_FAILED_IO_ERROR_CHECKING_SCHEMA,

[jsbell, 2014/03/26 18:16:58]

Add a new histogram value?

[cmumford, 2014/03/26 21:40:36]

Done.

           origin_url);
       db.reset();
       *data_loss = blink::WebIDBDataLossTotal;
-      *data_loss_message = "I/O error checking schema";
-    } else if (!is_schema_known) {
-      LOG(ERROR) << "IndexedDB backing store had unknown schema, treating it "
-                    "as failure to open";
-      HistogramOpenStatus(INDEXED_DB_BACKING_STORE_OPEN_FAILED_UNKNOWN_SCHEMA,
-                          origin_url);
-      db.reset();
-      *data_loss = blink::WebIDBDataLossTotal;
-      *data_loss_message = "Unknown schema";
+      *data_loss_message =
+          "IndexedDB (database was corrupt): " + corruption_message;
+    } else {
+      bool ok = IsSchemaKnown(db.get(), &is_schema_known);

[jsbell, 2014/03/26 18:16:58]

Merge this into the else i.e. "else if (IsSchemaKnown(...))" - save a level of
nesting (and minimize the diff)

+      if (!ok) {
+        LOG(ERROR) << "IndexedDB had IO error checking schema, treating it as "
+                      "failure to open";
+        HistogramOpenStatus(
+            INDEXED_DB_BACKING_STORE_OPEN_FAILED_IO_ERROR_CHECKING_SCHEMA,
+            origin_url);
+        db.reset();
+        *data_loss = blink::WebIDBDataLossTotal;
+        *data_loss_message = "I/O error checking schema";
+      } else if (!is_schema_known) {
+        LOG(ERROR) << "IndexedDB backing store had unknown schema, treating it "
+                      "as failure to open";
+        HistogramOpenStatus(INDEXED_DB_BACKING_STORE_OPEN_FAILED_UNKNOWN_SCHEMA,
+                            origin_url);
+        db.reset();
+        *data_loss = blink::WebIDBDataLossTotal;
+        *data_loss_message = "Unknown schema";
+      }
     }
   }
 
@@ -690,7 +812,7 @@ std::vector<base::string16> IndexedDBBackingStore::GetDatabaseNames() {
     StringPiece slice(it->Key());
     DatabaseNameKey database_name_key;
     if (!DatabaseNameKey::Decode(&slice, &database_name_key)) {
-      INTERNAL_CONSISTENCY_ERROR(GET_DATABASE_NAMES);
+      INTERNAL_CONSISTENCY_ERROR_UNTESTED(GET_DATABASE_NAMES);
       continue;
     }
     found_names.push_back(database_name_key.database_name());
@@ -707,7 +829,7 @@ leveldb::Status IndexedDBBackingStore::GetIDBDatabaseMetaData(
 
   leveldb::Status s = GetInt(db_.get(), key, &metadata->id, found);
   if (!s.ok()) {
-    INTERNAL_READ_ERROR(GET_IDBDATABASE_METADATA);
+    INTERNAL_READ_ERROR_UNTESTED(GET_IDBDATABASE_METADATA);
     return s;
   }
   if (!*found)
@@ -719,11 +841,11 @@ leveldb::Status IndexedDBBackingStore::GetIDBDatabaseMetaData(
                 &metadata->version,
                 found);
   if (!s.ok()) {
-    INTERNAL_READ_ERROR(GET_IDBDATABASE_METADATA);
+    INTERNAL_READ_ERROR_UNTESTED(GET_IDBDATABASE_METADATA);
     return s;
   }
   if (!*found) {
-    INTERNAL_CONSISTENCY_ERROR(GET_IDBDATABASE_METADATA);
+    INTERNAL_CONSISTENCY_ERROR_UNTESTED(GET_IDBDATABASE_METADATA);
     return InternalInconsistencyStatus();
   }
 
@@ -733,11 +855,11 @@ leveldb::Status IndexedDBBackingStore::GetIDBDatabaseMetaData(
                 &metadata->int_version,
                 found);
   if (!s.ok()) {
-    INTERNAL_READ_ERROR(GET_IDBDATABASE_METADATA);
+    INTERNAL_READ_ERROR_UNTESTED(GET_IDBDATABASE_METADATA);
     return s;
   }
   if (!*found) {
-    INTERNAL_CONSISTENCY_ERROR(GET_IDBDATABASE_METADATA);
+    INTERNAL_CONSISTENCY_ERROR_UNTESTED(GET_IDBDATABASE_METADATA);
     return InternalInconsistencyStatus();
   }
 
@@ -747,7 +869,7 @@ leveldb::Status IndexedDBBackingStore::GetIDBDatabaseMetaData(
   s = GetMaxObjectStoreId(
       db_.get(), metadata->id, &metadata->max_object_store_id);
   if (!s.ok()) {
-    INTERNAL_READ_ERROR(GET_IDBDATABASE_METADATA);
+    INTERNAL_READ_ERROR_UNTESTED(GET_IDBDATABASE_METADATA);
   }
 
   return s;
@@ -762,7 +884,7 @@ WARN_UNUSED_RESULT static leveldb::Status GetNewDatabaseId(
   leveldb::Status s =
       GetInt(transaction, MaxDatabaseIdKey::Encode(), &max_database_id, &found);
   if (!s.ok()) {
-    INTERNAL_READ_ERROR(GET_NEW_DATABASE_ID);
+    INTERNAL_READ_ERROR_UNTESTED(GET_NEW_DATABASE_ID);
     return s;
   }
   if (!found)
@@ -805,7 +927,7 @@ leveldb::Status IndexedDBBackingStore::CreateIDBDatabaseMetaData(
             int_version);
   s = transaction->Commit();
   if (!s.ok())
-    INTERNAL_WRITE_ERROR(CREATE_IDBDATABASE_METADATA);
+    INTERNAL_WRITE_ERROR_UNTESTED(CREATE_IDBDATABASE_METADATA);
   return s;
 }
 
@@ -861,7 +983,7 @@ leveldb::Status IndexedDBBackingStore::DeleteDatabase(
 
   s = transaction->Commit();
   if (!s.ok()) {
-    INTERNAL_WRITE_ERROR(DELETE_DATABASE);
+    INTERNAL_WRITE_ERROR_UNTESTED(DELETE_DATABASE);
     return s;
   }
   db_->Compact(start_key, stop_key);
@@ -909,7 +1031,7 @@ leveldb::Status IndexedDBBackingStore::GetObjectStores(
     bool ok = ObjectStoreMetaDataKey::Decode(&slice, &meta_data_key);
     DCHECK(ok);
     if (meta_data_key.MetaDataType() != ObjectStoreMetaDataKey::NAME) {
-      INTERNAL_CONSISTENCY_ERROR(GET_OBJECT_STORES);
+      INTERNAL_CONSISTENCY_ERROR_UNTESTED(GET_OBJECT_STORES);
       // Possible stale metadata, but don't fail the load.
       it->Next();
       continue;
@@ -923,7 +1045,7 @@ leveldb::Status IndexedDBBackingStore::GetObjectStores(
     {
       StringPiece slice(it->Value());
       if (!DecodeString(&slice, &object_store_name) || !slice.empty())
-        INTERNAL_CONSISTENCY_ERROR(GET_OBJECT_STORES);
+        INTERNAL_CONSISTENCY_ERROR_UNTESTED(GET_OBJECT_STORES);
     }
 
     it->Next();
@@ -931,14 +1053,14 @@ leveldb::Status IndexedDBBackingStore::GetObjectStores(
                                          stop_key,
                                          object_store_id,
                                          ObjectStoreMetaDataKey::KEY_PATH)) {
-      INTERNAL_CONSISTENCY_ERROR(GET_OBJECT_STORES);
+      INTERNAL_CONSISTENCY_ERROR_UNTESTED(GET_OBJECT_STORES);
       break;
     }
     IndexedDBKeyPath key_path;
     {
       StringPiece slice(it->Value());
       if (!DecodeIDBKeyPath(&slice, &key_path) || !slice.empty())
-        INTERNAL_CONSISTENCY_ERROR(GET_OBJECT_STORES);
+        INTERNAL_CONSISTENCY_ERROR_UNTESTED(GET_OBJECT_STORES);
     }
 
     it->Next();
@@ -947,14 +1069,14 @@ leveldb::Status IndexedDBBackingStore::GetObjectStores(
              stop_key,
              object_store_id,
              ObjectStoreMetaDataKey::AUTO_INCREMENT)) {
-      INTERNAL_CONSISTENCY_ERROR(GET_OBJECT_STORES);
+      INTERNAL_CONSISTENCY_ERROR_UNTESTED(GET_OBJECT_STORES);
       break;
     }
     bool auto_increment;
     {
       StringPiece slice(it->Value());
       if (!DecodeBool(&slice, &auto_increment) || !slice.empty())
-        INTERNAL_CONSISTENCY_ERROR(GET_OBJECT_STORES);
+        INTERNAL_CONSISTENCY_ERROR_UNTESTED(GET_OBJECT_STORES);
     }
 
     it->Next();  // Is evicatble.
@@ -962,7 +1084,7 @@ leveldb::Status IndexedDBBackingStore::GetObjectStores(
                                          stop_key,
                                          object_store_id,
                                          ObjectStoreMetaDataKey::EVICTABLE)) {
-      INTERNAL_CONSISTENCY_ERROR(GET_OBJECT_STORES);
+      INTERNAL_CONSISTENCY_ERROR_UNTESTED(GET_OBJECT_STORES);
       break;
     }
 
@@ -972,7 +1094,7 @@ leveldb::Status IndexedDBBackingStore::GetObjectStores(
              stop_key,
              object_store_id,
              ObjectStoreMetaDataKey::LAST_VERSION)) {
-      INTERNAL_CONSISTENCY_ERROR(GET_OBJECT_STORES);
+      INTERNAL_CONSISTENCY_ERROR_UNTESTED(GET_OBJECT_STORES);
       break;
     }
 
@@ -982,14 +1104,14 @@ leveldb::Status IndexedDBBackingStore::GetObjectStores(
              stop_key,
              object_store_id,
              ObjectStoreMetaDataKey::MAX_INDEX_ID)) {
-      INTERNAL_CONSISTENCY_ERROR(GET_OBJECT_STORES);
+      INTERNAL_CONSISTENCY_ERROR_UNTESTED(GET_OBJECT_STORES);
       break;
     }
     int64 max_index_id;
     {
       StringPiece slice(it->Value());
       if (!DecodeInt(&slice, &max_index_id) || !slice.empty())
-        INTERNAL_CONSISTENCY_ERROR(GET_OBJECT_STORES);
+        INTERNAL_CONSISTENCY_ERROR_UNTESTED(GET_OBJECT_STORES);
     }
 
     it->Next();  // [optional] has key path (is not null)
@@ -1001,7 +1123,7 @@ leveldb::Status IndexedDBBackingStore::GetObjectStores(
       {
         StringPiece slice(it->Value());
         if (!DecodeBool(&slice, &has_key_path))
-          INTERNAL_CONSISTENCY_ERROR(GET_OBJECT_STORES);
+          INTERNAL_CONSISTENCY_ERROR_UNTESTED(GET_OBJECT_STORES);
       }
       // This check accounts for two layers of legacy coding:
       // (1) Initially, has_key_path was added to distinguish null vs. string.
@@ -1010,7 +1132,7 @@ leveldb::Status IndexedDBBackingStore::GetObjectStores(
       if (!has_key_path &&
           (key_path.type() == blink::WebIDBKeyPathTypeString &&
            !key_path.string().empty())) {
-        INTERNAL_CONSISTENCY_ERROR(GET_OBJECT_STORES);
+        INTERNAL_CONSISTENCY_ERROR_UNTESTED(GET_OBJECT_STORES);
         break;
       }
       if (!has_key_path)
@@ -1026,7 +1148,7 @@ leveldb::Status IndexedDBBackingStore::GetObjectStores(
             ObjectStoreMetaDataKey::KEY_GENERATOR_CURRENT_NUMBER)) {
       StringPiece slice(it->Value());
       if (!DecodeInt(&slice, &key_generator_current_number) || !slice.empty())
-        INTERNAL_CONSISTENCY_ERROR(GET_OBJECT_STORES);
+        INTERNAL_CONSISTENCY_ERROR_UNTESTED(GET_OBJECT_STORES);
 
       // TODO(jsbell): Return key_generator_current_number, cache in
       // object store, and write lazily to backing store.  For now,
@@ -1059,18 +1181,20 @@ WARN_UNUSED_RESULT static leveldb::Status SetMaxObjectStoreId(
   leveldb::Status s = GetMaxObjectStoreId(
       transaction, max_object_store_id_key, &max_object_store_id);
   if (!s.ok()) {
-    INTERNAL_READ_ERROR(SET_MAX_OBJECT_STORE_ID);
+    INTERNAL_READ_ERROR_UNTESTED(SET_MAX_OBJECT_STORE_ID);
     return s;
   }
 
   if (object_store_id <= max_object_store_id) {
-    INTERNAL_CONSISTENCY_ERROR(SET_MAX_OBJECT_STORE_ID);
+    INTERNAL_CONSISTENCY_ERROR_UNTESTED(SET_MAX_OBJECT_STORE_ID);
     return InternalInconsistencyStatus();
   }
   PutInt(transaction, max_object_store_id_key, object_store_id);
   return s;
 }
 
+void IndexedDBBackingStore::Flush() { db_->CompactAll(); }
+
 leveldb::Status IndexedDBBackingStore::CreateObjectStore(
     IndexedDBBackingStore::Transaction* transaction,
     int64 database_id,
@@ -1140,11 +1264,11 @@ leveldb::Status IndexedDBBackingStore::DeleteObjectStore(
                 &object_store_name,
                 &found);
   if (!s.ok()) {
-    INTERNAL_READ_ERROR(DELETE_OBJECT_STORE);
+    INTERNAL_READ_ERROR_UNTESTED(DELETE_OBJECT_STORE);
     return s;
   }
   if (!found) {
-    INTERNAL_CONSISTENCY_ERROR(DELETE_OBJECT_STORE);
+    INTERNAL_CONSISTENCY_ERROR_UNTESTED(DELETE_OBJECT_STORE);
     return InternalInconsistencyStatus();
   }
 
@@ -1192,14 +1316,14 @@ leveldb::Status IndexedDBBackingStore::GetRecord(
   if (!found)
     return s;
   if (data.empty()) {
-    INTERNAL_READ_ERROR(GET_RECORD);
+    INTERNAL_READ_ERROR_UNTESTED(GET_RECORD);
     return leveldb::Status::NotFound("Record contained no data");
   }
 
   int64 version;
   StringPiece slice(data);
   if (!DecodeVarInt(&slice, &version)) {
-    INTERNAL_READ_ERROR(GET_RECORD);
+    INTERNAL_READ_ERROR_UNTESTED(GET_RECORD);
     return InternalInconsistencyStatus();
   }
 
@@ -1221,7 +1345,7 @@ WARN_UNUSED_RESULT static leveldb::Status GetNewVersionNumber(
   leveldb::Status s =
       GetInt(transaction, last_version_key, &last_version, &found);
   if (!s.ok()) {
-    INTERNAL_READ_ERROR(GET_NEW_VERSION_NUMBER);
+    INTERNAL_READ_ERROR_UNTESTED(GET_NEW_VERSION_NUMBER);
     return s;
   }
   if (!found)
@@ -1337,13 +1461,13 @@ leveldb::Status IndexedDBBackingStore::GetKeyGeneratorCurrentNumber(
   leveldb::Status s =
       leveldb_transaction->Get(key_generator_current_number_key, &data, &found);
   if (!s.ok()) {
-    INTERNAL_READ_ERROR(GET_KEY_GENERATOR_CURRENT_NUMBER);
+    INTERNAL_READ_ERROR_UNTESTED(GET_KEY_GENERATOR_CURRENT_NUMBER);
     return s;
   }
   if (found && !data.empty()) {
     StringPiece slice(data);
     if (!DecodeInt(&slice, key_generator_current_number) || !slice.empty()) {
-      INTERNAL_READ_ERROR(GET_KEY_GENERATOR_CURRENT_NUMBER);
+      INTERNAL_READ_ERROR_UNTESTED(GET_KEY_GENERATOR_CURRENT_NUMBER);
       return InternalInconsistencyStatus();
     }
     return s;
@@ -1368,7 +1492,7 @@ leveldb::Status IndexedDBBackingStore::GetKeyGeneratorCurrentNumber(
     StringPiece slice(it->Key());
     ObjectStoreDataKey data_key;
     if (!ObjectStoreDataKey::Decode(&slice, &data_key)) {
-      INTERNAL_READ_ERROR(GET_KEY_GENERATOR_CURRENT_NUMBER);
+      INTERNAL_READ_ERROR_UNTESTED(GET_KEY_GENERATOR_CURRENT_NUMBER);
       return InternalInconsistencyStatus();
     }
     scoped_ptr<IndexedDBKey> user_key = data_key.user_key();
@@ -1430,13 +1554,13 @@ leveldb::Status IndexedDBBackingStore::KeyExistsInObjectStore(
   leveldb::Status s =
       transaction->transaction()->Get(leveldb_key, &data, found);
   if (!s.ok()) {
-    INTERNAL_READ_ERROR(KEY_EXISTS_IN_OBJECT_STORE);
+    INTERNAL_READ_ERROR_UNTESTED(KEY_EXISTS_IN_OBJECT_STORE);
     return s;
   }
   if (!*found)
     return leveldb::Status::OK();
   if (!data.size()) {
-    INTERNAL_READ_ERROR(KEY_EXISTS_IN_OBJECT_STORE);
+    INTERNAL_READ_ERROR_UNTESTED(KEY_EXISTS_IN_OBJECT_STORE);
     return InternalInconsistencyStatus();
   }
 
@@ -1493,7 +1617,7 @@ leveldb::Status IndexedDBBackingStore::GetIndexes(
     bool ok = IndexMetaDataKey::Decode(&slice, &meta_data_key);
     DCHECK(ok);
     if (meta_data_key.meta_data_type() != IndexMetaDataKey::NAME) {
-      INTERNAL_CONSISTENCY_ERROR(GET_INDEXES);
+      INTERNAL_CONSISTENCY_ERROR_UNTESTED(GET_INDEXES);
       // Possible stale metadata due to http://webkit.org/b/85557 but don't fail
       // the load.
       it->Next();
@@ -1507,33 +1631,33 @@ leveldb::Status IndexedDBBackingStore::GetIndexes(
     {
       StringPiece slice(it->Value());
       if (!DecodeString(&slice, &index_name) || !slice.empty())
-        INTERNAL_CONSISTENCY_ERROR(GET_INDEXES);
+        INTERNAL_CONSISTENCY_ERROR_UNTESTED(GET_INDEXES);
     }
 
     it->Next();  // unique flag
     if (!CheckIndexAndMetaDataKey(
              it.get(), stop_key, index_id, IndexMetaDataKey::UNIQUE)) {
-      INTERNAL_CONSISTENCY_ERROR(GET_INDEXES);
+      INTERNAL_CONSISTENCY_ERROR_UNTESTED(GET_INDEXES);
       break;
     }
     bool index_unique;
     {
       StringPiece slice(it->Value());
       if (!DecodeBool(&slice, &index_unique) || !slice.empty())
-        INTERNAL_CONSISTENCY_ERROR(GET_INDEXES);
+        INTERNAL_CONSISTENCY_ERROR_UNTESTED(GET_INDEXES);
     }
 
     it->Next();  // key_path
     if (!CheckIndexAndMetaDataKey(
              it.get(), stop_key, index_id, IndexMetaDataKey::KEY_PATH)) {
-      INTERNAL_CONSISTENCY_ERROR(GET_INDEXES);
+      INTERNAL_CONSISTENCY_ERROR_UNTESTED(GET_INDEXES);
       break;
     }
     IndexedDBKeyPath key_path;
     {
       StringPiece slice(it->Value());
       if (!DecodeIDBKeyPath(&slice, &key_path) || !slice.empty())
-        INTERNAL_CONSISTENCY_ERROR(GET_INDEXES);
+        INTERNAL_CONSISTENCY_ERROR_UNTESTED(GET_INDEXES);
     }
 
     it->Next();  // [optional] multi_entry flag
@@ -1542,7 +1666,7 @@ leveldb::Status IndexedDBBackingStore::GetIndexes(
             it.get(), stop_key, index_id, IndexMetaDataKey::MULTI_ENTRY)) {
       StringPiece slice(it->Value());
       if (!DecodeBool(&slice, &index_multi_entry) || !slice.empty())
-        INTERNAL_CONSISTENCY_ERROR(GET_INDEXES);
+        INTERNAL_CONSISTENCY_ERROR_UNTESTED(GET_INDEXES);
 
       it->Next();
     }
@@ -1565,14 +1689,14 @@ WARN_UNUSED_RESULT static leveldb::Status SetMaxIndexId(
   leveldb::Status s =
       GetInt(transaction, max_index_id_key, &max_index_id, &found);
   if (!s.ok()) {
-    INTERNAL_READ_ERROR(SET_MAX_INDEX_ID);
+    INTERNAL_READ_ERROR_UNTESTED(SET_MAX_INDEX_ID);
     return s;
   }
   if (!found)
     max_index_id = kMinimumIndexId;
 
   if (index_id <= max_index_id) {
-    INTERNAL_CONSISTENCY_ERROR(SET_MAX_INDEX_ID);
+    INTERNAL_CONSISTENCY_ERROR_UNTESTED(SET_MAX_INDEX_ID);
     return InternalInconsistencyStatus();
   }
 
@@ -1710,7 +1834,7 @@ static leveldb::Status VersionExists(LevelDBTransaction* transaction,
 
   leveldb::Status s = transaction->Get(key, &data, exists);
   if (!s.ok()) {
-    INTERNAL_READ_ERROR(VERSION_EXISTS);
+    INTERNAL_READ_ERROR_UNTESTED(VERSION_EXISTS);
     return s;
   }
   if (!*exists)
@@ -1754,7 +1878,7 @@ leveldb::Status IndexedDBBackingStore::FindKeyInIndex(
 
     int64 version;
     if (!DecodeVarInt(&slice, &version)) {
-      INTERNAL_READ_ERROR(FIND_KEY_IN_INDEX);
+      INTERNAL_READ_ERROR_UNTESTED(FIND_KEY_IN_INDEX);
       return InternalInconsistencyStatus();
     }
     *found_encoded_primary_key = slice.as_string();
@@ -1800,13 +1924,13 @@ leveldb::Status IndexedDBBackingStore::GetPrimaryKeyViaIndex(
                                      &found_encoded_primary_key,
                                      &found);
   if (!s.ok()) {
-    INTERNAL_READ_ERROR(GET_PRIMARY_KEY_VIA_INDEX);
+    INTERNAL_READ_ERROR_UNTESTED(GET_PRIMARY_KEY_VIA_INDEX);
     return s;
   }
   if (!found)
     return s;
   if (!found_encoded_primary_key.size()) {
-    INTERNAL_READ_ERROR(GET_PRIMARY_KEY_VIA_INDEX);
+    INTERNAL_READ_ERROR_UNTESTED(GET_PRIMARY_KEY_VIA_INDEX);
     return InvalidDBKeyStatus();
   }
 
@@ -1839,13 +1963,13 @@ leveldb::Status IndexedDBBackingStore::KeyExistsInIndex(
                                      &found_encoded_primary_key,
                                      exists);
   if (!s.ok()) {
-    INTERNAL_READ_ERROR(KEY_EXISTS_IN_INDEX);
+    INTERNAL_READ_ERROR_UNTESTED(KEY_EXISTS_IN_INDEX);
     return s;
   }
   if (!*exists)
     return leveldb::Status::OK();
   if (found_encoded_primary_key.empty()) {
-    INTERNAL_READ_ERROR(KEY_EXISTS_IN_INDEX);
+    INTERNAL_READ_ERROR_UNTESTED(KEY_EXISTS_IN_INDEX);
     return InvalidDBKeyStatus();
   }
 
@@ -2088,7 +2212,7 @@ bool ObjectStoreKeyCursorImpl::LoadCurrentRow() {
   StringPiece slice(iterator_->Key());
   ObjectStoreDataKey object_store_data_key;
   if (!ObjectStoreDataKey::Decode(&slice, &object_store_data_key)) {
-    INTERNAL_READ_ERROR(LOAD_CURRENT_ROW);
+    INTERNAL_READ_ERROR_UNTESTED(LOAD_CURRENT_ROW);
     return false;
   }
 
@@ -2097,7 +2221,7 @@ bool ObjectStoreKeyCursorImpl::LoadCurrentRow() {
   int64 version;
   slice = StringPiece(iterator_->Value());
   if (!DecodeVarInt(&slice, &version)) {
-    INTERNAL_READ_ERROR(LOAD_CURRENT_ROW);
+    INTERNAL_READ_ERROR_UNTESTED(LOAD_CURRENT_ROW);
     return false;
   }
 
@@ -2145,7 +2269,7 @@ bool ObjectStoreCursorImpl::LoadCurrentRow() {
   StringPiece key_slice(iterator_->Key());
   ObjectStoreDataKey object_store_data_key;
   if (!ObjectStoreDataKey::Decode(&key_slice, &object_store_data_key)) {
-    INTERNAL_READ_ERROR(LOAD_CURRENT_ROW);
+    INTERNAL_READ_ERROR_UNTESTED(LOAD_CURRENT_ROW);
     return false;
   }
 
@@ -2154,7 +2278,7 @@ bool ObjectStoreCursorImpl::LoadCurrentRow() {
   int64 version;
   StringPiece value_slice = StringPiece(iterator_->Value());
   if (!DecodeVarInt(&value_slice, &version)) {
-    INTERNAL_READ_ERROR(LOAD_CURRENT_ROW);
+    INTERNAL_READ_ERROR_UNTESTED(LOAD_CURRENT_ROW);
     return false;
   }
 
@@ -2219,7 +2343,7 @@ bool IndexKeyCursorImpl::LoadCurrentRow() {
   StringPiece slice(iterator_->Key());
   IndexDataKey index_data_key;
   if (!IndexDataKey::Decode(&slice, &index_data_key)) {
-    INTERNAL_READ_ERROR(LOAD_CURRENT_ROW);
+    INTERNAL_READ_ERROR_UNTESTED(LOAD_CURRENT_ROW);
     return false;
   }
 
@@ -2229,12 +2353,12 @@ bool IndexKeyCursorImpl::LoadCurrentRow() {
   slice = StringPiece(iterator_->Value());
   int64 index_data_version;
   if (!DecodeVarInt(&slice, &index_data_version)) {
-    INTERNAL_READ_ERROR(LOAD_CURRENT_ROW);
+    INTERNAL_READ_ERROR_UNTESTED(LOAD_CURRENT_ROW);
     return false;
   }
 
   if (!DecodeIDBKey(&slice, &primary_key_) || !slice.empty()) {
-    INTERNAL_READ_ERROR(LOAD_CURRENT_ROW);
+    INTERNAL_READ_ERROR_UNTESTED(LOAD_CURRENT_ROW);
     return false;
   }
 
@@ -2247,7 +2371,7 @@ bool IndexKeyCursorImpl::LoadCurrentRow() {
   bool found = false;
   leveldb::Status s = transaction_->Get(primary_leveldb_key, &result, &found);
   if (!s.ok()) {
-    INTERNAL_READ_ERROR(LOAD_CURRENT_ROW);
+    INTERNAL_READ_ERROR_UNTESTED(LOAD_CURRENT_ROW);
     return false;
   }
   if (!found) {
@@ -2255,14 +2379,14 @@ bool IndexKeyCursorImpl::LoadCurrentRow() {
     return false;
   }
   if (!result.size()) {
-    INTERNAL_READ_ERROR(LOAD_CURRENT_ROW);
+    INTERNAL_READ_ERROR_UNTESTED(LOAD_CURRENT_ROW);
     return false;
   }
 
   int64 object_store_data_version;
   slice = StringPiece(result);
   if (!DecodeVarInt(&slice, &object_store_data_version)) {
-    INTERNAL_READ_ERROR(LOAD_CURRENT_ROW);
+    INTERNAL_READ_ERROR_UNTESTED(LOAD_CURRENT_ROW);
     return false;
   }
 
@@ -2327,7 +2451,7 @@ bool IndexCursorImpl::LoadCurrentRow() {
   StringPiece slice(iterator_->Key());
   IndexDataKey index_data_key;
   if (!IndexDataKey::Decode(&slice, &index_data_key)) {
-    INTERNAL_READ_ERROR(LOAD_CURRENT_ROW);
+    INTERNAL_READ_ERROR_UNTESTED(LOAD_CURRENT_ROW);
     return false;
   }
 
@@ -2337,11 +2461,11 @@ bool IndexCursorImpl::LoadCurrentRow() {
   slice = StringPiece(iterator_->Value());
   int64 index_data_version;
   if (!DecodeVarInt(&slice, &index_data_version)) {
-    INTERNAL_READ_ERROR(LOAD_CURRENT_ROW);
+    INTERNAL_READ_ERROR_UNTESTED(LOAD_CURRENT_ROW);
     return false;
   }
   if (!DecodeIDBKey(&slice, &primary_key_)) {
-    INTERNAL_READ_ERROR(LOAD_CURRENT_ROW);
+    INTERNAL_READ_ERROR_UNTESTED(LOAD_CURRENT_ROW);
     return false;
   }
 
@@ -2354,7 +2478,7 @@ bool IndexCursorImpl::LoadCurrentRow() {
   bool found = false;
   leveldb::Status s = transaction_->Get(primary_leveldb_key_, &result, &found);
   if (!s.ok()) {
-    INTERNAL_READ_ERROR(LOAD_CURRENT_ROW);
+    INTERNAL_READ_ERROR_UNTESTED(LOAD_CURRENT_ROW);
     return false;
   }
   if (!found) {
@@ -2362,14 +2486,14 @@ bool IndexCursorImpl::LoadCurrentRow() {
     return false;
   }
   if (!result.size()) {
-    INTERNAL_READ_ERROR(LOAD_CURRENT_ROW);
+    INTERNAL_READ_ERROR_UNTESTED(LOAD_CURRENT_ROW);
     return false;
   }
 
   int64 object_store_data_version;
   slice = StringPiece(result);
   if (!DecodeVarInt(&slice, &object_store_data_version)) {
-    INTERNAL_READ_ERROR(LOAD_CURRENT_ROW);
+    INTERNAL_READ_ERROR_UNTESTED(LOAD_CURRENT_ROW);
     return false;
   }
 
@@ -2640,7 +2764,7 @@ leveldb::Status IndexedDBBackingStore::Transaction::Commit() {
   leveldb::Status s = transaction_->Commit();
   transaction_ = NULL;
   if (!s.ok())
-    INTERNAL_WRITE_ERROR(TRANSACTION_COMMIT_METHOD);
+    INTERNAL_WRITE_ERROR_UNTESTED(TRANSACTION_COMMIT_METHOD);
   return s;
 }