[sql] Annotate sql::Recovery failure cases.

sql/recovery.cc

 // Copyright 2013 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "sql/recovery.h"
 
 #include "base/files/file_path.h"
 #include "base/format_macros.h"
 #include "base/logging.h"
+#include "base/metrics/histogram.h"
 #include "base/metrics/sparse_histogram.h"
 #include "base/strings/string_util.h"
 #include "base/strings/stringprintf.h"
 #include "sql/connection.h"
 #include "sql/statement.h"
 #include "third_party/sqlite/sqlite3.h"
 
 namespace sql {
 
+namespace {
+
+enum RecoveryEventType {
+  // Init() completed successfully.
+  RECOVERY_SUCCESS_INIT = 0,
+
+  // Failed to open temporary database to recover into.
+  RECOVERY_FAILED_OPEN_TEMPORARY,
+
+  // Failed to initialize recover vtable system.
+  RECOVERY_FAILED_VIRTUAL_TABLE_INIT,
+
+  // System SQLite doesn't support vtable.
+  RECOVERY_FAILED_VIRTUAL_TABLE_SYSTEM_SQLITE,
+
+  // Failed attempting to enable writable_schema.
+  RECOVERY_FAILED_WRITABLE_SCHEMA,
+
+  // Failed to attach the corrupt database to the temporary database.
+  RECOVERY_FAILED_ATTACH,
+
+  // Backup() successfully completed.
+  RECOVERY_SUCCESS_BACKUP,
+
+  // Failed sqlite3_backup_init().  Error code in Sqlite.RecoveryHandle.
+  RECOVERY_FAILED_BACKUP_INIT,
+
+  // Failed sqlite3_backup_step().  Error code in Sqlite.RecoveryStep.
+  RECOVERY_FAILED_BACKUP_STEP,
+
+  // AutoRecoverTable() successfully completed.
+  RECOVERY_SUCCESS_AUTORECOVER,
+
+  // The target table contained a type which the code is not equipped
+  // to handle.  This should only happen if things are fubar.
+  RECOVERY_FAILED_AUTORECOVER_UNRECOGNIZED_TYPE,
+
+  // The target table does not exist.
+  RECOVERY_FAILED_AUTORECOVER_MISSING_TABLE,
+
+  // The recovery virtual table creation failed.
+  RECOVERY_FAILED_AUTORECOVER_CREATE,
+
+  // Copying data from the recovery table to the target table failed.
+  RECOVERY_FAILED_AUTORECOVER_INSERT,
+
+  // Dropping the recovery virtual table failed.
+  RECOVERY_FAILED_AUTORECOVER_DROP,
+
+  // SetupMeta() successfully completed.
+  RECOVERY_SUCCESS_SETUP_META,
+
+  // Failure creating recovery meta table.
+  RECOVERY_FAILED_META_CREATE,
+
+  // GetMetaVersionNumber() successfully completed.
+  RECOVERY_SUCCESS_META_VERSION,
+
+  // Failed in querying recovery meta table.
+  RECOVERY_FAILED_META_QUERY,
+
+  // No version key in recovery meta table.
+  RECOVERY_FAILED_META_NO_VERSION,
+
+  // Always keep this at the end.
+  RECOVERY_EVENT_MAX,
+};
+
+void RecordRecoveryEvent(RecoveryEventType recovery_event) {
+  UMA_HISTOGRAM_ENUMERATION("Sqlite.RecoveryEvents",
+                            recovery_event, RECOVERY_EVENT_MAX);
+}
+
+}  // namespace
+
 // static
 bool Recovery::FullRecoverySupported() {
   // TODO(shess): See comment in Init().
 #if defined(USE_SYSTEM_SQLITE)
   return false;
 #else
   return true;
 #endif
 }
 
@@ skipping 66 matching lines @@
   // next database access, so immediately force an access.  Enabling
   // writable_schema allows processing through certain kinds of
   // corruption.
   // TODO(shess): It would be better to just close the handle, but it
   // is necessary for the final backup which rewrites things.  It
   // might be reasonable to close then re-open the handle.
   ignore_result(db_->Execute("PRAGMA writable_schema=1"));
   ignore_result(db_->Execute("PRAGMA locking_mode=NORMAL"));
   ignore_result(db_->Execute("SELECT COUNT(*) FROM sqlite_master"));
 
-  if (!recover_db_.OpenTemporary())
+  // TODO(shess): If this is a common failure case, it might be
+  // possible to fall back to a memory database.  But it probably
+  // implies that the SQLite tmpdir logic is busted, which could cause
+  // a variety of other random issues in our code.
+  if (!recover_db_.OpenTemporary()) {
+    RecordRecoveryEvent(RECOVERY_FAILED_OPEN_TEMPORARY);
     return false;
+  }
 
   // TODO(shess): Figure out a story for USE_SYSTEM_SQLITE.  The
   // virtual table implementation relies on SQLite internals for some
   // types and functions, which could be copied inline to make it
   // standalone.  Or an alternate implementation could try to read
   // through errors entirely at the SQLite level.
   //
   // For now, defer to the caller.  The setup will succeed, but the
   // later CREATE VIRTUAL TABLE call will fail, at which point the
   // caller can fire Unrecoverable().
 #if !defined(USE_SYSTEM_SQLITE)
   int rc = recoverVtableInit(recover_db_.db_);
   if (rc != SQLITE_OK) {
+    RecordRecoveryEvent(RECOVERY_FAILED_VIRTUAL_TABLE_INIT);
     LOG(ERROR) << "Failed to initialize recover module: "
                << recover_db_.GetErrorMessage();
     return false;
   }
+#else
+  // If this is infrequent enough, just wire it to Raze().
+  RecordRecoveryEvent(RECOVERY_FAILED_VIRTUAL_TABLE_SYSTEM_SQLITE);
 #endif
 
   // Turn on |SQLITE_RecoveryMode| for the handle, which allows
   // reading certain broken databases.
-  if (!recover_db_.Execute("PRAGMA writable_schema=1"))
+  if (!recover_db_.Execute("PRAGMA writable_schema=1")) {
+    RecordRecoveryEvent(RECOVERY_FAILED_WRITABLE_SCHEMA);
     return false;
+  }
 
-  if (!recover_db_.AttachDatabase(db_path, "corrupt"))
+  if (!recover_db_.AttachDatabase(db_path, "corrupt")) {
+    RecordRecoveryEvent(RECOVERY_FAILED_ATTACH);
     return false;
+  }
 
+  RecordRecoveryEvent(RECOVERY_SUCCESS_INIT);
   return true;
 }
 
 bool Recovery::Backup() {
   CHECK(db_);
   CHECK(recover_db_.is_open());
 
   // TODO(shess): Some of the failure cases here may need further
   // exploration.  Just as elsewhere, persistent problems probably
   // need to be razed, while anything which might succeed on a future
@@ skipping 22 matching lines @@
   // and attempt the backup again.
   //
   // For now, this code attempts a best effort and records histograms
   // to inform future development.
 
   // Backup the original db from the recovered db.
   const char* kMain = "main";
   sqlite3_backup* backup = sqlite3_backup_init(db_->db_, kMain,
                                                recover_db_.db_, kMain);
   if (!backup) {
+    RecordRecoveryEvent(RECOVERY_FAILED_BACKUP_INIT);
+
     // Error code is in the destination database handle.
-    int err = sqlite3_errcode(db_->db_);
+    int err = sqlite3_extended_errcode(db_->db_);
     UMA_HISTOGRAM_SPARSE_SLOWLY("Sqlite.RecoveryHandle", err);
     LOG(ERROR) << "sqlite3_backup_init() failed: "
                << sqlite3_errmsg(db_->db_);
+
     return false;
   }
 
   // -1 backs up the entire database.
   int rc = sqlite3_backup_step(backup, -1);
   int pages = sqlite3_backup_pagecount(backup);
   // TODO(shess): sqlite3_backup_finish() appears to allow returning a
   // different value from sqlite3_backup_step().  Circle back and
   // figure out if that can usefully inform the decision of whether to
   // retry or not.
   sqlite3_backup_finish(backup);
   DCHECK_GT(pages, 0);
 
   if (rc != SQLITE_DONE) {
+    RecordRecoveryEvent(RECOVERY_FAILED_BACKUP_STEP);
     UMA_HISTOGRAM_SPARSE_SLOWLY("Sqlite.RecoveryStep", rc);
     LOG(ERROR) << "sqlite3_backup_step() failed: "
                << sqlite3_errmsg(db_->db_);
   }
 
   // The destination database was locked.  Give up, but leave the data
   // in place.  Maybe it won't be locked next time.
   if (rc == SQLITE_BUSY || rc == SQLITE_LOCKED) {
     Shutdown(POISON);
     return false;
   }
 
   // Running out of memory should be transient, retry later.
   if (rc == SQLITE_NOMEM) {
     Shutdown(POISON);
     return false;
   }
 
   // TODO(shess): For now, leave the original database alone, pending
   // results from Sqlite.RecoveryStep.  Some errors should probably
   // route to RAZE_AND_POISON.
   if (rc != SQLITE_DONE) {
     Shutdown(POISON);
     return false;
   }
 
   // Clean up the recovery db, and terminate the main database
   // connection.
+  RecordRecoveryEvent(RECOVERY_SUCCESS_BACKUP);
   Shutdown(POISON);
   return true;
 }
 
 void Recovery::Shutdown(Recovery::Disposition raze) {
   if (!db_)
     return;
 
   recover_db_.Close();
   if (raze == RAZE_AND_POISON) {
@@ skipping 66 matching lines @@
       column_decl += " BLOB";
     } else {
       // TODO(shess): AFAICT, there remain:
       // - contains("CLOB") -> TEXT
       // - contains("REAL") -> REAL
       // - contains("FLOA") -> REAL
       // - contains("DOUB") -> REAL
       // - other -> "NUMERIC"
       // Just code those in as they come up.
       NOTREACHED() << " Unsupported type " << column_type;
+      RecordRecoveryEvent(RECOVERY_FAILED_AUTORECOVER_UNRECOGNIZED_TYPE);
       return false;
     }
 
     // If column has constraint "NOT NULL", then inserting NULL into
     // that column will fail.  If the column has a non-NULL DEFAULT
     // specified, the INSERT will handle it (see below).  If the
     // DEFAULT is also NULL, the row must be filtered out.
     // TODO(shess): The above scenario applies to INSERT OR REPLACE,
     // whereas INSERT OR IGNORE drops such rows.
     // http://www.sqlite.org/lang_conflict.html
@@ skipping 10 matching lines @@
     } else {
       // The default value appears to be pre-quoted, as if it is
       // literally from the sqlite_master CREATE statement.
       std::string default_value = s.ColumnString(4);
       insert_columns.push_back(base::StringPrintf(
           "IFNULL(%s,%s)", column_name.c_str(), default_value.c_str()));
     }
   }
 
   // Receiving no column information implies that the table doesn't exist.
-  if (create_column_decls.empty())
+  if (create_column_decls.empty()) {
+    RecordRecoveryEvent(RECOVERY_FAILED_AUTORECOVER_MISSING_TABLE);
     return false;
+  }
 
   // If the PRIMARY KEY was a single INTEGER column, convert it to ROWID.
   if (pk_column_count == 1 && !rowid_decl.empty())
     create_column_decls[rowid_ofs] = rowid_decl;
 
   // Additional columns accept anything.
   // TODO(shess): ignoreN isn't well namespaced.  But it will fail to
   // execute in case of conflicts.
   for (size_t i = 0; i < extend_columns; ++i) {
     create_column_decls.push_back(
         base::StringPrintf("ignore%" PRIuS " ANY", i));
   }
 
   std::string recover_create(base::StringPrintf(
       "CREATE VIRTUAL TABLE temp.recover_%s USING recover(corrupt.%s, %s)",
       table_name,
       table_name,
       JoinString(create_column_decls, ',').c_str()));
 
   std::string recover_insert(base::StringPrintf(
       "INSERT OR REPLACE INTO main.%s SELECT %s FROM temp.recover_%s",
       table_name,
       JoinString(insert_columns, ',').c_str(),
       table_name));
 
   std::string recover_drop(base::StringPrintf(
       "DROP TABLE temp.recover_%s", table_name));
 
-  if (!db()->Execute(recover_create.c_str()))
+  if (!db()->Execute(recover_create.c_str())) {
+    RecordRecoveryEvent(RECOVERY_FAILED_AUTORECOVER_CREATE);
     return false;
+  }
 
   if (!db()->Execute(recover_insert.c_str())) {
+    RecordRecoveryEvent(RECOVERY_FAILED_AUTORECOVER_INSERT);
     ignore_result(db()->Execute(recover_drop.c_str()));
     return false;
   }
 
   *rows_recovered = db()->GetLastChangeCount();
 
   // TODO(shess): Is leaving the recover table around a breaker?
-  return db()->Execute(recover_drop.c_str());
+  if (!db()->Execute(recover_drop.c_str())) {
+    RecordRecoveryEvent(RECOVERY_FAILED_AUTORECOVER_DROP);
+    return false;
+  }
+  RecordRecoveryEvent(RECOVERY_SUCCESS_AUTORECOVER);
+  return true;
 }
 
 bool Recovery::SetupMeta() {
   const char kCreateSql[] =
       "CREATE VIRTUAL TABLE temp.recover_meta USING recover"
       "("
       "corrupt.meta,"
       "key TEXT NOT NULL,"
       "value ANY"  // Whatever is stored.
       ")";
-  return db()->Execute(kCreateSql);
+  if (!db()->Execute(kCreateSql)) {
+    RecordRecoveryEvent(RECOVERY_FAILED_META_CREATE);
+    return false;
+  }
+  RecordRecoveryEvent(RECOVERY_SUCCESS_SETUP_META);
+  return true;
 }
 
 bool Recovery::GetMetaVersionNumber(int* version) {
   DCHECK(version);
   // TODO(shess): DCHECK(db()->DoesTableExist("temp.recover_meta"));
   // Unfortunately, DoesTableExist() queries sqlite_master, not
   // sqlite_temp_master.
 
   const char kVersionSql[] =
       "SELECT value FROM temp.recover_meta WHERE key = 'version'";
   sql::Statement recovery_version(db()->GetUniqueStatement(kVersionSql));
-  if (!recovery_version.Step())
+  if (!recovery_version.Step()) {
+    if (!recovery_version.Succeeded()) {
+      RecordRecoveryEvent(RECOVERY_FAILED_META_QUERY);
+    } else {
+      RecordRecoveryEvent(RECOVERY_FAILED_META_NO_VERSION);
+    }
     return false;
+  }
 
+  RecordRecoveryEvent(RECOVERY_SUCCESS_META_VERSION);
   *version = recovery_version.ColumnInt(0);
   return true;
 }
 
 }  // namespace sql