[sql] Validate database files before enabling memory-mapping.

sql/connection.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "sql/connection.h"
 
 #include <string.h>
 
 #include "base/bind.h"
 #include "base/debug/dump_without_crashing.h"
@@ skipping 172 matching lines @@
 
   // TODO(shess): NULL in file->pMethods has been observed on android_dbg
   // content_unittests, even though it should not be possible.
   // http://crbug.com/329982
   if (!*file || !(*file)->pMethods)
     return SQLITE_ERROR;
 
   return rc;
 }
 
+// Convenience to get the sqlite3_file* and the size for the "main" database.
+int GetSqlite3FileAndSize(sqlite3* db,
+                          sqlite3_file** file, sqlite3_int64* db_size) {
+  int rc = GetSqlite3File(db, file);
+  if (rc != SQLITE_OK)
+    return rc;
+
+  return (*file)->pMethods->xFileSize(*file, db_size);
+}
+
 // This should match UMA_HISTOGRAM_MEDIUM_TIMES().
 base::HistogramBase* GetMediumTimeHistogram(const std::string& name) {
   return base::Histogram::FactoryTimeGet(
       name,
       base::TimeDelta::FromMilliseconds(10),
       base::TimeDelta::FromMinutes(3),
       50,
       base::HistogramBase::kUmaTargetedHistogramFlag);
 }
 
@@ skipping 305 matching lines @@
   }
 
   // Use local settings if provided, otherwise use documented defaults.  The
   // actual results could be fetching via PRAGMA calls.
   const int page_size = page_size_ ? page_size_ : 1024;
   sqlite3_int64 preload_size = page_size * (cache_size_ ? cache_size_ : 2000);
   if (preload_size < 1)
     return;
 
   sqlite3_file* file = NULL;
-  int rc = GetSqlite3File(db_, &file);
+  sqlite3_int64 file_size = 0;
+  int rc = GetSqlite3FileAndSize(db_, &file, &file_size);
   if (rc != SQLITE_OK)
     return;
 
-  sqlite3_int64 file_size = 0;
-  rc = file->pMethods->xFileSize(file, &file_size);
-  if (rc != SQLITE_OK)
-    return;
-
   // Don't preload more than the file contains.
   if (preload_size > file_size)
     preload_size = file_size;
 
   scoped_ptr<char[]> buf(new char[page_size]);
   for (sqlite3_int64 pos = 0; pos < preload_size; pos += page_size) {
     rc = file->pMethods->xRead(file, buf.get(), page_size, pos);
+
+    // TODO(shess): Consider calling OnSqliteError().
     if (rc != SQLITE_OK)
       return;
   }
 }
 
 // SQLite keeps unused pages associated with a connection in a cache.  It asks
 // the cache for pages by an id, and if the page is present and the database is
 // unchanged, it considers the content of the page valid and doesn't read it
 // from disk.  When memory-mapped I/O is enabled, on read SQLite uses page
 // structures created from the memory map data before consulting the cache.  On
@@ skipping 304 matching lines @@
     // keep close to the 2000-character size limit for dumping.
     const size_t kMaxMessages = 20;
     for (size_t i = 0; i < kMaxMessages && i < messages.size(); ++i) {
       base::StringAppendF(&debug_info, "%s\n", messages[i].c_str());
     }
   }
 
   return debug_info;
 }
 
+size_t Connection::GetAppropriateMmapSize() {
+  AssertIOAllowed();
+
+  // TODO(shess): Using sql::MetaTable seems indicated, but mixing
+  // sql::MetaTable and direct access seems error-prone.  It might make sense to
+  // simply integrate sql::MetaTable functionality into sql::Connection.
+
+#if defined(OS_IOS)
+  // iOS SQLite does not support memory mapping.
+  return 0;
+#endif
+
+  // If the database doesn't have a place to track progress, assume the worst.
+  // This will happen when new databases are created.
+  if (!DoesTableExist("meta")) {
+    RecordOneEvent(EVENT_MMAP_META_MISSING);
+    return 0;
+  }
+
+  // Key into meta table to get status from a previous run.  The value
+  // represents how much data in bytes has successfully been read from the
+  // database.  |kMmapFailure| indicates that there was a read error and the
+  // database should not be memory-mapped, while |kMmapSuccess| indicates that
+  // the entire file was read at some point and can be memory-mapped without
+  // constraint.
+  const char* kMmapStatusKey = "mmap_status";
+  static const sqlite3_int64 kMmapFailure = -2;
+  static const sqlite3_int64 kMmapSuccess = -1;
+
+  // Start reading from 0 unless status is found in meta table.
+  sqlite3_int64 mmap_ofs = 0;
+
+  // Retrieve the current status.  It is fine for the status to be missing
+  // entirely, but any error prevents memory-mapping.
+  {
+    const char* kMmapStatusSql = "SELECT value FROM meta WHERE key = ?";
+    Statement s(GetUniqueStatement(kMmapStatusSql));
+    s.BindString(0, kMmapStatusKey);
+    if (s.Step()) {
+      mmap_ofs = s.ColumnInt64(0);
+    } else if (!s.Succeeded()) {
+      RecordOneEvent(EVENT_MMAP_META_FAILURE_READ);
+      return 0;
+    }
+  }
+
+  // Database read failed in the past, don't memory map.
+  if (mmap_ofs == kMmapFailure) {
+    RecordOneEvent(EVENT_MMAP_FAILED);
+    return 0;
+  } else if (mmap_ofs != kMmapSuccess) {
+    // Continue reading from previous offset.
+    DCHECK_GE(mmap_ofs, 0);
+
+    // TODO(shess): Could this reading code be shared with Preload()?  It would
+    // require locking twice (this code wouldn't be able to access |db_size| so
+    // the helper would have to return amount read).
+
+    // Read more of the database looking for errors.  The VFS interface is used
+    // to assure that the reads are valid for SQLite.  |g_reads_allowed| is used
+    // to limit checking to 20MB per run of Chromium.
+    sqlite3_file* file = NULL;
+    sqlite3_int64 db_size = 0;
+    if (SQLITE_OK != GetSqlite3FileAndSize(db_, &file, &db_size)) {

[Scott Hess - ex-Googler, 2015/11/05 18:25:07]

I made this change because I found myself pondering whether the change to enter
the read loop with amount==0 would have any impact.  |rc| there clearly needed
to be local, but this |rc| doesn't need to stick around.

+      RecordOneEvent(EVENT_MMAP_VFS_FAILURE);
+      return 0;
+    }
+
+    // Read the data left, or |g_reads_allowed|, whichever is smaller.
+    // |g_reads_allowed| limits the total amount of I/O to spend verifying data
+    // in a single Chromium run.
+    sqlite3_int64 amount = db_size - mmap_ofs;
+    if (amount < 0)
+      amount = 0;
+    if (amount > 0) {
+      base::AutoLock lock(g_sqlite_init_lock.Get());
+      static sqlite3_int64 g_reads_allowed = 20 * 1024 * 1024;
+      if (g_reads_allowed < amount)
+        amount = g_reads_allowed;
+      g_reads_allowed -= amount;
+    }
+
+    // |amount| can be <= 0 if |g_reads_allowed| ran out of quota, or if the
+    // database was truncated after a previous pass.
+    if (amount <= 0 && mmap_ofs < db_size) {

[Scott Hess - ex-Googler, 2015/11/05 18:25:07]

Took this approach because replicating a subset of the else clause seems
brittle.

+      DCHECK_EQ(0, amount);
+      RecordOneEvent(EVENT_MMAP_SUCCESS_NO_PROGRESS);
+    } else {
+      static const int kPageSize = 4096;
+      char buf[kPageSize];
+      while (amount > 0) {
+        int rc = file->pMethods->xRead(file, buf, sizeof(buf), mmap_ofs);
+        if (rc == SQLITE_OK) {
+          mmap_ofs += sizeof(buf);
+          amount -= sizeof(buf);
+        } else if (rc == SQLITE_IOERR_SHORT_READ) {
+          // Reached EOF for a database with page size < |kPageSize|.
+          mmap_ofs = db_size;
+          break;
+        } else {
+          // TODO(shess): Consider calling OnSqliteError().
+          mmap_ofs = kMmapFailure;
+          break;
+        }
+      }
+
+      // Log these events after update to distinguish meta update failure.
+      Events event;
+      if (mmap_ofs >= db_size) {
+        mmap_ofs = kMmapSuccess;
+        event = EVENT_MMAP_SUCCESS_NEW;
+      } else if (mmap_ofs > 0) {
+        event = EVENT_MMAP_SUCCESS_PARTIAL;
+      } else {
+        DCHECK_EQ(kMmapFailure, mmap_ofs);
+        event = EVENT_MMAP_FAILED_NEW;
+      }
+
+      const char* kMmapUpdateStatusSql = "REPLACE INTO meta VALUES (?, ?)";
+      Statement s(GetUniqueStatement(kMmapUpdateStatusSql));
+      s.BindString(0, kMmapStatusKey);
+      s.BindInt64(1, mmap_ofs);
+      if (!s.Run()) {
+        RecordOneEvent(EVENT_MMAP_META_FAILURE_UPDATE);
+        return 0;
+      }
+
+      RecordOneEvent(event);
+    }
+  }
+
+  if (mmap_ofs == kMmapFailure)
+    return 0;
+  if (mmap_ofs == kMmapSuccess)
+    return 256 * 1024 * 1024;
+  return mmap_ofs;
+}
+
 void Connection::TrimMemory(bool aggressively) {
   if (!db_)
     return;
 
   // TODO(shess): investigate using sqlite3_db_release_memory() when possible.
   int original_cache_size;
   {
     Statement sql_get_original(GetUniqueStatement("PRAGMA cache_size"));
     if (!sql_get_original.Step()) {
       DLOG(WARNING) << "Could not get cache size " << GetErrorMessage();
@@ skipping 753 matching lines @@
 
   // http://www.sqlite.org/pragma.html#pragma_journal_mode
   // DELETE (default) - delete -journal file to commit.
   // TRUNCATE - truncate -journal file to commit.
   // PERSIST - zero out header of -journal file to commit.
   // TRUNCATE should be faster than DELETE because it won't need directory
   // changes for each transaction.  PERSIST may break the spirit of using
   // secure_delete.
   ignore_result(Execute("PRAGMA journal_mode = TRUNCATE"));
 
-  // Enable memory-mapped access.  This value will be capped by
-  // SQLITE_MAX_MMAP_SIZE, which could be different between 32-bit and 64-bit
-  // platforms.
-  mmap_enabled_ = false;
-  if (!mmap_disabled_)
-    ignore_result(Execute("PRAGMA mmap_size = 268435456"));  // 256MB.
-  {
-    Statement s(GetUniqueStatement("PRAGMA mmap_size"));
-    if (s.Step() && s.ColumnInt64(0) > 0)
-      mmap_enabled_ = true;
-  }
-
   const base::TimeDelta kBusyTimeout =
     base::TimeDelta::FromSeconds(kBusyTimeoutSeconds);
 
   if (page_size_ != 0) {
     // Enforce SQLite restrictions on |page_size_|.
     DCHECK(!(page_size_ & (page_size_ - 1)))
         << " page_size_ " << page_size_ << " is not a power of two.";
     const int kSqliteMaxPageSize = 32768;  // from sqliteLimit.h
     DCHECK_LE(page_size_, kSqliteMaxPageSize);
     const std::string sql =
         base::StringPrintf("PRAGMA page_size=%d", page_size_);
     ignore_result(ExecuteWithTimeout(sql.c_str(), kBusyTimeout));
   }
 
   if (cache_size_ != 0) {
     const std::string sql =
         base::StringPrintf("PRAGMA cache_size=%d", cache_size_);
     ignore_result(ExecuteWithTimeout(sql.c_str(), kBusyTimeout));
   }
 
   if (!ExecuteWithTimeout("PRAGMA secure_delete=ON", kBusyTimeout)) {
     bool was_poisoned = poisoned_;
     Close();
     if (was_poisoned && retry_flag == RETRY_ON_POISON)
       return OpenInternal(file_name, NO_RETRY);
     return false;
   }
 
+  // Enable memory-mapped access.  The explicit-disable case is because SQLite
+  // can be built to default-enable mmap.  GetAppropriateMmapSize() calculates a
+  // safe range to memory-map based on past regular I/O.  This value will be
+  // capped by SQLITE_MAX_MMAP_SIZE, which could be different between 32-bit and
+  // 64-bit platforms.
+  size_t mmap_size = mmap_disabled_ ? 0 : GetAppropriateMmapSize();
+  std::string mmap_sql =
+      base::StringPrintf("PRAGMA mmap_size = %" PRIuS, mmap_size);
+  ignore_result(Execute(mmap_sql.c_str()));
+
+  // Determine if memory-mapping has actually been enabled.  The Execute() above
+  // can succeed without changing the amount mapped.
+  mmap_enabled_ = false;
+  {
+    Statement s(GetUniqueStatement("PRAGMA mmap_size"));
+    if (s.Step() && s.ColumnInt64(0) > 0)
+      mmap_enabled_ = true;
+  }
+
   return true;
 }
 
 void Connection::DoRollback() {
   Statement rollback(GetCachedStatement(SQL_FROM_HERE, "ROLLBACK"));
 
   // Collect the rollback time manually, sql::Statement would register it as
   // query time only.
   const base::TimeTicks before = Now();
   rollback.RunWithoutTimers();
@@ skipping 119 matching lines @@
   ignore_result(Execute(kNoWritableSchema));
 
   return ret;
 }
 
 base::TimeTicks TimeSource::Now() {
   return base::TimeTicks::Now();
 }
 
 }  // namespace sql