[sql] Consider fresh databases suitable for memory-mapped I/O.

sql/connection.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "sql/connection.h"
 
 #include <string.h>
 
 #include "base/bind.h"
 #include "base/debug/dump_without_crashing.h"
 #include "base/files/file_path.h"
 #include "base/files/file_util.h"
 #include "base/format_macros.h"
 #include "base/json/json_file_value_serializer.h"
 #include "base/lazy_instance.h"
 #include "base/logging.h"
 #include "base/message_loop/message_loop.h"
 #include "base/metrics/histogram.h"
 #include "base/metrics/sparse_histogram.h"
 #include "base/strings/string_split.h"
 #include "base/strings/string_util.h"
 #include "base/strings/stringprintf.h"
 #include "base/strings/utf_string_conversions.h"
 #include "base/synchronization/lock.h"
 #include "base/trace_event/memory_dump_manager.h"
 #include "base/trace_event/process_memory_dump.h"
+#include "sql/meta_table.h"
 #include "sql/statement.h"
 #include "third_party/sqlite/sqlite3.h"
 
 #if defined(OS_IOS) && defined(USE_SYSTEM_SQLITE)
 #include "third_party/sqlite/src/ext/icu/sqliteicu.h"
 #endif
 
 namespace {
 
 // Spin for up to a second waiting for the lock to clear when setting
@@ skipping 414 matching lines @@
     int64_t size_64 = 0;
     if (base::GetFileSize(path, &size_64)) {
       size_t sample = static_cast<size_t>(size_64 / 1024);
       std::string full_histogram_name = "Sqlite.SizeKB." + histogram_tag_;
       base::HistogramBase* histogram =
           base::Histogram::FactoryGet(
               full_histogram_name, 1, 1000000, 50,
               base::HistogramBase::kUmaTargetedHistogramFlag);
       if (histogram)
         histogram->Add(sample);
+      UMA_HISTOGRAM_COUNTS("Sqlite.SizeKB", sample);

[Scott Hess - ex-Googler, 2015/12/16 23:09:08]

This is sneaking in because when I went to verify the 99th-percentile for file
size, I noticed it was missing.  That was not intended, I suspect FOWL PLAY.

     }
   }
 
   return OpenInternal(AsUTF8ForSQL(path), RETRY_ON_POISON);
 }
 
 bool Connection::OpenInMemory() {
   in_memory_ = true;
   return OpenInternal(":memory:", NO_RETRY);
 }
@@ skipping 405 matching lines @@
       base::StringAppendF(&debug_info, "%s\n", messages[i].c_str());
     }
   }
 
   return debug_info;
 }
 
 size_t Connection::GetAppropriateMmapSize() {
   AssertIOAllowed();
 
-  // TODO(shess): Using sql::MetaTable seems indicated, but mixing
-  // sql::MetaTable and direct access seems error-prone.  It might make sense to
-  // simply integrate sql::MetaTable functionality into sql::Connection.
-
 #if defined(OS_IOS)
   // iOS SQLite does not support memory mapping.
   return 0;
 #endif
 
-  // If the database doesn't have a place to track progress, assume the worst.
-  // This will happen when new databases are created.
-  if (!DoesTableExist("meta")) {
+  // How much to map if no errors are found.  50MB encompasses the 99th
+  // percentile of Chrome databases in the wild, so this should be good.
+  const size_t kMmapEverything = 256 * 1024 * 1024;
+
+  // If the database doesn't have a place to track progress, assume the best.
+  // This will happen when new databases are created, or if a database doesn't
+  // use a meta table.  sql::MetaTable::Init() will preload kMmapSuccess.
+  // TODO(shess): Databases not using meta include:
+  //   DOMStorageDatabase (localstorage)
+  //   ActivityDatabase (extensions activity log)
+  //   PredictorDatabase (prefetch and autocomplete predictor data)
+  //   SyncDirectory (sync metadata storage)
+  // For now, these all have mmap disabled to allow other databases to get the
+  // default-enable path.  sqlite-diag could be an alternative for all but
+  // DOMStorageDatabase, which creates many small databases.
+  // http://crbug.com/537742
+  if (!MetaTable::DoesTableExist(this)) {
     RecordOneEvent(EVENT_MMAP_META_MISSING);
+    return kMmapEverything;
+  }
+
+  int64_t mmap_ofs = 0;
+  if (!MetaTable::GetMmapStatus(this, &mmap_ofs)) {
+    RecordOneEvent(EVENT_MMAP_META_FAILURE_READ);
     return 0;
   }
 
-  // Key into meta table to get status from a previous run.  The value
-  // represents how much data in bytes has successfully been read from the
-  // database.  |kMmapFailure| indicates that there was a read error and the
-  // database should not be memory-mapped, while |kMmapSuccess| indicates that
-  // the entire file was read at some point and can be memory-mapped without
-  // constraint.
-  const char* kMmapStatusKey = "mmap_status";
-  static const sqlite3_int64 kMmapFailure = -2;
-  static const sqlite3_int64 kMmapSuccess = -1;
-
-  // Start reading from 0 unless status is found in meta table.
-  sqlite3_int64 mmap_ofs = 0;
-
-  // Retrieve the current status.  It is fine for the status to be missing
-  // entirely, but any error prevents memory-mapping.
-  {
-    const char* kMmapStatusSql = "SELECT value FROM meta WHERE key = ?";
-    Statement s(GetUniqueStatement(kMmapStatusSql));
-    s.BindString(0, kMmapStatusKey);
-    if (s.Step()) {
-      mmap_ofs = s.ColumnInt64(0);
-    } else if (!s.Succeeded()) {
-      RecordOneEvent(EVENT_MMAP_META_FAILURE_READ);
-      return 0;
-    }
-  }
-
   // Database read failed in the past, don't memory map.
-  if (mmap_ofs == kMmapFailure) {
+  if (mmap_ofs == MetaTable::kMmapFailure) {
     RecordOneEvent(EVENT_MMAP_FAILED);
     return 0;
-  } else if (mmap_ofs != kMmapSuccess) {
+  } else if (mmap_ofs != MetaTable::kMmapSuccess) {
     // Continue reading from previous offset.
     DCHECK_GE(mmap_ofs, 0);
 
     // TODO(shess): Could this reading code be shared with Preload()?  It would
     // require locking twice (this code wouldn't be able to access |db_size| so
     // the helper would have to return amount read).
 
     // Read more of the database looking for errors.  The VFS interface is used
     // to assure that the reads are valid for SQLite.  |g_reads_allowed| is used
     // to limit checking to 20MB per run of Chromium.
@@ skipping 30 matching lines @@
         int rc = file->pMethods->xRead(file, buf, sizeof(buf), mmap_ofs);
         if (rc == SQLITE_OK) {
           mmap_ofs += sizeof(buf);
           amount -= sizeof(buf);
         } else if (rc == SQLITE_IOERR_SHORT_READ) {
           // Reached EOF for a database with page size < |kPageSize|.
           mmap_ofs = db_size;
           break;
         } else {
           // TODO(shess): Consider calling OnSqliteError().
-          mmap_ofs = kMmapFailure;
+          mmap_ofs = MetaTable::kMmapFailure;
           break;
         }
       }
 
       // Log these events after update to distinguish meta update failure.
       Events event;
       if (mmap_ofs >= db_size) {
-        mmap_ofs = kMmapSuccess;
+        mmap_ofs = MetaTable::kMmapSuccess;
         event = EVENT_MMAP_SUCCESS_NEW;
       } else if (mmap_ofs > 0) {
         event = EVENT_MMAP_SUCCESS_PARTIAL;
       } else {
-        DCHECK_EQ(kMmapFailure, mmap_ofs);
+        DCHECK_EQ(MetaTable::kMmapFailure, mmap_ofs);
         event = EVENT_MMAP_FAILED_NEW;
       }
 
-      const char* kMmapUpdateStatusSql = "REPLACE INTO meta VALUES (?, ?)";
-      Statement s(GetUniqueStatement(kMmapUpdateStatusSql));
-      s.BindString(0, kMmapStatusKey);
-      s.BindInt64(1, mmap_ofs);
-      if (!s.Run()) {
+      if (!MetaTable::SetMmapStatus(this, mmap_ofs)) {
         RecordOneEvent(EVENT_MMAP_META_FAILURE_UPDATE);
         return 0;
       }
 
       RecordOneEvent(event);
     }
   }
 
-  if (mmap_ofs == kMmapFailure)
+  if (mmap_ofs == MetaTable::kMmapFailure)
     return 0;
-  if (mmap_ofs == kMmapSuccess)
-    return 256 * 1024 * 1024;
+  if (mmap_ofs == MetaTable::kMmapSuccess)
+    return kMmapEverything;
   return mmap_ofs;
 }
 
 void Connection::TrimMemory(bool aggressively) {
   if (!db_)
     return;
 
   // TODO(shess): investigate using sqlite3_db_release_memory() when possible.
   int original_cache_size;
   {
@@ skipping 965 matching lines @@
   ignore_result(Execute(kNoWritableSchema));
 
   return ret;
 }
 
 base::TimeTicks TimeSource::Now() {
   return base::TimeTicks::Now();
 }
 
 }  // namespace sql