Purge browser cache for dom storage in a smarter way

content/browser/dom_storage/dom_storage_namespace.cc

 // Copyright 2013 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "content/browser/dom_storage/dom_storage_namespace.h"
 
+#include <algorithm>
+
 #include "base/bind.h"
 #include "base/location.h"
 #include "base/logging.h"
+#include "base/sys_info.h"
 #include "content/browser/dom_storage/dom_storage_area.h"
 #include "content/browser/dom_storage/dom_storage_task_runner.h"
 #include "content/browser/dom_storage/session_storage_database.h"
 #include "content/common/dom_storage/dom_storage_types.h"
 
 namespace content {
 
 DOMStorageNamespace::DOMStorageNamespace(
     const base::FilePath& directory,
     DOMStorageTaskRunner* task_runner)
     : namespace_id_(kLocalStorageNamespaceId),
       directory_(directory),
       task_runner_(task_runner) {
 }
 
 DOMStorageNamespace::DOMStorageNamespace(
     int64_t namespace_id,
     const std::string& persistent_namespace_id,
     SessionStorageDatabase* session_storage_database,
     DOMStorageTaskRunner* task_runner)
     : namespace_id_(namespace_id),
       persistent_namespace_id_(persistent_namespace_id),
       task_runner_(task_runner),
-      session_storage_database_(session_storage_database) {
+      session_storage_database_(session_storage_database),
+      is_low_end_device_(base::SysInfo::IsLowEndDevice()) {
   DCHECK_NE(kLocalStorageNamespaceId, namespace_id);
 }
 
 DOMStorageNamespace::~DOMStorageNamespace() {
 }
 
 DOMStorageArea* DOMStorageNamespace::OpenStorageArea(const GURL& origin) {
   if (AreaHolder* holder = GetAreaHolder(origin)) {
     ++(holder->open_count_);
     return holder->area_.get();
@@ skipping 69 matching lines @@
 
 void DOMStorageNamespace::DeleteSessionStorageOrigin(const GURL& origin) {
   DOMStorageArea* area = OpenStorageArea(origin);
   area->FastClear();
   CloseStorageArea(area);
 }
 
 void DOMStorageNamespace::PurgeMemory(PurgeOption option) {
   if (directory_.empty())
     return;  // We can't purge w/o backing on disk.
+
+  option = std::max(option, FindPurgeLevel());
+  if (option == PURGE_UNSPECIFIED)
+    return;
+
   AreaMap::iterator it = areas_.begin();
   while (it != areas_.end()) {
     const AreaHolder& holder = it->second;
 
     // We can't purge if there are changes pending.
     if (holder.area_->HasUncommittedChanges()) {
       if (holder.open_count_ == 0) {
          // Schedule an immediate commit so the next time we're asked to purge,
          // we can drop it from memory.
          holder.area_->ScheduleImmediateCommit();
       }
-      ++it;
+
+      if (option == PURGE_AGGRESSIVE)
+        holder.area_->TrimDatabase();
+      it++;
       continue;
     }
 
     // If not in use, we can shut it down and remove
     // it from our collection entirely.
     if (holder.open_count_ == 0) {
       holder.area_->Shutdown();
       areas_.erase(it++);
       continue;
     }
 
     if (option == PURGE_AGGRESSIVE) {
       // If aggressive is true, we clear caches and such
       // for opened areas.
       holder.area_->PurgeMemory();
     }
-
-    ++it;
+    it++;
   }
 }
 
+// This function works based on the cache sizes without including the database
+// size since it can be expensive trying to estimate the sqlite usage for all
+// databases.
+DOMStorageNamespace::PurgeOption DOMStorageNamespace::FindPurgeLevel() {
+  // Maximum in-memory cache usage of all databases.
+  size_t in_memory_areas_size_budget = 20 * 1024 * 1024;
+  // Maximum in-memory cache usage of databases that are open.
+  size_t inactive_in_memory_area_size_budget = 20 * 1024 * 1024;
+  // Number of areas over which the open databases can be purged.
+  size_t active_in_memory_areas_unpurged = 100;
+#if defined(OS_ANDROID)

[michaeln, 2016/05/09 22:12:17]

This might be more complicated then it needs to be? What if we trust the the
base::MemPressureListener will tell us when we need to PURGE_AGGRESSIVE and in
normal operation PURGE_UNOPENED when mem_used > kMemLimit || num_areas >
kAreaLimit.

We're working on a new impl of DomStorage and this logic will eventually want to
be ported over to the new impl too. The simpler it is, the simpler that will be.

[ssid, 2016/05/10 02:04:16]

I agree this is very complicated. I was expecting better suggestions on what is
useful.
I have added just 2 cases now. One more than what you suggested, just for low
end device. wdyt?

+  if (is_low_end_device_) {
+    // For low-end devices, when the cache size reaches 5Mib, only 10 open
+    // databases are allowed before being purged.
+    active_in_memory_areas_unpurged = 10;
+    in_memory_areas_size_budget = 5 * 1024 * 1024;
+    // Clear most of the inactive database on low-end device.
+    inactive_in_memory_area_size_budget = 100 * 1024;
+  } else {
+    // Set a low cache budget for inactive database cache on android.
+    inactive_in_memory_area_size_budget = 1024 * 1024;

[ssid, 2016/05/06 03:28:51]

I chose these values based on what sizes I observed from the UMA metric
LocalStorage.BrowserLocalStorageSizeInKB and Sqlite.SizeKB.DOMStorageDatabase.
But it is still unclear how many databases does a single tab uses, or the total
number of databases in memory.

[michaeln, 2016/05/09 22:12:17]

we don't have stats for the number of 'area in use', that might be nice to know,
also how many are inuse by multiple renderers at the same time

+  }
+#endif
+
+  size_t total_cache_size = 0;
+  size_t area_count = 0;
+  size_t active_areas_cache_size = 0;
+  unsigned active_areas_count = 0;
+  for (AreaMap::const_iterator it = areas_.begin(); it != areas_.end(); ++it) {
+    if (it->second.area_->IsLoadedInMemory()) {
+      size_t cache_size = it->second.area_->map_usage_in_bytes();

[ssid, 2016/05/06 03:28:52]

Do you think just the cache size is a good estimate of the memory usage?
From what I understand the cache is always bigger than the sql databases, since
the cache is not cleared but the databases get committed to file.

[michaeln, 2016/05/09 22:12:17]

Sure, this is what we have to work with.

+      total_cache_size += cache_size;
+      ++area_count;
+      if (it->second.open_count_ > 0) {
+        active_areas_cache_size += cache_size;
+        ++active_areas_count;
+      }
+    }
+  }
+
+  if (active_areas_cache_size > in_memory_areas_size_budget &&
+      active_areas_count > active_in_memory_areas_unpurged) {
+    return PURGE_AGGRESSIVE;
+  }
+
+  size_t purgeable_cache_size = total_cache_size - active_areas_cache_size;
+  if (purgeable_cache_size) {
+    if (purgeable_cache_size > inactive_in_memory_area_size_budget ||
+        total_cache_size > in_memory_areas_size_budget) {
+      return PURGE_UNOPENED;
+    }
+  }
+  return PURGE_UNSPECIFIED;
+}
+
 void DOMStorageNamespace::Shutdown() {
   AreaMap::const_iterator it = areas_.begin();
   for (; it != areas_.end(); ++it)
     it->second.area_->Shutdown();
 }
 
 void DOMStorageNamespace::Flush() {
   for (auto& entry : areas_) {
     if (!entry.second.area_->HasUncommittedChanges())
       continue;
     entry.second.area_->ScheduleImmediateCommit();
   }
 }
 
-unsigned int DOMStorageNamespace::CountInMemoryAreas() const {
-  unsigned int area_count = 0;
-  for (AreaMap::const_iterator it = areas_.begin(); it != areas_.end(); ++it) {
-    if (it->second.area_->IsLoadedInMemory())
-      ++area_count;
-  }
-  return area_count;
-}
-
 void DOMStorageNamespace::OnMemoryDump(
     base::trace_event::ProcessMemoryDump* pmd) {
   DCHECK(task_runner_->IsRunningOnPrimarySequence());
   for (const auto& it : areas_)
     it.second.area_->OnMemoryDump(pmd);
 }
 
 DOMStorageNamespace::AreaHolder*
 DOMStorageNamespace::GetAreaHolder(const GURL& origin) {
   AreaMap::iterator found = areas_.find(origin);
@@ skipping 12 matching lines @@
     DOMStorageArea* area, int count)
     : area_(area), open_count_(count) {
 }
 
 DOMStorageNamespace::AreaHolder::AreaHolder(const AreaHolder& other) = default;
 
 DOMStorageNamespace::AreaHolder::~AreaHolder() {
 }
 
 }  // namespace content