Quota: Add metrics about the age and amount of data stored in temporary storage, and a metric about…

storage/browser/quota/quota_manager.cc

 // Copyright 2013 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "storage/browser/quota/quota_manager.h"
 
 #include <stddef.h>
 #include <stdint.h>
 
 #include <algorithm>
@@ skipping 313 matching lines @@
   int64_t quota = usage_and_quota.quota;
 
   if (type == kStorageTypeTemporary && !is_unlimited) {
     quota = CalculateTemporaryHostQuota(
         usage, quota, usage_and_quota.global_limited_usage);
   }
 
   if (is_incognito) {
     quota = std::min(quota, QuotaManager::kIncognitoDefaultQuotaLimit);
     callback.Run(status, usage, quota);
+    UMA_HISTOGRAM_MBYTES("Quota.QuotaForOriginIncognito", quota);

[Ilya Sherman, 2016/05/19 04:06:01]

It's worth double-checking with the privacy team, but I don't think it's
generally accepted to record metrics specifically about incognito usage.  It
kind of goes against the grain of the philosophy of incognito mode.

[michaeln, 2016/05/19 19:09:15]

hmmm... i see many metrics that are specific to 'incognito'. to my mind,
history,cookies,cache, info attributable to specific sites are verboten. the
fact that incognito was invoked to view a page is not verboten

i could remove this for now because i'm interested in getting the main stream
metrics in prior to the branch being cut and bring this one up with the privacy
folks in a sperarte cl if need be... but it doesn't seem like it needs to be
excluded?

[Ilya Sherman, 2016/05/19 22:31:16]

I'd recommend removing this for now, and checking with the privacy team.  We
often take a fairly conservative approach with Incognito mode, because
subverting some users' expectations is worse than providing some users more
privacy than they expected.  But, it's something of an art to find the precise
balance, and I'll defer to the privacy team on the current best practices.

[michaeln, 2016/05/19 22:56:30]

Done.

     return;
   }
 
   // For apps with unlimited permission or can_query_disk_size is true (and not
   // in incognito mode).
   // We assume we can expose the actual disk size for them and cap the quota by
   // the available disk space.
   if (is_unlimited || can_query_disk_size) {
-    callback.Run(
-        status, usage,
-        CalculateQuotaWithDiskSpace(
-            usage_and_quota.available_disk_space,
-            usage, quota));
-    return;
+    quota = CalculateQuotaWithDiskSpace(
+        usage_and_quota.available_disk_space,
+        usage, quota);
   }
 
   callback.Run(status, usage, quota);
+
+  if (type == kStorageTypeTemporary && !is_unlimited)
+    UMA_HISTOGRAM_MBYTES("Quota.QuotaForOrigin", quota);
 }
 
 }  // namespace
 
 UsageAndQuota::UsageAndQuota()
     : usage(0),
       global_limited_usage(0),
       quota(0),
       available_disk_space(0) {
 }
@@ skipping 1109 matching lines @@
     return;
   }
 
   DCHECK(origin == origin.GetOrigin());
   OriginDataDeleter* deleter = new OriginDataDeleter(
       this, origin, type, quota_client_mask, is_eviction, callback);
   deleter->Start();
 }
 
 void QuotaManager::ReportHistogram() {
+  DCHECK(!is_incognito_);
   GetGlobalUsage(kStorageTypeTemporary,
                  base::Bind(
                      &QuotaManager::DidGetTemporaryGlobalUsageForHistogram,
                      weak_factory_.GetWeakPtr()));
-  GetGlobalUsage(kStorageTypePersistent,
-                 base::Bind(
-                     &QuotaManager::DidGetPersistentGlobalUsageForHistogram,
-                     weak_factory_.GetWeakPtr()));
 }
 
 void QuotaManager::DidGetTemporaryGlobalUsageForHistogram(
     int64_t usage,
     int64_t unlimited_usage) {
   UMA_HISTOGRAM_MBYTES("Quota.GlobalUsageOfTemporaryStorage", usage);
 
   std::set<GURL> origins;
   GetCachedOrigins(kStorageTypeTemporary, &origins);
 
   size_t num_origins = origins.size();
   size_t protected_origins = 0;
   size_t unlimited_origins = 0;
   CountOriginType(origins,
                   special_storage_policy_.get(),
                   &protected_origins,
                   &unlimited_origins);
-
   UMA_HISTOGRAM_COUNTS("Quota.NumberOfTemporaryStorageOrigins",
                        num_origins);
   UMA_HISTOGRAM_COUNTS("Quota.NumberOfProtectedTemporaryStorageOrigins",
                        protected_origins);
   UMA_HISTOGRAM_COUNTS("Quota.NumberOfUnlimitedTemporaryStorageOrigins",
                        unlimited_origins);
+
+  GetGlobalUsage(kStorageTypePersistent,
+                 base::Bind(
+                     &QuotaManager::DidGetPersistentGlobalUsageForHistogram,
+                     weak_factory_.GetWeakPtr()));
 }
 
 void QuotaManager::DidGetPersistentGlobalUsageForHistogram(
     int64_t usage,
     int64_t unlimited_usage) {
   UMA_HISTOGRAM_MBYTES("Quota.GlobalUsageOfPersistentStorage", usage);
 
   std::set<GURL> origins;
   GetCachedOrigins(kStorageTypePersistent, &origins);
 
   size_t num_origins = origins.size();
   size_t protected_origins = 0;
   size_t unlimited_origins = 0;
   CountOriginType(origins,
                   special_storage_policy_.get(),
                   &protected_origins,
                   &unlimited_origins);
-
   UMA_HISTOGRAM_COUNTS("Quota.NumberOfPersistentStorageOrigins",
                        num_origins);
   UMA_HISTOGRAM_COUNTS("Quota.NumberOfProtectedPersistentStorageOrigins",
                        protected_origins);
   UMA_HISTOGRAM_COUNTS("Quota.NumberOfUnlimitedPersistentStorageOrigins",
                        unlimited_origins);
+
+  // We DumpOriginInfoTable last to ensure the trackers caches are loaded.
+  DumpOriginInfoTable(
+      base::Bind(&QuotaManager::DidDumpOriginInfoTableForHistogram,
+                 weak_factory_.GetWeakPtr()));
+}
+
+void QuotaManager::DidDumpOriginInfoTableForHistogram(
+    const OriginInfoTableEntries& entries) {
+  using UsageMap = std::map<GURL, int64_t>;
+  UsageMap usage_map;
+  GetUsageTracker(kStorageTypeTemporary)->GetCachedOriginsUsage(&usage_map);
+
+  int64_t usage_day_old = 0;
+  int64_t usage_week_old = 0;
+  int64_t usage_month_old = 0;
+  int64_t usage_three_month_old = 0;
+  int64_t usage_six_month_old = 0;
+  int64_t usage_year_old = 0;
+  int64_t usage_over_year_old = 0;
+  base::Time now = base::Time::Now();
+
+  for (const auto& info : entries) {
+    if (info.type != kStorageTypeTemporary)
+      continue;
+
+    // Ignore stale database entries. If there is no map entry, the origin's
+    // data has been deleted.
+    UsageMap::const_iterator found = usage_map.find(info.origin);
+    if (found == usage_map.end() || found->second == 0)
+      continue;
+
+    base::TimeDelta age = now - std::max(info.last_access_time,
+                                         info.last_modified_time);
+    UMA_HISTOGRAM_COUNTS_1000("Quota.OriginAgeInDays", age.InDays());
+
+    if (age < base::TimeDelta::FromDays(1))
+      usage_day_old += found->second;
+    else if (age < base::TimeDelta::FromDays(7))
+      usage_week_old += found->second;
+    else if (age < base::TimeDelta::FromDays(30))
+      usage_month_old += found->second;
+    else if (age < base::TimeDelta::FromDays(90))
+      usage_three_month_old += found->second;
+    else if (age < base::TimeDelta::FromDays(180))
+      usage_six_month_old += found->second;
+    else if (age < base::TimeDelta::FromDays(365))
+      usage_year_old += found->second;
+    else
+      usage_over_year_old += found->second;
+  }
+
+  UMA_HISTOGRAM_MBYTES("Quota.AmountDayOldData", usage_day_old);
+  UMA_HISTOGRAM_MBYTES("Quota.AmountWeekOldData", usage_week_old);
+  UMA_HISTOGRAM_MBYTES("Quota.AmountMonthOldData", usage_month_old);
+  UMA_HISTOGRAM_MBYTES("Quota.AmountThreeMonthOldData", usage_three_month_old);
+  UMA_HISTOGRAM_MBYTES("Quota.AmountSixMonthOldData", usage_six_month_old);
+  UMA_HISTOGRAM_MBYTES("Quota.AmountYearOldData", usage_year_old);
+  UMA_HISTOGRAM_MBYTES("Quota.AmountOverYearOldData", usage_over_year_old);

[Ilya Sherman, 2016/05/19 04:06:01]

This is a fairly significant number of similar histograms.  It might be okay,
but are you sure that you need quite so much granularity?  How do you plan to
use this data?

[michaeln, 2016/05/19 19:09:15]

Two reasons for the granularity.

- to get a sense of kind of impact "evict everything <xxx> old" would have

- to help evaluate if future changes we make to the quota system are hurting or
helping, larger amounts of fresher data being preferable.

[michaeln, 2016/05/19 19:51:24]

Oh... I'll redo these to use a single histogram with a bucket for each day and
add a sample for each KB of data that is <n> days old.

I don't think there's a macro to add multiple samples to a bucket, but i should
be able to call AddCount() directly.

The OriginAgeInDays hsito counts number of origins. This one will will count
"AgeOfMBInDays".

[Ilya Sherman, 2016/05/19 22:31:16]

Yep, that's a reasonable approach, as long as you're not needing to see the
distribution of memory usage within each bucket.  Just be aware that outliers
can significantly influence the data.

[michaeln, 2016/05/19 22:56:30]

Done

 }
 
 std::set<GURL> QuotaManager::GetEvictionOriginExceptions(
     const std::set<GURL>& extra_exceptions) {
   std::set<GURL> exceptions = extra_exceptions;
   for (const auto& p : origins_in_use_) {
     if (p.second > 0)
       exceptions.insert(p.first);
   }
 
@@ skipping 163 matching lines @@
 }
 
 void QuotaManager::DidInitialize(int64_t* temporary_quota_override,
                                  int64_t* desired_available_space,
                                  bool success) {
   temporary_quota_override_ = *temporary_quota_override;
   desired_available_space_ = *desired_available_space;
   temporary_quota_initialized_ = true;
   DidDatabaseWork(success);
 
-  histogram_timer_.Start(FROM_HERE,
-                         base::TimeDelta::FromMilliseconds(
-                             kReportHistogramInterval),
-                         this, &QuotaManager::ReportHistogram);
+  if (!is_incognito_) {
+    histogram_timer_.Start(FROM_HERE,
+                           base::TimeDelta::FromMilliseconds(
+                               kReportHistogramInterval),
+                           this, &QuotaManager::ReportHistogram);
+  }
 
   db_initialization_callbacks_.Run();
   GetTemporaryGlobalQuota(
       base::Bind(&QuotaManager::DidGetInitialTemporaryGlobalQuota,
                  weak_factory_.GetWeakPtr(), base::TimeTicks::Now()));
 }
 
 void QuotaManager::DidGetLRUOrigin(const GURL* origin,
                                    bool success) {
   DidDatabaseWork(success);
@@ skipping 102 matching lines @@
     return false;
   *available_space = static_cast<uint64_t>(stats.f_bavail) * stats.f_frsize;
   *total_size = static_cast<uint64_t>(stats.f_blocks) * stats.f_frsize;
 #else
 #error Not implemented
 #endif
   return true;
 }
 
 }  // namespace storage