Support for process-wide incidents in the safe browsing incident reporting service.

chrome/browser/safe_browsing/incident_reporting_service.cc

 // Copyright 2014 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "chrome/browser/safe_browsing/incident_reporting_service.h"
 
 #include <math.h>
 
 #include <algorithm>
 #include <vector>
 
 #include "base/metrics/histogram.h"
 #include "base/prefs/pref_service.h"
 #include "base/prefs/scoped_user_pref_update.h"
 #include "base/process/process_info.h"
+#include "base/single_thread_task_runner.h"
 #include "base/stl_util.h"
 #include "base/strings/string_number_conversions.h"
+#include "base/thread_task_runner_handle.h"
 #include "base/threading/sequenced_worker_pool.h"
 #include "base/values.h"
 #include "chrome/browser/browser_process.h"
 #include "chrome/browser/chrome_notification_types.h"
 #include "chrome/browser/prefs/tracked/tracked_preference_validation_delegate.h"
 #include "chrome/browser/profiles/profile.h"
 #include "chrome/browser/safe_browsing/database_manager.h"
 #include "chrome/browser/safe_browsing/environment_data_collection.h"
 #include "chrome/browser/safe_browsing/incident_report_uploader_impl.h"
 #include "chrome/browser/safe_browsing/preference_validation_delegate.h"
@@ skipping 35 matching lines @@
   // The key for a specific instance of an incident.
   std::string key;
 
   // A hash digest representing a specific instance of an incident.
   uint32_t digest;
 };
 
 // The amount of time the service will wait to collate incidents.
 const int64 kDefaultUploadDelayMs = 1000 * 60;  // one minute
 
+// The amount of time between running delayed analysis callbacks.
+const int64 kDefaultCallbackMs = 1000 * 20;

[robertshield, 2014/08/05 21:26:28]

how about kDefaultCallbackIntervalMs ?

[grt (UTC plus 2), 2014/08/06 02:37:31]

Done.

+
 // Returns the number of incidents contained in |incident|. The result is
 // expected to be 1. Used in DCHECKs.
 size_t CountIncidents(const ClientIncidentReport_IncidentData& incident) {
   size_t result = 0;
   if (incident.has_tracked_preference())
     ++result;
   // Add detection for new incident types here.
   return result;
 }
 
@@ skipping 67 matching lines @@
     if (!incidents_sent->GetDictionaryWithoutPathExpansion(type_string,
                                                            &type_dict)) {
       type_dict = new base::DictionaryValue();
       incidents_sent->SetWithoutPathExpansion(type_string, type_dict);
     }
     type_dict->SetStringWithoutPathExpansion(data.key,
                                              base::UintToString(data.digest));
   }
 }
 
+// Runs |callback| on the thread to which |thread_runner| belongs. The callback
+// is run immediately if this function is called on |thread_runner|'s thread.
+void AddIncidentOnOriginThread(
+    const AddIncidentCallback& callback,
+    scoped_refptr<base::SingleThreadTaskRunner> thread_runner,
+    scoped_ptr<ClientIncidentReport_IncidentData> incident) {
+  if (thread_runner->BelongsToCurrentThread())
+    callback.Run(incident.Pass());
+  else
+    thread_runner->PostTask(FROM_HERE,
+                            base::Bind(callback, base::Passed(&incident)));
+}
+
 }  // namespace
 
 struct IncidentReportingService::ProfileContext {
   ProfileContext();
   ~ProfileContext();
 
   // The incidents collected for this profile pending creation and/or upload.
   ScopedVector<ClientIncidentReport_IncidentData> incidents;
 
   // False until PROFILE_ADDED notification is received.
@@ skipping 43 matching lines @@
     const scoped_refptr<net::URLRequestContextGetter>& request_context_getter)
     : database_manager_(safe_browsing_service ?
                         safe_browsing_service->database_manager() : NULL),
       url_request_context_getter_(request_context_getter),
       collect_environment_data_fn_(&CollectEnvironmentData),
       environment_collection_task_runner_(
           content::BrowserThread::GetBlockingPool()
               ->GetTaskRunnerWithShutdownBehavior(
                   base::SequencedWorkerPool::SKIP_ON_SHUTDOWN)),
       environment_collection_pending_(),
-      collection_timeout_pending_(),
-      upload_timer_(FROM_HERE,
-                    base::TimeDelta::FromMilliseconds(kDefaultUploadDelayMs),
-                    this,
-                    &IncidentReportingService::OnCollectionTimeout),
+      collation_timeout_pending_(),
+      collation_timer_(FROM_HERE,
+                       base::TimeDelta::FromMilliseconds(kDefaultUploadDelayMs),
+                       this,
+                       &IncidentReportingService::OnCollationTimeout),
+      delayed_analysis_callbacks_(
+          base::TimeDelta::FromMilliseconds(kDefaultCallbackMs),
+          content::BrowserThread::GetBlockingPool()),
       receiver_weak_ptr_factory_(this),
       weak_ptr_factory_(this) {
   notification_registrar_.Add(this,
                               chrome::NOTIFICATION_PROFILE_ADDED,
                               content::NotificationService::AllSources());
   notification_registrar_.Add(this,
                               chrome::NOTIFICATION_PROFILE_DESTROYED,
                               content::NotificationService::AllSources());
 }
 
@@ skipping 25 matching lines @@
 scoped_ptr<TrackedPreferenceValidationDelegate>
 IncidentReportingService::CreatePreferenceValidationDelegate(Profile* profile) {
   DCHECK(thread_checker_.CalledOnValidThread());
 
   if (profile->IsOffTheRecord())
     return scoped_ptr<TrackedPreferenceValidationDelegate>();
   return scoped_ptr<TrackedPreferenceValidationDelegate>(
       new PreferenceValidationDelegate(GetAddIncidentCallback(profile)));
 }
 
+void IncidentReportingService::RegisterDelayedAnalysisCallback(
+    const DelayedAnalysisCallback& callback) {
+  DCHECK(thread_checker_.CalledOnValidThread());
+
+  // |callback| will be run on the blocking pool, so it will likely run the
+  // AddIncidentCallback there as well. Bounce the run of that callback back to
+  // the main thread via AddIncidentOnOriginThread.

[robertshield, 2014/08/05 21:26:28]

s/main/current ?

[grt (UTC plus 2), 2014/08/06 02:37:31]

Done.

+  delayed_analysis_callbacks_.RegisterCallback(
+      base::Bind(callback,
+                 base::Bind(&AddIncidentOnOriginThread,
+                            GetAddIncidentCallback(NULL),
+                            base::ThreadTaskRunnerHandle::Get())));
+
+  // Start running the callbacks if any profiles are participating in safe
+  // browsing. If none are now, running will commence if/when a participaing
+  // profile is added.
+  if (FindEligibleProfile())
+    delayed_analysis_callbacks_.Start();
+}
+
 void IncidentReportingService::SetCollectEnvironmentHook(
     CollectEnvironmentDataFn collect_environment_data_hook,
     const scoped_refptr<base::TaskRunner>& task_runner) {
   if (collect_environment_data_hook) {
     collect_environment_data_fn_ = collect_environment_data_hook;
     environment_collection_task_runner_ = task_runner;
   } else {
     collect_environment_data_fn_ = &CollectEnvironmentData;
     environment_collection_task_runner_ =
         content::BrowserThread::GetBlockingPool()
             ->GetTaskRunnerWithShutdownBehavior(
                 base::SequencedWorkerPool::SKIP_ON_SHUTDOWN);
   }
 }
 
 void IncidentReportingService::OnProfileAdded(Profile* profile) {
   DCHECK(thread_checker_.CalledOnValidThread());
 
   // Track the addition of all profiles even when no report is being assembled
   // so that the service can determine whether or not it can evaluate a
   // profile's preferences at the time of incident addition.
   ProfileContext* context = GetOrCreateProfileContext(profile);
   context->added = true;
 
+  const bool safe_browsing_enabled =
+      profile->GetPrefs()->GetBoolean(prefs::kSafeBrowsingEnabled);
+
+  // Start processing delayed analysis callbacks if this new profile
+  // participates in safe browsing.

[robertshield, 2014/08/05 21:26:28]

Re-mention here how Start() is idempotent for repeated calls?

[grt (UTC plus 2), 2014/08/06 02:37:31]

Done.

+  if (safe_browsing_enabled)
+    delayed_analysis_callbacks_.Start();
+
+  // Start a new report if this profile participates in safe browsing and there
+  // are process-wide incidents.
+  if (safe_browsing_enabled && GetProfileContext(NULL))
+    BeginReportProcessing();
+
+  // TODO(grt): register for pref change notifications to start delayed analysis
+  // and/or report processing if sb is currently disabled but subsequently
+  // enabled.
+
   // Nothing else to do if a report is not being assembled.
   if (!report_)
     return;
 
-  // Drop all incidents received prior to creation if the profile is not
-  // participating in safe browsing.
-  if (!context->incidents.empty() &&
-      !profile->GetPrefs()->GetBoolean(prefs::kSafeBrowsingEnabled)) {
-    for (size_t i = 0; i < context->incidents.size(); ++i) {
+  // Drop all incidents associated with this profile that were received prior to
+  // its addition if the profile is not participating in safe browsing.
+  if (!context->incidents.empty() && !safe_browsing_enabled) {
+    for (size_t i = 0; i < context->incidents.size(); ++i)
       LogIncidentDataType(DROPPED, *context->incidents[i]);
-    }
     context->incidents.clear();
   }
 
   // Take another stab at finding the most recent download if a report is being
   // assembled and one hasn't been found yet (the LastDownloadFinder operates
   // only on profiles that have been added to the ProfileManager).
   BeginDownloadCollection();
 }
 
 scoped_ptr<LastDownloadFinder> IncidentReportingService::CreateDownloadFinder(
@@ skipping 37 matching lines @@
   // Forget about this profile. Incidents not yet sent for upload are lost.
   // No new incidents will be accepted for it.
   delete it->second;
   profiles_.erase(it);
 
   // Remove the association with this profile from all pending uploads.
   for (size_t i = 0; i < uploads_.size(); ++i)
     uploads_[i]->profiles_to_state.erase(profile);
 }
 
+Profile* IncidentReportingService::FindEligibleProfile() const {
+  Profile* candidate = NULL;
+  for (ProfileContextCollection::const_iterator scan = profiles_.begin();
+       scan != profiles_.end();
+       ++scan) {
+    // Skip over profiles that have yet to be added to the profile manager.
+    // This will also skip over the NULL-profile context used to hold
+    // process-wide incidents.
+    if (!scan->second->added)
+      continue;
+    PrefService* prefs = scan->first->GetPrefs();
+    if (prefs->GetBoolean(prefs::kSafeBrowsingExtendedReportingEnabled)) {

[mattm, 2014/08/06 02:10:17]

Need to check prefs::kSafeBrowsingEnabled too (extended reporting flag could be
set while safe browsing is actually disabled)

[grt (UTC plus 2), 2014/08/06 02:37:31]

Done.

+      candidate = scan->first;
+      break;
+    }
+    if (!candidate && prefs->GetBoolean(prefs::kSafeBrowsingEnabled))
+      candidate = scan->first;
+  }
+  return candidate;
+}
+
 void IncidentReportingService::AddIncident(
     Profile* profile,
     scoped_ptr<ClientIncidentReport_IncidentData> incident_data) {
   DCHECK(thread_checker_.CalledOnValidThread());
-  // Incidents outside the context of a profile are not supported at the moment.
-  DCHECK(profile);
   DCHECK_EQ(1U, CountIncidents(*incident_data));
 
   ProfileContext* context = GetProfileContext(profile);
   // It is forbidden to call this function with a destroyed profile.
   DCHECK(context);
+  // If this is a process-wide incident, the context must not indicate that the
+  // profile (which is NULL) has been added to the profile manager.
+  DCHECK(profile || !context->added);
 
-  // Drop the incident immediately if profile creation has completed and the
-  // profile is not participating in safe browsing. Preference evaluation is
-  // deferred until OnProfileAdded() if profile creation has not yet
-  // completed.
+  // Drop the incident immediately if the profile has already been added to the
+  // manager and is not participating in safe browsing. Preference evaluation is
+  // deferred until OnProfileAdded() otherwise.
   if (context->added &&
       !profile->GetPrefs()->GetBoolean(prefs::kSafeBrowsingEnabled)) {
     LogIncidentDataType(DROPPED, *incident_data);
     return;
   }
 
-  // Start assembling a new report if this is the first incident ever or the
-  // first since the last upload.
-  if (!report_) {
-    report_.reset(new ClientIncidentReport());
-    first_incident_time_ = base::Time::Now();
-  }
-
   // Provide time to the new incident if the caller didn't do so.
   if (!incident_data->has_incident_time_msec())
     incident_data->set_incident_time_msec(base::Time::Now().ToJavaTime());
 
   // Take ownership of the incident.
   context->incidents.push_back(incident_data.release());
 
+  // Remember when the first incident for this report arrived.
+  if (first_incident_time_.is_null())
+    first_incident_time_ = base::Time::Now();
+  // Log the time between the previous incident and this one.
   if (!last_incident_time_.is_null()) {
     UMA_HISTOGRAM_TIMES("SBIRS.InterIncidentTime",
                         base::TimeTicks::Now() - last_incident_time_);
   }
   last_incident_time_ = base::TimeTicks::Now();
 
   // Persist the incident data.
 
-  // Restart the delay timer to send the report upon expiration.
-  collection_timeout_pending_ = true;
-  upload_timer_.Reset();
+  // Start assembling a new report if this is the first incident ever or the
+  // first since the last upload.
+  BeginReportProcessing();
+}
 
+void IncidentReportingService::BeginReportProcessing() {
+  DCHECK(thread_checker_.CalledOnValidThread());
+
+  // Createa a new report if needed.

[mattm, 2014/08/06 02:10:17]

Createa

[grt (UTC plus 2), 2014/08/06 02:37:31]

Done.

+  if (!report_)
+    report_.reset(new ClientIncidentReport());
+
+  // Ensure that collection tasks are running (calls are idempotent).
+  BeginIncidentCollation();
   BeginEnvironmentCollection();
   BeginDownloadCollection();
 }
 
+void IncidentReportingService::BeginIncidentCollation() {
+  // Restart the delay timer to send the report upon expiration.
+  collation_timeout_pending_ = true;
+  collation_timer_.Reset();
+}
+
 void IncidentReportingService::BeginEnvironmentCollection() {
   DCHECK(thread_checker_.CalledOnValidThread());
   DCHECK(report_);
   // Nothing to do if environment collection is pending or has already
   // completed.
   if (environment_collection_pending_ || report_->has_environment())
     return;
 
   environment_collection_begin_ = base::TimeTicks::Now();
   ClientIncidentReport_EnvironmentData* environment_data =
@@ skipping 28 matching lines @@
   DCHECK(environment_collection_pending_);
   DCHECK(report_ && !report_->has_environment());
   environment_collection_pending_ = false;
 
 // CurrentProcessInfo::CreationTime() is missing on some platforms.
 #if defined(OS_MACOSX) || defined(OS_WIN) || defined(OS_LINUX)
   base::TimeDelta uptime =
       first_incident_time_ - base::CurrentProcessInfo::CreationTime();
   environment_data->mutable_process()->set_uptime_msec(uptime.InMilliseconds());
 #endif
-  first_incident_time_ = base::Time();
 
   report_->set_allocated_environment(environment_data.release());
 
   UMA_HISTOGRAM_TIMES("SBIRS.EnvCollectionTime",
                       base::TimeTicks::Now() - environment_collection_begin_);
   environment_collection_begin_ = base::TimeTicks();
 
   UploadIfCollectionComplete();
 }
 
 bool IncidentReportingService::WaitingToCollateIncidents() {
-  return collection_timeout_pending_;
+  return collation_timeout_pending_;
 }
 
 void IncidentReportingService::CancelIncidentCollection() {
-  collection_timeout_pending_ = false;
+  collation_timeout_pending_ = false;
   last_incident_time_ = base::TimeTicks();
   report_.reset();
 }
 
-void IncidentReportingService::OnCollectionTimeout() {
+void IncidentReportingService::OnCollationTimeout() {
   DCHECK(thread_checker_.CalledOnValidThread());
 
   // Exit early if collection was cancelled.
-  if (!collection_timeout_pending_)
+  if (!collation_timeout_pending_)
     return;
 
-  // Wait another round if incidents have come in from a profile that has yet to
-  // complete creation.
+  // Wait another round if profile-bound incidents have come in from a profile
+  // that has yet to complete creation.
   for (ProfileContextCollection::iterator scan = profiles_.begin();
        scan != profiles_.end();
        ++scan) {
-    if (!scan->second->added && !scan->second->incidents.empty()) {
-      upload_timer_.Reset();
+    if (scan->first && !scan->second->added &&
+        !scan->second->incidents.empty()) {
+      collation_timer_.Reset();
       return;
     }
   }
 
-  collection_timeout_pending_ = false;
+  collation_timeout_pending_ = false;
 
   UploadIfCollectionComplete();
 }
 
 void IncidentReportingService::BeginDownloadCollection() {
   DCHECK(thread_checker_.CalledOnValidThread());
   DCHECK(report_);
   // Nothing to do if a search for the most recent download is already pending
   // or if one has already been found.
   if (last_download_finder_ || report_->has_download())
@@ skipping 59 matching lines @@
   // Bail out if there are still outstanding collection tasks. Completion of any
   // of these will start another upload attempt.
   if (WaitingForEnvironmentCollection() ||
       WaitingToCollateIncidents() ||
       WaitingForMostRecentDownload()) {
     return;
   }
 
   // Take ownership of the report and clear things for future reports.
   scoped_ptr<ClientIncidentReport> report(report_.Pass());
+  first_incident_time_ = base::Time();
   last_incident_time_ = base::TimeTicks();
 
   // Drop the report if no executable download was found.
   if (!report->has_download()) {
     UMA_HISTOGRAM_ENUMERATION("SBIRS.UploadResult",
                               IncidentReportUploader::UPLOAD_NO_DOWNLOAD,
                               IncidentReportUploader::NUM_UPLOAD_RESULTS);
     return;
   }
 
   ClientIncidentReport_EnvironmentData_Process* process =
       report->mutable_environment()->mutable_process();
 
   // Not all platforms have a metrics reporting preference.
   if (g_browser_process->local_state()->FindPreference(
           prefs::kMetricsReportingEnabled)) {
     process->set_metrics_consent(g_browser_process->local_state()->GetBoolean(
         prefs::kMetricsReportingEnabled));
   }
 
-  // Check for extended consent in any profile while collecting incidents.
-  process->set_extended_consent(false);
+  // Find a profile suitable for tracking process-wide incidents.
+  Profile* analyses_profile = FindEligibleProfile();

[mattm, 2014/08/06 02:10:17]

is "analyses" intentional here? (vs analysis)

[grt (UTC plus 2), 2014/08/06 02:37:31]

Yeah, it's the profile used for the plural analyses. Kinda confusing, huh. I've
tweaked the names and comments. What do you think?

+  process->set_extended_consent(
+      analyses_profile ? analyses_profile->GetPrefs()->GetBoolean(
+                             prefs::kSafeBrowsingExtendedReportingEnabled) :
+                       false);
+
+  // Associate process-wide incidents with the analyses profile.
+  ProfileContext* null_context = GetProfileContext(NULL);
+  if (null_context && analyses_profile) {
+    DCHECK(!null_context->incidents.empty());
+    ProfileContext* analyses_context = GetProfileContext(analyses_profile);
+    // Move the incidents to the target context.
+    analyses_context->incidents.insert(analyses_context->incidents.end(),
+                                       null_context->incidents.begin(),
+                                       null_context->incidents.end());
+    null_context->incidents.weak_clear();
+    // Delete the process-wide context.
+    delete null_context;
+    profiles_.erase(NULL);

[mattm, 2014/08/06 02:10:17]

Does this mean no more process-wide incidents can be added after this?

[grt (UTC plus 2), 2014/08/06 02:37:31]

Nice catch. Any added after this should go into a new report. The incidents in
the context should be cleared, but the context must live.

+  }
+
   // Collect incidents across all profiles participating in safe browsing. Drop
   // incidents if the profile stopped participating before collection completed.
   // Prune previously submitted incidents.
   // Associate the profiles and their incident data with the upload.
   size_t prune_count = 0;
   UploadContext::PersistentIncidentStateCollection profiles_to_state;
   for (ProfileContextCollection::iterator scan = profiles_.begin();
        scan != profiles_.end();
        ++scan) {
+    // Bypass process-wide incidents that have not yet been associated with a
+    // profile.
+    if (!scan->first)
+      continue;
     PrefService* prefs = scan->first->GetPrefs();
-    if (process &&
-        prefs->GetBoolean(prefs::kSafeBrowsingExtendedReportingEnabled)) {
-      process->set_extended_consent(true);
-      process = NULL;  // Don't check any more once one is found.
-    }
     ProfileContext* context = scan->second;
     if (context->incidents.empty())
       continue;
     if (!prefs->GetBoolean(prefs::kSafeBrowsingEnabled)) {
       for (size_t i = 0; i < context->incidents.size(); ++i) {
         LogIncidentDataType(DROPPED, *context->incidents[i]);
       }
       context->incidents.clear();
       continue;
     }
@@ skipping 162 matching lines @@
       if (!profile->IsOffTheRecord())
         OnProfileDestroyed(profile);
       break;
     }
     default:
       break;
   }
 }
 
 }  // namespace safe_browsing