Move logic for uploading logs into a ReportingService object.

components/metrics/metrics_service.cc

 // Copyright 2014 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 //------------------------------------------------------------------------------
 // Description of the life cycle of a instance of MetricsService.
 //
 //  OVERVIEW
 //
 // A MetricsService instance is typically created at application startup.  It is
@@ skipping 113 matching lines @@
 
 #include "components/metrics/metrics_service.h"
 
 #include <stddef.h>
 
 #include <algorithm>
 #include <utility>
 
 #include "base/bind.h"
 #include "base/callback.h"
-#include "base/feature_list.h"
 #include "base/location.h"
 #include "base/memory/ptr_util.h"
 #include "base/metrics/histogram_base.h"
 #include "base/metrics/histogram_macros.h"
 #include "base/metrics/histogram_samples.h"
 #include "base/metrics/persistent_histogram_allocator.h"
 #include "base/metrics/sparse_histogram.h"
 #include "base/metrics/statistics_recorder.h"
 #include "base/single_thread_task_runner.h"
-#include "base/strings/string_number_conversions.h"
-#include "base/strings/utf_string_conversions.h"
 #include "base/threading/thread_task_runner_handle.h"
 #include "base/time/time.h"
 #include "base/tracked_objects.h"
 #include "build/build_config.h"
-#include "components/metrics/data_use_tracker.h"
 #include "components/metrics/environment_recorder.h"
 #include "components/metrics/metrics_log.h"
 #include "components/metrics/metrics_log_manager.h"
 #include "components/metrics/metrics_log_uploader.h"
 #include "components/metrics/metrics_pref_names.h"
-#include "components/metrics/metrics_reporting_scheduler.h"
 #include "components/metrics/metrics_rotation_scheduler.h"
 #include "components/metrics/metrics_service_client.h"
 #include "components/metrics/metrics_state_manager.h"
-#include "components/metrics/metrics_upload_scheduler.h"
 #include "components/metrics/stability_metrics_provider.h"
 #include "components/metrics/url_constants.h"
 #include "components/prefs/pref_registry_simple.h"
 #include "components/prefs/pref_service.h"
 #include "components/variations/entropy_provider.h"
 
 namespace metrics {
 
 namespace {
 
@@ skipping 10 matching lines @@
 // TODO(dfalcantara): To avoid delaying startup, tighten up initialization so
 //                    that it occurs after the user gets their initial page.
 const int kInitializationDelaySeconds = 5;
 #else
 const int kInitializationDelaySeconds = 30;
 #endif
 
 // The maximum number of events in a log uploaded to the UMA server.
 const int kEventLimit = 2400;
 
-// If an upload fails, and the transmission was over this byte count, then we
-// will discard the log, and not try to retransmit it.  We also don't persist
-// the log to the prefs for transmission during the next chrome session if this
-// limit is exceeded.
-const size_t kUploadLogAvoidRetransmitSize = 100 * 1024;
-
-enum ResponseStatus {
-  UNKNOWN_FAILURE,
-  SUCCESS,
-  BAD_REQUEST,  // Invalid syntax or log too large.
-  NO_RESPONSE,
-  NUM_RESPONSE_STATUSES
-};
-
-ResponseStatus ResponseCodeToStatus(int response_code) {
-  switch (response_code) {
-    case -1:
-      return NO_RESPONSE;
-    case 200:
-      return SUCCESS;
-    case 400:
-      return BAD_REQUEST;
-    default:
-      return UNKNOWN_FAILURE;
-  }
-}
-
 #if defined(OS_ANDROID) || defined(OS_IOS)
 void MarkAppCleanShutdownAndCommit(CleanExitBeacon* clean_exit_beacon,
                                    PrefService* local_state) {
   clean_exit_beacon->WriteBeaconValue(true);
   ExecutionPhaseManager(local_state).OnAppEnterBackground();
   // Start writing right away (write happens on a different thread).
   local_state->CommitPendingWrite();
 }
 #endif  // defined(OS_ANDROID) || defined(OS_IOS)
 
@@ skipping 19 matching lines @@
   registry->RegisterListPref(prefs::kMetricsInitialLogs);
   registry->RegisterListPref(prefs::kMetricsOngoingLogs);
 
   registry->RegisterInt64Pref(prefs::kUninstallLaunchCount, 0);
   registry->RegisterInt64Pref(prefs::kUninstallMetricsUptimeSec, 0);
 }
 
 MetricsService::MetricsService(MetricsStateManager* state_manager,
                                MetricsServiceClient* client,
                                PrefService* local_state)
-    : log_store_(local_state, kUploadLogAvoidRetransmitSize),
+    : reporting_service_(client, local_state),
       histogram_snapshot_manager_(this),
       state_manager_(state_manager),
       client_(client),
       local_state_(local_state),
       clean_exit_beacon_(client->GetRegistryBackupKey(), local_state),
       recording_state_(UNSET),
-      reporting_active_(false),
       test_mode_active_(false),
       state_(INITIALIZED),
-      log_upload_in_progress_(false),
       idle_since_last_transmission_(false),
       session_id_(-1),
-      data_use_tracker_(DataUseTracker::Create(local_state_)),
       self_ptr_factory_(this) {
   DCHECK(thread_checker_.CalledOnValidThread());
   DCHECK(state_manager_);
   DCHECK(client_);
   DCHECK(local_state_);
 
   // Set the install date if this is our first run.
   int64_t install_date = local_state_->GetInt64(prefs::kInstallDate);
   if (install_date == 0)
     local_state_->SetInt64(prefs::kInstallDate, base::Time::Now().ToTimeT());
 
   RegisterMetricsProvider(std::unique_ptr<metrics::MetricsProvider>(
       new StabilityMetricsProvider(local_state_)));
 }
 
 MetricsService::~MetricsService() {
   DisableRecording();
 }
 
 void MetricsService::InitializeMetricsRecordingState() {
+  reporting_service_.Initialize();
   InitializeMetricsState();
 
   base::Closure upload_callback =
       base::Bind(&MetricsService::StartScheduledUpload,
                  self_ptr_factory_.GetWeakPtr());
 
   rotation_scheduler_.reset(new MetricsRotationScheduler(
       upload_callback,
       // MetricsServiceClient outlives MetricsService, and
       // MetricsRotationScheduler is tied to the lifetime of |this|.
       base::Bind(&MetricsServiceClient::GetStandardUploadInterval,
                  base::Unretained(client_))));
-  base::Closure send_next_log_callback =
-      base::Bind(&MetricsService::SendNextLog, self_ptr_factory_.GetWeakPtr());
-  upload_scheduler_.reset(new MetricsUploadScheduler(send_next_log_callback));
 
   for (auto& provider : metrics_providers_)
     provider->Init();
 }
 
 void MetricsService::Start() {
   HandleIdleSinceLastTransmission(false);
   EnableRecording();
   EnableReporting();
 }
 
 void MetricsService::StartRecordingForTests() {
   test_mode_active_ = true;
   EnableRecording();
   DisableReporting();
 }
 
 void MetricsService::Stop() {
   HandleIdleSinceLastTransmission(false);
   DisableReporting();
   DisableRecording();
 }
 
 void MetricsService::EnableReporting() {
-  if (reporting_active_)
+  if (reporting_service_.reporting_active())
     return;
-  reporting_active_ = true;
+  reporting_service_.EnableReporting();
   StartSchedulerIfNecessary();
 }
 
 void MetricsService::DisableReporting() {
-  reporting_active_ = false;
+  reporting_service_.DisableReporting();
 }
 
 std::string MetricsService::GetClientId() {
   return state_manager_->client_id();
 }
 
 int64_t MetricsService::GetInstallDate() {
   return local_state_->GetInt64(prefs::kInstallDate);
 }
 
@@ skipping 41 matching lines @@
   PushPendingLogsToPersistentStorage();
 }
 
 bool MetricsService::recording_active() const {
   DCHECK(thread_checker_.CalledOnValidThread());
   return recording_state_ == ACTIVE;
 }
 
 bool MetricsService::reporting_active() const {
   DCHECK(thread_checker_.CalledOnValidThread());
-  return reporting_active_;
+  return reporting_service_.reporting_active();
+}
+
+bool MetricsService::has_unsent_logs() const {
+  return reporting_service_.metrics_log_store()->has_unsent_logs();

[Alexei Svitkine (slow), 2017/02/28 18:03:45]

Nit: can you just use log_store().has_unsent_logs() and not have this extra
function?

[Steven Holte, 2017/03/01 02:02:33]

Not quite.  I would need a const version of log_store(), so I'd still have an
extra function.

 }
 
 void MetricsService::RecordDelta(const base::HistogramBase& histogram,
                                  const base::HistogramSamples& snapshot) {
   log_manager_.current_log()->RecordHistogramDelta(histogram.histogram_name(),
                                                    snapshot);
 }
 
 void MetricsService::InconsistencyDetected(
     base::HistogramBase::Inconsistency problem) {
@@ skipping 30 matching lines @@
   LogCleanShutdown(false);
 }
 
 void MetricsService::RecordCompletedSessionEnd() {
   LogCleanShutdown(true);
 }
 
 #if defined(OS_ANDROID) || defined(OS_IOS)
 void MetricsService::OnAppEnterBackground() {
   rotation_scheduler_->Stop();
-  upload_scheduler_->Stop();
+  reporting_service_.Stop();
 
   MarkAppCleanShutdownAndCommit(&clean_exit_beacon_, local_state_);
 
   // Give providers a chance to persist histograms as part of being
   // backgrounded.
   for (auto& provider : metrics_providers_)
     provider->OnAppEnterBackground();
 
   // At this point, there's no way of knowing when the process will be
   // killed, so this has to be treated similar to a shutdown, closing and
@@ skipping 35 matching lines @@
   StabilityMetricsProvider(local_state_)
       .RecordBreakpadHasDebugger(has_debugger);
 }
 
 void MetricsService::ClearSavedStabilityMetrics() {
   for (auto& provider : metrics_providers_)
     provider->ClearSavedStabilityMetrics();
 }
 
 void MetricsService::PushExternalLog(const std::string& log) {
-  log_store_.StoreLog(log, MetricsLog::ONGOING_LOG);
+  log_store()->StoreLog(log, MetricsLog::ONGOING_LOG);
 }
 
 void MetricsService::UpdateMetricsUsagePrefs(const std::string& service_name,
                                              int message_size,
                                              bool is_cellular) {
   DCHECK(thread_checker_.CalledOnValidThread());
-  if (data_use_tracker_) {
-    data_use_tracker_->UpdateMetricsUsagePrefs(service_name,
-                                               message_size,
-                                               is_cellular);
-  }
+  reporting_service_.UpdateMetricsUsagePrefs(service_name, message_size,
+                                             is_cellular);
 }
 
 //------------------------------------------------------------------------------
 // private methods
 //------------------------------------------------------------------------------
 
 
 //------------------------------------------------------------------------------
 // Initialization methods
 
 void MetricsService::InitializeMetricsState() {
   const int64_t buildtime = MetricsLog::GetBuildTime();
   const std::string version = client_->GetVersionString();
 
   bool version_changed = false;
   EnvironmentRecorder recorder(local_state_);
   int64_t previous_buildtime = recorder.GetLastBuildtime();
   std::string previous_version = recorder.GetLastVersion();
   if (previous_buildtime != buildtime || previous_version != version) {
     recorder.SetBuildtimeAndVersion(buildtime, version);
     version_changed = true;
   }
 
-  log_store_.LoadPersistedUnsentLogs();
+  reporting_service_.Initialize();
 
   session_id_ = local_state_->GetInteger(prefs::kMetricsSessionID);
 
   StabilityMetricsProvider provider(local_state_);
   if (!clean_exit_beacon_.exited_cleanly()) {
     provider.LogCrash();
     // Reset flag, and wait until we call LogNeedForCleanShutdown() before
     // monitoring.
     clean_exit_beacon_.WriteBeaconValue(true);
     ExecutionPhaseManager manager(local_state_);
@@ skipping 162 matching lines @@
   RecordCurrentEnvironment(current_log);
   base::TimeDelta incremental_uptime;
   base::TimeDelta uptime;
   GetUptimes(local_state_, &incremental_uptime, &uptime);
   current_log->RecordStabilityMetrics(metrics_providers_, incremental_uptime,
                                       uptime);
 
   current_log->RecordGeneralMetrics(metrics_providers_);
   RecordCurrentHistograms();
   DVLOG(1) << "Generated an ongoing log.";
-  log_manager_.FinishCurrentLog(&log_store_);
+  log_manager_.FinishCurrentLog(log_store());
 }
 
 void MetricsService::PushPendingLogsToPersistentStorage() {
   if (state_ < SENDING_LOGS)
     return;  // We didn't and still don't have time to get plugin list etc.
 
   CloseCurrentLog();
-  log_store_.PersistUnsentLogs();
+  log_store()->PersistUnsentLogs();
 }
 
 //------------------------------------------------------------------------------
 // Transmission of logs methods
 
 void MetricsService::StartSchedulerIfNecessary() {
   // Never schedule cutting or uploading of logs in test mode.
   if (test_mode_active_)
     return;
 
   // Even if reporting is disabled, the scheduler is needed to trigger the
   // creation of the initial log, which must be done in order for any logs to be
   // persisted on shutdown or backgrounding.
   if (recording_active() &&
       (reporting_active() || state_ < SENDING_LOGS)) {
     rotation_scheduler_->Start();
-    upload_scheduler_->Start();
+    reporting_service_.Start();
   }
 }
 
 void MetricsService::StartScheduledUpload() {
   DVLOG(1) << "StartScheduledUpload";
   DCHECK(state_ >= INIT_TASK_DONE);
   // If we're getting no notifications, then the log won't have much in it, and
   // it's possible the computer is about to go to sleep, so don't upload and
   // stop the scheduler.
   // If recording has been turned off, the scheduler doesn't need to run.
   // If reporting is off, proceed if the initial log hasn't been created, since
   // that has to happen in order for logs to be cut and stored when persisting.
   // TODO(stuartmorgan): Call Stop() on the scheduler when reporting and/or
   // recording are turned off instead of letting it fire and then aborting.
   if (idle_since_last_transmission_ ||
       !recording_active() ||
       (!reporting_active() && state_ >= SENDING_LOGS)) {
     rotation_scheduler_->Stop();
     rotation_scheduler_->RotationFinished();
     return;
   }
 
   // If there are unsent logs, send the next one. If not, start the asynchronous
   // process of finalizing the current log for upload.
-  if (state_ == SENDING_LOGS && log_store_.has_unsent_logs()) {
-    upload_scheduler_->Start();
+  if (state_ == SENDING_LOGS && has_unsent_logs()) {
+    reporting_service_.Start();
     rotation_scheduler_->RotationFinished();
   } else {
     // There are no logs left to send, so start creating a new one.
     client_->CollectFinalMetricsForLog(
         base::Bind(&MetricsService::OnFinalLogInfoCollectionDone,
                    self_ptr_factory_.GetWeakPtr()));
   }
 }
 
 void MetricsService::OnFinalLogInfoCollectionDone() {
   DVLOG(1) << "OnFinalLogInfoCollectionDone";
   // Abort if metrics were turned off during the final info gathering.
   if (!recording_active()) {
     rotation_scheduler_->Stop();
     rotation_scheduler_->RotationFinished();
     return;
   }
 
   if (state_ == INIT_TASK_DONE) {
     PrepareInitialMetricsLog();
   } else {
     DCHECK_EQ(SENDING_LOGS, state_);
     CloseCurrentLog();
     OpenNewLog();
   }
+  reporting_service_.Start();
+  rotation_scheduler_->RotationFinished();
   HandleIdleSinceLastTransmission(true);
-  upload_scheduler_->Start();
-  rotation_scheduler_->RotationFinished();
-}
-
-void MetricsService::SendNextLog() {
-  DVLOG(1) << "SendNextLog";
-  if (!reporting_active()) {
-    upload_scheduler_->StopAndUploadCancelled();
-    return;
-  }
-  if (!log_store_.has_unsent_logs()) {
-    // Should only get here if serializing the log failed somehow.
-    upload_scheduler_->Stop();
-    // Reset backoff interval
-    upload_scheduler_->UploadFinished(true);
-    return;
-  }
-  if (!log_store_.has_staged_log())
-    log_store_.StageNextLog();
-
-  // Proceed to stage the log for upload if log size satisfies cellular log
-  // upload constrains.
-  bool upload_canceled = false;
-  bool is_cellular_logic = client_->IsUMACellularUploadLogicEnabled();
-  if (is_cellular_logic && data_use_tracker_ &&
-      !data_use_tracker_->ShouldUploadLogOnCellular(
-          log_store_.staged_log_hash().size())) {
-    upload_scheduler_->UploadOverDataUsageCap();
-    upload_canceled = true;
-  } else {
-    SendStagedLog();
-  }
-  if (is_cellular_logic) {
-    UMA_HISTOGRAM_BOOLEAN("UMA.LogUpload.Canceled.CellularConstraint",
-                          upload_canceled);
-  }
 }
 
 bool MetricsService::ProvidersHaveInitialStabilityMetrics() {
   // Check whether any metrics provider has initial stability metrics.
   // All providers are queried (rather than stopping after the first "true"
   // response) in case they do any kind of setup work in preparation for
   // the later call to RecordInitialHistogramSnapshots().
   bool has_stability_metrics = false;
   for (auto& provider : metrics_providers_)
     has_stability_metrics |= provider->HasInitialStabilityMetrics();
@@ skipping 24 matching lines @@
   // Note: Some stability providers may record stability stats via histograms,
   //       so this call has to be after BeginLoggingWithLog().
   log_manager_.current_log()->RecordStabilityMetrics(
       metrics_providers_, base::TimeDelta(), base::TimeDelta());
   RecordCurrentStabilityHistograms();
 
   // Note: RecordGeneralMetrics() intentionally not called since this log is for
   //       stability stats from a previous session only.
 
   DVLOG(1) << "Generated an stability log.";
-  log_manager_.FinishCurrentLog(&log_store_);
+  log_manager_.FinishCurrentLog(log_store());
   log_manager_.ResumePausedLog();
 
   // Store unsent logs, including the stability log that was just saved, so
   // that they're not lost in case of a crash before upload time.
-  log_store_.PersistUnsentLogs();
+  log_store()->PersistUnsentLogs();
 
   return true;
 }
 
 void MetricsService::PrepareInitialMetricsLog() {
   DCHECK_EQ(INIT_TASK_DONE, state_);
 
   RecordCurrentEnvironment(initial_metrics_log_.get());
   base::TimeDelta incremental_uptime;
   base::TimeDelta uptime;
   GetUptimes(local_state_, &incremental_uptime, &uptime);
 
   // Histograms only get written to the current log, so make the new log current
   // before writing them.
   log_manager_.PauseCurrentLog();
   log_manager_.BeginLoggingWithLog(std::move(initial_metrics_log_));
 
   // Note: Some stability providers may record stability stats via histograms,
   //       so this call has to be after BeginLoggingWithLog().
   MetricsLog* current_log = log_manager_.current_log();
   current_log->RecordStabilityMetrics(metrics_providers_, base::TimeDelta(),
                                       base::TimeDelta());
   current_log->RecordGeneralMetrics(metrics_providers_);
   RecordCurrentHistograms();
 
   DVLOG(1) << "Generated an initial log.";
-  log_manager_.FinishCurrentLog(&log_store_);
+  log_manager_.FinishCurrentLog(log_store());
   log_manager_.ResumePausedLog();
 
   // Store unsent logs, including the initial log that was just saved, so
   // that they're not lost in case of a crash before upload time.
-  log_store_.PersistUnsentLogs();
+  log_store()->PersistUnsentLogs();
 
   state_ = SENDING_LOGS;
 }
 
-void MetricsService::SendStagedLog() {
-  DCHECK(log_store_.has_staged_log());
-  if (!log_store_.has_staged_log())
-    return;
-
-  DCHECK(!log_upload_in_progress_);
-  log_upload_in_progress_ = true;
-
-  if (!log_uploader_) {
-    log_uploader_ = client_->CreateUploader(
-        client_->GetMetricsServerUrl(), metrics::kDefaultMetricsMimeType,
-        metrics::MetricsLogUploader::UMA,
-        base::Bind(&MetricsService::OnLogUploadComplete,
-                   self_ptr_factory_.GetWeakPtr()));
-  }
-  const std::string hash = base::HexEncode(log_store_.staged_log_hash().data(),
-                                           log_store_.staged_log_hash().size());
-  log_uploader_->UploadLog(log_store_.staged_log(), hash);
-}
-
-
-void MetricsService::OnLogUploadComplete(int response_code) {
-  DVLOG(1) << "OnLogUploadComplete:" << response_code;
-  DCHECK(log_upload_in_progress_);
-  log_upload_in_progress_ = false;
-
-  // Log a histogram to track response success vs. failure rates.
-  UMA_HISTOGRAM_ENUMERATION("UMA.UploadResponseStatus.Protobuf",
-                            ResponseCodeToStatus(response_code),
-                            NUM_RESPONSE_STATUSES);
-
-  bool upload_succeeded = response_code == 200;
-
-  // Provide boolean for error recovery (allow us to ignore response_code).
-  bool discard_log = false;
-  const size_t log_size = log_store_.staged_log().length();
-  if (upload_succeeded) {
-    UMA_HISTOGRAM_COUNTS_10000("UMA.LogSize.OnSuccess", log_size / 1024);
-  } else if (log_size > kUploadLogAvoidRetransmitSize) {
-    UMA_HISTOGRAM_COUNTS("UMA.Large Rejected Log was Discarded",
-                         static_cast<int>(log_size));
-    discard_log = true;
-  } else if (response_code == 400) {
-    // Bad syntax.  Retransmission won't work.
-    discard_log = true;
-  }
-
-  if (upload_succeeded || discard_log) {
-    log_store_.DiscardStagedLog();
-    // Store the updated list to disk now that the removed log is uploaded.
-    log_store_.PersistUnsentLogs();
-  }
-
-  // Error 400 indicates a problem with the log, not with the server, so
-  // don't consider that a sign that the server is in trouble.
-  bool server_is_healthy = upload_succeeded || response_code == 400;
-  if (!log_store_.has_unsent_logs()) {
-    DVLOG(1) << "Stopping upload_scheduler_";
-    upload_scheduler_->Stop();
-  }
-  upload_scheduler_->UploadFinished(server_is_healthy);
-}
-
 void MetricsService::IncrementLongPrefsValue(const char* path) {
   int64_t value = local_state_->GetInt64(path);
   local_state_->SetInt64(path, value + 1);
 }
 
 bool MetricsService::UmaMetricsProperlyShutdown() {
   CHECK(clean_shutdown_status_ == CLEANLY_SHUTDOWN ||
         clean_shutdown_status_ == NEED_TO_SHUTDOWN);
   return clean_shutdown_status_ == CLEANLY_SHUTDOWN;
 }
@@ skipping 135 matching lines @@
   // Redundant setting to assure that we always reset this value at shutdown
   // (and that we don't use some alternate path, and not call LogCleanShutdown).
   clean_shutdown_status_ = CLEANLY_SHUTDOWN;
   client_->OnLogCleanShutdown();
   clean_exit_beacon_.WriteBeaconValue(true);
   SetExecutionPhase(ExecutionPhase::SHUTDOWN_COMPLETE, local_state_);
   StabilityMetricsProvider(local_state_).MarkSessionEndCompleted(end_completed);
 }
 
 }  // namespace metrics