[OBSOLETE] Reporting: Initial implementation.

net/reporting/reporting_service.cc

+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "net/reporting/reporting_service.h"
+
+#include "base/bind.h"
+#include "base/json/json_reader.h"
+#include "base/json/json_writer.h"
+#include "base/memory/ptr_util.h"
+#include "base/strings/string_number_conversions.h"
+#include "base/strings/string_util.h"
+#include "base/time/default_tick_clock.h"
+#include "net/http/http_request_info.h"
+#include "net/http/http_response_headers.h"
+#include "net/http/http_response_info.h"
+#include "net/reporting/reporting_metrics.h"
+#include "net/url_request/url_fetcher.h"
+#include "net/url_request/url_request_context_getter.h"
+
+namespace {
+const char kDefaultGroupName[] = "default";
+
+// Per
+// https://greenbytes.de/tech/webdav/draft-reschke-http-jfv-02.html#rfc.section.4
+// assuming |normalized_header| is the result of completing step 1.
+std::unique_ptr<base::Value> ParseJFV(
+    const std::string& normalized_header_value) {
+  std::string value = "[" + normalized_header_value + "]";
+  return base::JSONReader::Read(value);
+}
+
+bool IsOriginSecure(const GURL& url) {
+  return url.SchemeIsCryptographic();
+}
+}  // namespace
+
+namespace net {
+
+// static
+ReportingService::Policy ReportingService::Policy::GetDefault() {
+  Policy policy;
+
+  policy.endpoint_lifetime = base::TimeDelta::FromDays(7);
+  policy.endpoint_backoff.num_errors_to_ignore = 0;
+  policy.endpoint_backoff.initial_delay_ms = 5 * 1000;
+  policy.endpoint_backoff.multiply_factor = 2.0;
+  policy.endpoint_backoff.jitter_factor = 0.1;
+  policy.endpoint_backoff.maximum_backoff_ms = 60 * 60 * 1000;
+  policy.endpoint_backoff.entry_lifetime_ms = 7 * 24 * 60 * 60 * 1000;
+  policy.endpoint_backoff.always_use_initial_delay = false;
+  policy.max_endpoint_failures = 5;
+  policy.max_endpoint_count = 100;
+
+  policy.report_lifetime = base::TimeDelta::FromDays(2);
+  policy.max_report_failures = 5;
+  policy.max_report_count = 100;
+
+  policy.persist_reports_across_network_changes = false;
+
+  return policy;
+}
+
+ReportingService::ReportingService(const Policy& policy)
+    : policy_(policy), clock_(new base::DefaultTickClock()) {}
+
+ReportingService::~ReportingService() {
+  for (auto& report : reports_)
+    HistogramReportInternal(REPORT_FATE_SHUTDOWN, *report);
+  size_t endpoint_count = endpoints_.size();
+  for (size_t i = 0; i < endpoint_count; ++i)
+    HistogramEndpoint(ENDPOINT_FATE_SHUTDOWN);
+}
+
+void ReportingService::set_uploader(
+    std::unique_ptr<ReportingUploader> uploader) {
+  uploader_ = std::move(uploader);
+}
+
+void ReportingService::QueueReport(std::unique_ptr<base::Value> body,
+                                   const GURL& url,
+                                   const GURL& origin,
+                                   const std::string& group,
+                                   const std::string& type) {
+  auto report = base::MakeUnique<ReportingReport>();
+  report->body = std::move(body);
+  report->url = url.GetAsReferrer();
+  report->origin = origin;
+  report->group = group;
+  report->type = type;
+  report->timestamp = clock_->NowTicks();
+  report->attempts = 0;
+  report->pending = false;
+  reports_.push_back(std::move(report));
+}
+
+void ReportingService::ProcessHeader(const GURL& origin,
+                                     const std::string& header_value) {
+  if (!IsOriginSecure(origin)) {
+    HistogramHeader(HEADER_FATE_REJECTED_INSECURE_ORIGIN);
+    return;
+  }
+
+  std::vector<std::string> errors;
+  std::vector<EndpointTuple> tuples;
+  if (!EndpointTuple::FromHeader(header_value, &tuples, &errors)) {
+    HistogramHeader(HEADER_FATE_REJECTED_INVALID_JSON);
+    return;
+  }
+
+  if (errors.empty())
+    HistogramHeader(HEADER_FATE_ACCEPTED);
+  else
+    HistogramHeader(HEADER_FATE_ACCEPTED_WITH_INVALID_ENDPOINT);
+
+  // TODO: Plumb these out to the DevTools console somehow.
+  for (const std::string& error : errors) {
+    LOG(WARNING) << "Origin " << origin.spec() << " sent "
+                 << "Report-To header with error: " << error << ": "
+                 << header_value;
+  }
+
+  for (auto& tuple : tuples)
+    ProcessEndpointTuple(origin, tuple);
+
+  CollectGarbage();
+}
+
+void ReportingService::SendReports() {
+  if (!uploader_)
+    return;
+
+  std::map<Endpoint*, std::vector<ReportingReport*>> endpoint_reports;
+  for (auto& report : reports_) {
+    // If the report is already contained in another pending upload, don't
+    // upload it twice.
+    if (report->pending)
+      continue;
+    Endpoint* endpoint = FindEndpointForReport(*report);
+    // If there's no available endpoint for the report, leave it for later.
+    if (!endpoint)
+      continue;
+    // If the chosen endpoint is pending, don't start another upload; let this
+    // report go in the next upload instead.
+    if (endpoint->pending)
+      continue;
+
+    report->pending = true;
+    endpoint_reports[endpoint].push_back(report.get());
+  }
+
+  for (auto& pair : endpoint_reports) {
+    Endpoint* endpoint = pair.first;
+    const std::vector<ReportingReport*>& reports = pair.second;
+
+    std::string json = SerializeReports(reports);
+
+    uploader_->AttemptDelivery(
+        endpoint->url, json,
+        base::Bind(&ReportingService::OnDeliveryAttemptComplete,
+                   base::Unretained(this),
+                   base::MakeUnique<Delivery>(endpoint->url, reports)));
+
+    endpoint->last_used = clock_->NowTicks();
+
+    for (auto& report : reports)
+      ++report->attempts;
+
+    HistogramDeliveryContent(reports.size(), json.length());
+  }
+}
+
+void ReportingService::set_clock_for_testing(
+    std::unique_ptr<base::TickClock> clock) {
+  clock_ = std::move(clock);
+}
+
+bool ReportingService::HasEndpointForTesting(const GURL& endpoint_url) {
+  return GetEndpointByURL(endpoint_url);
+}
+
+bool ReportingService::HasClientForTesting(const GURL& endpoint_url,
+                                           const GURL& origin) {
+  Endpoint* endpoint = GetEndpointByURL(endpoint_url);
+  if (!endpoint)
+    return false;
+  return endpoint->clients.count(origin) > 0;
+}
+
+int ReportingService::GetEndpointFailuresForTesting(const GURL& endpoint_url) {
+  Endpoint* endpoint = GetEndpointByURL(endpoint_url);
+  if (!endpoint)
+    return -1;
+  return endpoint->backoff.failure_count();
+}
+
+void ReportingService::CollectGarbageForTesting() {
+  CollectGarbage();
+}
+
+ReportingService::Client::Client(const GURL& origin,
+                                 bool subdomains,
+                                 const std::string& group,
+                                 base::TimeDelta ttl,
+                                 base::TimeTicks creation)
+    : origin(origin),
+      subdomains(subdomains),
+      group(group),
+      ttl(ttl),
+      creation(creation) {}
+
+ReportingService::Endpoint::Endpoint(const GURL& url,
+                                     const BackoffEntry::Policy& backoff_policy,
+                                     base::TickClock* clock)
+    : url(url),
+      backoff(&backoff_policy, clock),
+      last_used(clock->NowTicks()),
+      pending(false) {}
+ReportingService::Endpoint::~Endpoint() {}
+
+bool ReportingService::Endpoint::is_expired(base::TimeTicks now) const {
+  for (auto& pair : clients)
+    if (!pair.second.is_expired(now))
+      return false;
+  return true;
+}
+
+// static
+bool ReportingService::EndpointTuple::FromDictionary(
+    const base::DictionaryValue& dictionary,
+    EndpointTuple* tuple_out,
+    std::string* error_out) {
+  if (!dictionary.HasKey("url")) {
+    *error_out = "url missing";
+    return false;
+  }
+  std::string url_string;
+  if (!dictionary.GetString("url", &url_string)) {
+    *error_out = "url not a string";
+    return false;
+  }
+  tuple_out->url = GURL(url_string);
+
+  tuple_out->group = kDefaultGroupName;
+  if (dictionary.HasKey("group")) {
+    if (!dictionary.GetString("group", &tuple_out->group)) {
+      *error_out = "group present but not a string";
+      return false;
+    }
+  }
+
+  tuple_out->subdomains = false;
+  if (dictionary.HasKey("includeSubdomains")) {
+    if (!dictionary.GetBoolean("includeSubdomains", &tuple_out->subdomains)) {
+      *error_out = "includeSubdomains present but not boolean";
+      return false;
+    }
+  }
+
+  if (!dictionary.HasKey("max-age")) {
+    *error_out = "max-age missing";
+    return false;
+  }
+  int ttl_sec;
+  if (!dictionary.GetInteger("max-age", &ttl_sec)) {
+    *error_out = "max-age not an integer";
+    return false;
+  }
+  tuple_out->ttl = base::TimeDelta::FromSeconds(ttl_sec);
+
+  return true;
+}
+
+// static
+bool ReportingService::EndpointTuple::FromHeader(
+    const std::string& header,
+    std::vector<ReportingService::EndpointTuple>* tuples_out,
+    std::vector<std::string>* errors_out) {
+  tuples_out->clear();
+  errors_out->clear();
+
+  std::unique_ptr<base::Value> value(ParseJFV(header));
+  if (!value) {
+    errors_out->push_back("failed to parse JSON field value.");
+    return false;
+  }
+
+  base::ListValue* list;
+  bool was_list = value->GetAsList(&list);
+  DCHECK(was_list);
+
+  base::DictionaryValue* item;
+  for (size_t i = 0; i < list->GetSize(); i++) {
+    std::string error_prefix = "endpoint " + base::SizeTToString(i + 1) +
+                               " of " + base::SizeTToString(list->GetSize()) +
+                               ": ";
+    if (!list->GetDictionary(i, &item)) {
+      errors_out->push_back(error_prefix + "is not a dictionary");
+      continue;
+    }
+    EndpointTuple tuple;
+    std::string error;
+    if (!EndpointTuple::FromDictionary(*item, &tuple, &error)) {
+      errors_out->push_back(error_prefix + error);
+      continue;
+    }
+    if (!IsOriginSecure(tuple.url)) {
+      errors_out->push_back(error_prefix + "url " + tuple.url.spec() +
+                            " is insecure");
+      continue;
+    }
+    if (tuple.ttl < base::TimeDelta()) {
+      errors_out->push_back(error_prefix + "ttl is negative");
+      continue;
+    }
+    tuples_out->push_back(tuple);
+  }
+  return true;
+}
+
+std::string ReportingService::EndpointTuple::ToString() const {
+  return "(url=" + url.spec() + ", subdomains=" +
+         (subdomains ? "true" : "false") + ", ttl=" +
+         base::Int64ToString(ttl.InSeconds()) + "s" + ", group=" + group + ")";
+}
+
+ReportingService::Delivery::Delivery(
+    const GURL& endpoint_url,
+    const std::vector<ReportingReport*>& reports)
+    : endpoint_url(endpoint_url), reports(reports) {}
+
+ReportingService::Delivery::~Delivery() {}
+
+void ReportingService::ProcessEndpointTuple(const GURL& origin,
+                                            const EndpointTuple& tuple) {
+  Endpoint* endpoint = GetEndpointByURL(tuple.url);
+
+  bool endpoint_exists = endpoint;
+  bool client_exists = endpoint && endpoint->clients.count(origin) > 0;
+
+  HistogramHeaderEndpointInternal(endpoint_exists, client_exists, tuple.ttl);
+
+  if (client_exists)
+    endpoint->clients.erase(origin);
+
+  if (tuple.ttl <= base::TimeDelta())
+    return;
+
+  if (!endpoint_exists) {
+    endpoint = new Endpoint(tuple.url, policy_.endpoint_backoff, clock_.get());
+    endpoints_.insert(std::make_pair(tuple.url, base::WrapUnique(endpoint)));
+  }
+
+  if (!client_exists)
+    HistogramClient(tuple.ttl);
+
+  Client client(origin, tuple.subdomains, tuple.group, tuple.ttl,
+                clock_->NowTicks());
+  endpoint->clients.insert(std::make_pair(origin, client));
+}
+
+void ReportingService::OnDeliveryAttemptComplete(
+    const std::unique_ptr<Delivery>& delivery,
+    ReportingUploader::Outcome outcome) {
+  // Note: HistogramDeliveryOutcome is called from within the uploader since it
+  // has access to the net error code and HTTP response.
+
+  for (auto report : delivery->reports) {
+    DCHECK(report->pending);
+    report->pending = false;
+  }
+
+  Endpoint* endpoint = GetEndpointByURL(delivery->endpoint_url);
+  if (endpoint) {
+    endpoint->backoff.InformOfRequest(outcome == ReportingUploader::SUCCESS);
+    endpoint->pending = false;
+  }
+
+  switch (outcome) {
+    case ReportingUploader::SUCCESS:
+      for (auto report : delivery->reports) {
+        HistogramReportInternal(REPORT_FATE_DELIVERED, *report);
+        DequeueReport(report);
+      }
+      break;
+    case ReportingUploader::FAILURE:
+      // Reports have been marked not-pending and can be retried later.
+      // BackoffEntry has been informed of failure.
+      break;
+    case ReportingUploader::REMOVE_ENDPOINT:
+      // Note: This is not specified, but seems the obvious intention.
+      if (endpoint) {
+        HistogramEndpoint(ENDPOINT_FATE_REQUESTED_REMOVAL);
+        endpoints_.erase(delivery->endpoint_url);
+      }
+      break;
+  }
+
+  CollectGarbage();
+}
+
+void ReportingService::CollectGarbage() {
+  base::TimeTicks now = clock_->NowTicks();
+
+  {
+    std::vector<ReportVector::iterator> to_erase;
+    for (auto it = reports_.begin(); it != reports_.end(); ++it) {
+      ReportingReport* report = it->get();
+      if (report->pending)
+        continue;
+      if (policy_.max_report_failures > 0 &&
+          report->attempts >= policy_.max_report_failures) {
+        HistogramReportInternal(REPORT_FATE_FAILED, *report);
+        to_erase.push_back(it);
+      } else if (!policy_.report_lifetime.is_zero() &&
+                 now - report->timestamp > policy_.report_lifetime) {
+        HistogramReportInternal(REPORT_FATE_EXPIRED, *report);
+        to_erase.push_back(it);
+      }
+    }
+
+    for (auto it = reports_.begin();
+         it != reports_.end() &&
+         reports_.size() - to_erase.size() > policy_.max_report_count;
+         ++it) {
+      if (it->get()->pending)
+        continue;
+
+      HistogramReportInternal(REPORT_FATE_EVICTED, *it->get());
+      to_erase.push_back(it);
+    }
+
+    for (auto it : to_erase)
+      reports_.erase(it);
+  }
+
+  {
+    std::vector<EndpointMap::iterator> to_erase;
+    for (auto it = endpoints_.begin(); it != endpoints_.end(); ++it) {
+      Endpoint* endpoint = it->second.get();
+      if (endpoint->pending)
+        continue;
+      if (endpoint->is_expired(now)) {
+        HistogramEndpoint(ENDPOINT_FATE_EXPIRED);
+        to_erase.push_back(it);
+      } else if (!policy_.endpoint_lifetime.is_zero() &&
+                 now - endpoint->last_used > policy_.endpoint_lifetime) {
+        HistogramEndpoint(ENDPOINT_FATE_UNUSED);
+        to_erase.push_back(it);
+        // Don't remove failed endpoints until the BackoffEntry okays it, to
+        // avoid
+        // hammering failing endpoints by removing and re-adding them
+        // constantly.
+      } else if (policy_.max_endpoint_failures >= 0 &&
+                 endpoint->backoff.CanDiscard() &&
+                 endpoint->backoff.failure_count() >
+                     policy_.max_endpoint_failures) {
+        HistogramEndpoint(ENDPOINT_FATE_FAILED);
+        to_erase.push_back(it);
+      }
+    }
+
+    while (endpoints_.size() - to_erase.size() > policy_.max_endpoint_count) {
+      auto oldest_it = endpoints_.end();
+
+      for (auto it = endpoints_.begin(); it != endpoints_.end(); ++it) {
+        if (it->second->pending)
+          continue;
+        if (oldest_it == endpoints_.end() ||
+            it->second->last_used < oldest_it->second->last_used) {
+          oldest_it = it;
+        }
+      }
+
+      if (oldest_it == endpoints_.end())
+        break;
+
+      HistogramEndpoint(ENDPOINT_FATE_EVICTED);
+      to_erase.push_back(oldest_it);
+
+      // Gross kludge: Keep us from picking this endpoint again.
+      oldest_it->second->pending = true;
+    }
+
+    for (auto it : to_erase)
+      endpoints_.erase(it);
+  }
+}
+
+ReportingService::Endpoint* ReportingService::FindEndpointForReport(
+    const ReportingReport& report) {
+  // TODO: This is O(count of all endpoints, regardless of client origins).
+  // TODO: The spec doesn't prioritize *.bar.foo.com over *.foo.com when
+  // choosing which endpoint to upload a report for baz.bar.foo.com to.
+  // That seems wrong, but we need clarification on the spec end.
+  for (auto& pair : endpoints_) {
+    Endpoint* endpoint = pair.second.get();
+    if (endpoint->is_expired(clock_->NowTicks()) ||
+        endpoint->backoff.ShouldRejectRequest() ||
+        !DoesEndpointMatchReport(*endpoint, report)) {
+      continue;
+    }
+    return endpoint;
+  }
+  return nullptr;
+}
+
+bool ReportingService::DoesEndpointMatchReport(const Endpoint& endpoint,
+                                               const ReportingReport& report) {
+  for (auto& pair : endpoint.clients) {
+    const Client& client = pair.second;
+    if (client.is_expired(clock_->NowTicks()))
+      continue;
+    if (!base::EqualsCaseInsensitiveASCII(client.group, report.group))
+      continue;
+    if (client.origin == report.origin)
+      return true;
+    if (client.subdomains && report.origin.DomainIs(client.origin.host_piece()))
+      return true;
+  }
+  return false;
+}
+
+std::string ReportingService::SerializeReports(
+    const std::vector<ReportingReport*>& reports) {
+  base::ListValue collection;
+  for (auto& report : reports) {
+    std::unique_ptr<base::DictionaryValue> data(new base::DictionaryValue());
+    data->SetInteger("age",
+                     (clock_->NowTicks() - report->timestamp).InMilliseconds());
+    data->SetString("type", report->type);
+    data->SetString("url", report->url.spec());
+    data->Set("report", report->body->DeepCopy());
+    collection.Append(std::move(data));
+  }
+
+  std::string json = "";
+  bool written = base::JSONWriter::Write(collection, &json);
+  DCHECK(written);
+  return json;
+}
+
+ReportingService::Endpoint* ReportingService::GetEndpointByURL(
+    const GURL& url) {
+  auto it = endpoints_.find(url);
+  if (it == endpoints_.end())
+    return nullptr;
+  return it->second.get();
+}
+
+void ReportingService::DequeueReport(ReportingReport* report) {
+  // TODO: This is O(N).
+  for (auto it = reports_.begin(); it != reports_.end(); ++it) {
+    if (it->get() == report) {
+      reports_.erase(it);
+      return;
+    }
+  }
+}
+
+void ReportingService::HistogramHeaderEndpointInternal(
+    bool endpoint_exists,
+    bool client_exists,
+    base::TimeDelta ttl) const {
+  HeaderEndpointFate fate;
+  if (ttl > base::TimeDelta()) {
+    if (client_exists)
+      fate = HEADER_ENDPOINT_FATE_SET_CLIENT_UPDATED;
+    else if (endpoint_exists)
+      fate = HEADER_ENDPOINT_FATE_SET_CLIENT_CREATED;
+    else
+      fate = HEADER_ENDPOINT_FATE_SET_ENDPOINT_CREATED;
+  } else {
+    if (client_exists)
+      fate = HEADER_ENDPOINT_FATE_CLEAR_CLIENT_REMOVED;
+    else if (endpoint_exists)
+      fate = HEADER_ENDPOINT_FATE_CLEAR_NO_CLIENT;
+    else
+      fate = HEADER_ENDPOINT_FATE_CLEAR_NO_ENDPOINT;
+  }
+  HistogramHeaderEndpoint(fate, ttl);
+}
+
+void ReportingService::HistogramReportInternal(
+    ReportFate fate,
+    const ReportingReport& report) const {
+  HistogramReport(fate, clock_->NowTicks() - report.timestamp, report.attempts);
+}
+
+}  // namespace net