[OBSOLETE] Reporting: Initial implementation.

net/reporting/reporting_service.cc

+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "net/reporting/reporting_service.h"
+
+#include "base/bind.h"
+#include "base/json/json_reader.h"
+#include "base/json/json_writer.h"
+#include "base/memory/ptr_util.h"
+#include "base/metrics/histogram_macros.h"
+#include "base/strings/string_number_conversions.h"
+#include "base/strings/string_util.h"
+#include "base/time/default_tick_clock.h"
+#include "net/http/http_request_info.h"
+#include "net/http/http_response_headers.h"
+#include "net/http/http_response_info.h"
+#include "net/url_request/url_fetcher.h"
+#include "net/url_request/url_request_context_getter.h"
+
+namespace {
+
+const char kDefaultGroupName[] = "default";
+
+// Per
+// https://greenbytes.de/tech/webdav/draft-reschke-http-jfv-02.html#rfc.section.4
+// assuming |normalized_header| is the result of completing step 1.
+std::unique_ptr<base::Value> ParseJFV(
+    const std::string& normalized_header_value) {
+  std::string value = "[" + normalized_header_value + "]";
+  return base::JSONReader::Read(value);
+}
+
+bool IsOriginSecure(const GURL& url) {
+  return url.SchemeIsCryptographic();
+}
+
+}  // namespace
+
+namespace net {
+
+ReportingService::Policy::Policy()
+    : endpoint_lifetime(base::TimeDelta::FromDays(7)),
+      max_endpoint_failures(5),
+      max_endpoint_count(100),
+      report_lifetime(base::TimeDelta::FromHours(1)),
+      max_report_failures(5),
+      max_report_count(100),
+      persist_reports_across_network_changes(false) {
+  endpoint_backoff.num_errors_to_ignore = 0;
+  endpoint_backoff.initial_delay_ms = 5 * 1000;
+  endpoint_backoff.multiply_factor = 2.0;
+  endpoint_backoff.jitter_factor = 0.1;
+  endpoint_backoff.maximum_backoff_ms = 60 * 60 * 1000;
+  endpoint_backoff.entry_lifetime_ms = 7 * 24 * 60 * 60 * 1000;
+  endpoint_backoff.always_use_initial_delay = false;
+}
+
+// static
+ReportingService::Policy ReportingService::Policy::GetNonBrowserDefault() {
+  Policy policy;
+  // In non-browser contexts, there is less concern that Reporting could
+  // accidentally track users across networks, so don't discard reports as
+  // readily.
+  policy.report_lifetime = base::TimeDelta::FromDays(2);
+  policy.persist_reports_across_network_changes = true;
+  return policy;
+}
+
+struct ReportingService::Client {
+ public:
+  Client(const GURL& origin,
+         bool subdomains,
+         const std::string& group,
+         base::TimeDelta ttl,
+         base::TimeTicks creation)
+      : origin(origin),
+        subdomains(subdomains),
+        group(group),
+        ttl(ttl),
+        creation(creation) {}
+
+  GURL origin;
+  bool subdomains;
+  std::string group;
+  base::TimeDelta ttl;
+  base::TimeTicks creation;
+
+  bool is_expired(base::TimeTicks now) const { return creation + ttl < now; }
+};
+
+struct ReportingService::Endpoint {
+ public:
+  Endpoint(const GURL& url,
+           const BackoffEntry::Policy& backoff_policy,
+           base::TickClock* clock)
+      : url(url),
+        backoff(&backoff_policy, clock),
+        last_used(clock->NowTicks()),
+        pending(false) {}
+  ~Endpoint() {}
+
+  const GURL url;
+
+  BackoffEntry backoff;
+  // For LRU eviction of endpoints.
+  base::TimeTicks last_used;
+  // Whether we currently have an upload in progress to this endpoint.
+  bool pending;
+
+  // Map from client.origin to client.
+  std::map<GURL, Client> clients;
+
+  bool is_expired(base::TimeTicks now) const {
+    for (auto& pair : clients)
+      if (!pair.second.is_expired(now))
+        return false;
+    return true;
+  }
+};
+
+class ReportingService::Report {
+ public:
+  Report() = default;
+  ~Report() = default;
+
+  // Report body. Can be any valid JSON. Passed to endpoint unmodified.
+  std::unique_ptr<base::Value> body;
+
+  // URL of content that triggered report. Passed to endpoint unmodified.
+  GURL url;
+
+  // Origin of content that triggered report. Should be url.GetOrigin(), but
+  // the spec doesn't guarantee this, and allows callers to provide URL and
+  // origin separately, so they are separate fields for now. Passed to
+  // endpoint unmodified.
+  GURL origin;
+
+  // Report group. Used to select endpoints for upload.
+  std::string group;
+
+  // Report type. Passed to endpoint unmodified.
+  std::string type;
+
+  // Timestamp when the report was queued. Passed to endpoint as a relative
+  // age.
+  base::TimeTicks queued;
+
+  // Number of delivery attempts made.
+  size_t attempts;
+
+  // Whether this report is currently part of a pending delivery attempt.
+  bool pending;
+
+ private:
+  DISALLOW_COPY_AND_ASSIGN(Report);
+};
+
+struct ReportingService::EndpointTuple {
+  GURL url;
+  bool subdomains;
+  base::TimeDelta ttl;
+  std::string group;
+
+  static bool FromDictionary(const base::DictionaryValue& dictionary,
+                             EndpointTuple* tuple_out,
+                             std::string* error_out);
+  static bool FromHeader(const std::string& header,
+                         std::vector<EndpointTuple>* tuples_out,
+                         std::vector<std::string>* errors_out);
+};
+
+struct ReportingService::Delivery {
+  Delivery(const GURL& endpoint_url, const std::vector<Report*>& reports)
+      : endpoint_url(endpoint_url), reports(reports) {}
+  ~Delivery() = default;
+
+  const GURL& endpoint_url;
+  const std::vector<Report*> reports;
+};
+
+ReportingService::ReportingService(const Policy& policy)
+    : policy_(policy), clock_(new base::DefaultTickClock()) {
+  NetworkChangeNotifier::AddNetworkChangeObserver(this);
+}
+
+ReportingService::~ReportingService() {
+  NetworkChangeNotifier::RemoveNetworkChangeObserver(this);
+}
+
+void ReportingService::set_uploader(
+    std::unique_ptr<ReportingUploader> uploader) {
+  uploader_ = std::move(uploader);
+}
+
+void ReportingService::QueueReport(std::unique_ptr<base::Value> body,
+                                   const GURL& url,
+                                   const GURL& origin,
+                                   const std::string& group,
+                                   const std::string& type) {
+  auto report = base::MakeUnique<Report>();
+  report->body = std::move(body);
+  report->url = url.GetAsReferrer();
+  report->origin = origin;
+  report->group = group;
+  report->type = type;
+  report->queued = clock_->NowTicks();
+  report->attempts = 0;
+  report->pending = false;
+  reports_.push_back(std::move(report));
+
+  CollectGarbage();
+}
+
+void ReportingService::ProcessHeader(const GURL& origin,
+                                     const std::string& header_value) {
+  if (!IsOriginSecure(origin))
+    return;
+
+  std::vector<std::string> errors;
+  std::vector<EndpointTuple> tuples;
+  if (!EndpointTuple::FromHeader(header_value, &tuples, &errors))
+    return;
+
+  // TODO: Plumb these out to the DevTools console somehow.
+  for (const std::string& error : errors) {
+    LOG(WARNING) << "Origin " << origin.spec() << " sent "
+                 << "Report-To header with error: " << error << ": "
+                 << header_value;
+  }
+
+  for (auto& tuple : tuples)
+    ProcessEndpointTuple(origin, tuple);
+
+  CollectGarbage();
+}
+
+void ReportingService::SendReports() {
+  if (!uploader_)
+    return;
+
+  base::TimeTicks now = clock_->NowTicks();
+
+  std::map<Endpoint*, std::vector<Report*>> endpoint_reports;
+  for (auto& report : reports_) {
+    // If the report is already contained in another pending upload, don't
+    // upload it twice.
+    if (report->pending)
+      continue;
+
+    Endpoint* endpoint = FindEndpointForReport(*report);
+    // If there's no available endpoint for the report, leave it for later.
+    if (!endpoint)
+      continue;
+
+    // If the chosen endpoint is pending, don't start another upload; let this
+    // report go in the next upload instead.
+    if (endpoint->pending)
+      continue;
+
+    report->pending = true;
+    endpoint_reports[endpoint].push_back(report.get());
+  }
+
+  for (auto& pair : endpoint_reports) {
+    Endpoint* endpoint = pair.first;
+    const std::vector<Report*>& reports = pair.second;
+
+    std::string json = SerializeReports(reports);
+
+    uploader_->AttemptDelivery(
+        endpoint->url, json,
+        base::Bind(&ReportingService::OnDeliveryAttemptComplete,
+                   base::Unretained(this),
+                   base::MakeUnique<Delivery>(endpoint->url, reports)));
+
+    endpoint->last_used = now;
+
+    for (auto& report : reports)
+      ++report->attempts;
+  }
+}
+
+void ReportingService::OnNetworkChanged(
+    NetworkChangeNotifier::ConnectionType connection_type) {
+  if (policy_.persist_reports_across_network_changes)
+    return;
+
+  reports_.clear();
+}
+
+void ReportingService::set_clock_for_testing(
+    std::unique_ptr<base::TickClock> clock) {
+  clock_ = std::move(clock);
+}
+
+size_t ReportingService::GetReportCountForTesting() {
+  return reports_.size();
+}
+
+bool ReportingService::HasEndpointForTesting(const GURL& endpoint_url) {
+  return GetEndpointByURL(endpoint_url) != nullptr;
+}
+
+bool ReportingService::HasClientForTesting(const GURL& endpoint_url,
+                                           const GURL& origin) {
+  Endpoint* endpoint = GetEndpointByURL(endpoint_url);
+  if (!endpoint)
+    return false;
+  return endpoint->clients.count(origin) > 0;
+}
+
+int ReportingService::GetEndpointFailuresForTesting(const GURL& endpoint_url) {
+  Endpoint* endpoint = GetEndpointByURL(endpoint_url);
+  if (!endpoint)
+    return -1;
+  return endpoint->backoff.failure_count();
+}
+
+void ReportingService::CollectGarbageForTesting() {
+  CollectGarbage();
+}
+
+// static
+bool ReportingService::EndpointTuple::FromDictionary(
+    const base::DictionaryValue& dictionary,
+    EndpointTuple* tuple_out,
+    std::string* error_out) {
+  if (!dictionary.HasKey("url")) {
+    *error_out = "url missing";
+    return false;
+  }
+  std::string url_string;
+  if (!dictionary.GetString("url", &url_string)) {
+    *error_out = "url not a string";
+    return false;
+  }
+  tuple_out->url = GURL(url_string);
+
+  tuple_out->group = kDefaultGroupName;
+  if (dictionary.HasKey("group")) {
+    if (!dictionary.GetString("group", &tuple_out->group)) {
+      *error_out = "group present but not a string";
+      return false;
+    }
+  }
+
+  tuple_out->subdomains = false;
+  if (dictionary.HasKey("includeSubdomains")) {
+    if (!dictionary.GetBoolean("includeSubdomains", &tuple_out->subdomains)) {
+      *error_out = "includeSubdomains present but not boolean";
+      return false;
+    }
+  }
+
+  if (!dictionary.HasKey("max-age")) {
+    *error_out = "max-age missing";
+    return false;
+  }
+  int ttl_sec;
+  if (!dictionary.GetInteger("max-age", &ttl_sec)) {
+    *error_out = "max-age not an integer";
+    return false;
+  }
+  tuple_out->ttl = base::TimeDelta::FromSeconds(ttl_sec);
+
+  return true;
+}
+
+// static
+bool ReportingService::EndpointTuple::FromHeader(
+    const std::string& header,
+    std::vector<ReportingService::EndpointTuple>* tuples_out,
+    std::vector<std::string>* errors_out) {
+  tuples_out->clear();
+  errors_out->clear();
+
+  std::unique_ptr<base::Value> value(ParseJFV(header));
+  if (!value) {
+    errors_out->push_back("failed to parse JSON field value.");
+    return false;
+  }
+
+  base::ListValue* list;
+  bool was_list = value->GetAsList(&list);
+  DCHECK(was_list);
+
+  base::DictionaryValue* item;
+  for (size_t i = 0; i < list->GetSize(); i++) {
+    std::string error_prefix = "endpoint " + base::SizeTToString(i + 1) +
+                               " of " + base::SizeTToString(list->GetSize()) +
+                               ": ";
+    if (!list->GetDictionary(i, &item)) {
+      errors_out->push_back(error_prefix + "is not a dictionary");
+      continue;
+    }
+    EndpointTuple tuple;
+    std::string error;
+    if (!EndpointTuple::FromDictionary(*item, &tuple, &error)) {
+      errors_out->push_back(error_prefix + error);
+      continue;
+    }
+    if (!IsOriginSecure(tuple.url)) {
+      errors_out->push_back(error_prefix + "url " + tuple.url.spec() +
+                            " is insecure");
+      continue;
+    }
+    if (tuple.ttl < base::TimeDelta()) {
+      errors_out->push_back(error_prefix + "ttl is negative");
+      continue;
+    }
+    tuples_out->push_back(tuple);
+  }
+  return true;
+}
+
+void ReportingService::ProcessEndpointTuple(const GURL& origin,
+                                            const EndpointTuple& tuple) {
+  Endpoint* endpoint = GetEndpointByURL(tuple.url);
+
+  bool endpoint_exists = endpoint != nullptr;
+  bool client_exists = endpoint && endpoint->clients.count(origin) > 0;
+
+  if (client_exists)
+    endpoint->clients.erase(origin);
+
+  if (tuple.ttl <= base::TimeDelta())
+    return;
+
+  if (!endpoint_exists) {
+    endpoint = new Endpoint(tuple.url, policy_.endpoint_backoff, clock_.get());
+    endpoints_.insert(std::make_pair(tuple.url, base::WrapUnique(endpoint)));
+  }
+
+  base::TimeTicks now = clock_->NowTicks();
+
+  Client client(origin, tuple.subdomains, tuple.group, tuple.ttl, now);
+  endpoint->clients.insert(std::make_pair(origin, client));
+}
+
+void ReportingService::OnDeliveryAttemptComplete(
+    const std::unique_ptr<Delivery>& delivery,
+    ReportingUploader::Outcome outcome) {
+  for (auto report : delivery->reports) {
+    DCHECK(report->pending);
+    report->pending = false;
+  }
+
+  Endpoint* endpoint = GetEndpointByURL(delivery->endpoint_url);
+  if (endpoint) {
+    endpoint->backoff.InformOfRequest(outcome == ReportingUploader::SUCCESS);
+    endpoint->pending = false;
+  }
+
+  switch (outcome) {
+    case ReportingUploader::SUCCESS:
+      for (auto report : delivery->reports)
+        DequeueReport(report);
+      break;
+    case ReportingUploader::FAILURE:
+      // Reports have been marked not-pending and can be retried later.
+      // BackoffEntry has been informed of failure.
+      break;
+    case ReportingUploader::REMOVE_ENDPOINT:
+      // Note: This is not specified, but seems the obvious intention.
+      if (endpoint)
+        endpoints_.erase(delivery->endpoint_url);
+      break;
+  }
+
+  CollectGarbage();
+}
+
+void ReportingService::CollectGarbage() {
+  base::TimeTicks now = clock_->NowTicks();
+
+  {
+    std::vector<Report*> to_dequeue;
+
+    for (auto it = reports_.begin(); it != reports_.end(); ++it) {
+      Report* report = it->get();
+      base::TimeDelta age = now - report->queued;
+
+      if (report->pending)
+        continue;
+
+      if (report->attempts > policy_.max_report_failures ||
+          age > policy_.report_lifetime) {
+        to_dequeue.push_back(report);
+      }
+    }
+
+    for (auto it = reports_.begin();
+         it != reports_.end() &&
+         reports_.size() - to_dequeue.size() > policy_.max_report_count;
+         ++it) {
+      if (it->get()->pending)
+        continue;
+
+      to_dequeue.push_back(it->get());
+    }
+
+    for (auto it : to_dequeue)
+      DequeueReport(it);
+  }
+
+  // TODO: Individually garbage-collect clients.
+
+  {
+    std::vector<EndpointMap::iterator> to_erase;
+    for (auto it = endpoints_.begin(); it != endpoints_.end(); ++it) {
+      Endpoint* endpoint = it->second.get();
+
+      if (endpoint->pending)
+        continue;
+
+      // Don't remove failed endpoints until the BackoffEntry okays it, to
+      // avoid hammering failing endpoints by removing and re-adding them
+      // constantly.
+      if (endpoint->is_expired(now) ||
+          now - endpoint->last_used > policy_.endpoint_lifetime ||
+          (endpoint->backoff.CanDiscard() &&
+           endpoint->backoff.failure_count() > policy_.max_endpoint_failures)) {
+        to_erase.push_back(it);
+      }
+    }
+
+    while (endpoints_.size() - to_erase.size() > policy_.max_endpoint_count) {
+      auto oldest_it = endpoints_.end();
+
+      for (auto it = endpoints_.begin(); it != endpoints_.end(); ++it) {
+        if (it->second->pending)
+          continue;
+
+        if (oldest_it == endpoints_.end() ||
+            it->second->last_used < oldest_it->second->last_used) {
+          oldest_it = it;
+        }
+      }
+
+      if (oldest_it == endpoints_.end())
+        break;
+
+      to_erase.push_back(oldest_it);
+
+      // Gross kludge: Keep us from choosing this endpoint again.
+      oldest_it->second->pending = true;
+    }
+
+    for (auto it : to_erase)
+      endpoints_.erase(it);
+  }
+}
+
+ReportingService::Endpoint* ReportingService::FindEndpointForReport(
+    const Report& report) {
+  // TODO: This is O(count of all endpoints, regardless of client origins).
+  // TODO: The spec doesn't prioritize *.bar.foo.com over *.foo.com when
+  // choosing which endpoint to upload a report for baz.bar.foo.com to.
+  // That seems wrong, but we need clarification on the spec end.
+  for (auto& pair : endpoints_) {
+    Endpoint* endpoint = pair.second.get();
+    if (endpoint->is_expired(clock_->NowTicks()) ||
+        endpoint->backoff.ShouldRejectRequest() ||
+        !DoesEndpointMatchReport(*endpoint, report)) {
+      continue;
+    }
+    return endpoint;
+  }
+  return nullptr;
+}
+
+bool ReportingService::DoesEndpointMatchReport(const Endpoint& endpoint,
+                                               const Report& report) {
+  for (auto& pair : endpoint.clients) {
+    const Client& client = pair.second;
+    if (client.is_expired(clock_->NowTicks()))
+      continue;
+    if (!base::EqualsCaseInsensitiveASCII(client.group, report.group))
+      continue;
+    if (client.origin == report.origin)
+      return true;
+    if (client.subdomains && report.origin.DomainIs(client.origin.host_piece()))
+      return true;
+  }
+  return false;
+}
+
+std::string ReportingService::SerializeReports(
+    const std::vector<Report*>& reports) {
+  base::ListValue collection;
+  for (auto& report : reports) {
+    std::unique_ptr<base::DictionaryValue> data(new base::DictionaryValue());
+    data->SetInteger("age",
+                     (clock_->NowTicks() - report->queued).InMilliseconds());
+    data->SetString("type", report->type);
+    data->SetString("url", report->url.spec());
+    data->Set("report", report->body->DeepCopy());
+    collection.Append(std::move(data));
+  }
+
+  std::string json = "";
+  bool written = base::JSONWriter::Write(collection, &json);
+  DCHECK(written);
+  return json;
+}
+
+ReportingService::Endpoint* ReportingService::GetEndpointByURL(
+    const GURL& url) {
+  auto it = endpoints_.find(url);
+  if (it == endpoints_.end())
+    return nullptr;
+  return it->second.get();
+}
+
+void ReportingService::DequeueReport(Report* report) {
+  // TODO: This is O(N).
+  for (auto it = reports_.begin(); it != reports_.end(); ++it) {
+    if (it->get() == report) {
+      reports_.erase(it);
+      return;
+    }
+  }
+}
+
+}  // namespace net