[PageLoadMetrics] Keep track of Ad Sizes on Pages

chrome/browser/page_load_metrics/observers/ads_page_load_metrics_observer.cc

+// Copyright 2017 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "chrome/browser/page_load_metrics/observers/ads_page_load_metrics_observer.h"
+
+#include <string>
+
+#include "base/feature_list.h"
+#include "base/logging.h"
+#include "base/memory/ptr_util.h"
+#include "base/strings/string_util.h"
+#include "chrome/browser/page_load_metrics/page_load_metrics_util.h"
+#include "content/public/browser/navigation_handle.h"
+#include "content/public/browser/render_frame_host.h"
+#include "content/public/browser/web_contents.h"
+#include "url/gurl.h"
+
+namespace {
+
+const base::Feature kAdsFeature{"AdsMetrics", base::FEATURE_ENABLED_BY_DEFAULT};
+
+bool FrameIsAd(content::NavigationHandle* navigation_handle) {
+  content::RenderFrameHost* current_frame_host =
+      navigation_handle->GetRenderFrameHost();
+  DCHECK(current_frame_host);
+  const std::string& name = current_frame_host->GetFrameName();
+  const GURL& url = navigation_handle->GetURL();
+
+  // Because sub-resource filtering isn't always enabled, and doesn't work
+  // well in monitoring mode (no CSS enforcement), it's difficult to identify
+  // ads. Google ads are prevalent and easy to track, so we'll start by
+  // tracking those. Note that the frame name can be very large, so be careful
+  // to avoid full string searches if possible.
+  // TODO(jkarlin): Track other ad networks that are easy to identify.
+  return base::StartsWith(name, "google_ads_iframe",
+                          base::CompareCase::SENSITIVE) ||
+         base::StartsWith(name, "google_ads_frame",
+                          base::CompareCase::SENSITIVE) ||
+         (url.host_piece() == "tpc.googlesyndication.com" &&
+          base::StartsWith(url.path_piece(), "/safeframe",
+                           base::CompareCase::SENSITIVE));
+}
+
+}  // namespace
+
+// static
+std::unique_ptr<AdsPageLoadMetricsObserver>
+AdsPageLoadMetricsObserver::CreateIfNeeded() {
+  if (!base::FeatureList::IsEnabled(kAdsFeature))
+    return nullptr;
+  return base::MakeUnique<AdsPageLoadMetricsObserver>();
+}
+
+AdsPageLoadMetricsObserver::AdsPageLoadMetricsObserver() = default;
+AdsPageLoadMetricsObserver::~AdsPageLoadMetricsObserver() = default;
+
+page_load_metrics::PageLoadMetricsObserver::ObservePolicy
+AdsPageLoadMetricsObserver::OnCommit(
+    content::NavigationHandle* navigation_handle) {
+  DCHECK(ad_frames_data_.empty());
+
+  // The main frame is never considered an ad.
+  ad_frames_data_[navigation_handle->GetFrameTreeNodeId()] = nullptr;
+  ProcessOngoingNavigationResource(navigation_handle->GetFrameTreeNodeId());
+  return CONTINUE_OBSERVING;
+}
+
+page_load_metrics::PageLoadMetricsObserver::ObservePolicy
+AdsPageLoadMetricsObserver::OnDidFinishSubFrameNavigation(
+    content::NavigationHandle* navigation_handle) {
+  FrameTreeNodeId frame_tree_node_id = navigation_handle->GetFrameTreeNodeId();
+
+  if (!navigation_handle->HasCommitted() ||
+      navigation_handle->IsSameDocument() || navigation_handle->IsErrorPage()) {
+    // We're not interested in tracking this navigation. In case we've seen a
+    // resource for the navigation before this message, clear it from
+    // ongoing_navigation_resources_.
+    ongoing_navigation_resources_.erase(frame_tree_node_id);
+    return CONTINUE_OBSERVING;
+  }
+
+  content::RenderFrameHost* parent_frame_host =
+      navigation_handle->GetRenderFrameHost()->GetParent();
+  DCHECK(parent_frame_host);
+
+  bool top_level_subframe = !parent_frame_host->GetParent();
+
+  const auto& id_and_data = ad_frames_data_.find(frame_tree_node_id);
+  if (id_and_data != ad_frames_data_.end()) {
+    // An existing subframe is navigating again.
+    if (id_and_data->second) {
+      // The subframe was an ad to begin with, keep tracking it as an ad.
+      ProcessOngoingNavigationResource(frame_tree_node_id);
+      return CONTINUE_OBSERVING;
+    }
+    // This frame was previously not an ad, process it as usual. If it had
+    // any child frames that were ads, those will still be recorded.
+  } else if (top_level_subframe) {
+    top_level_subframe_count_ += 1;
+  }
+
+  // Determine who the parent frame's ad ancestor is.
+  const auto& parent_id_and_data =
+      ad_frames_data_.find(parent_frame_host->GetFrameTreeNodeId());
+  DCHECK(parent_id_and_data != ad_frames_data_.end());
+  AdFrameData* ad_data = parent_id_and_data->second;
+
+  if (!ad_data && FrameIsAd(navigation_handle)) {
+    // This frame is not nested within an ad frame but is itself an ad.
+    ad_frames_data_storage_.emplace_back();

[Charlie Harrison, 2017/04/24 19:18:26]

It looks like this call can initialize the storage with uninitialized values.
Can you make sure the struct initializes the data to 0 in the header?

[jkarlin, 2017/04/25 15:56:50]

Eek!. I thought it was initialized. Thanks!

+    ad_data = &ad_frames_data_storage_.back();
+  }
+
+  ad_frames_data_[frame_tree_node_id] = ad_data;
+
+  if (top_level_subframe && ad_data)
+    top_level_ad_frame_count_ += 1;
+
+  ProcessOngoingNavigationResource(frame_tree_node_id);
+  return CONTINUE_OBSERVING;
+}
+
+page_load_metrics::PageLoadMetricsObserver::ObservePolicy
+AdsPageLoadMetricsObserver::FlushMetricsOnAppEnterBackground(
+    const page_load_metrics::PageLoadTiming& timing,
+    const page_load_metrics::PageLoadExtraInfo& extra_info) {
+  // The browser may come back, but there is no guarantee. To be safe, record
+  // what we have now and ignore future changes to this navigation.
+  if (extra_info.did_commit)
+    RecordHistograms();
+
+  return STOP_OBSERVING;
+}
+
+void AdsPageLoadMetricsObserver::OnLoadedResource(
+    const page_load_metrics::ExtraRequestInfo& extra_request_info) {
+  ProcessLoadedResource(extra_request_info);
+}
+
+void AdsPageLoadMetricsObserver::OnComplete(
+    const page_load_metrics::PageLoadTiming& timing,
+    const page_load_metrics::PageLoadExtraInfo& info) {
+  RecordHistograms();
+}
+
+void AdsPageLoadMetricsObserver::ProcessLoadedResource(
+    const page_load_metrics::ExtraRequestInfo& extra_request_info) {
+  if (!extra_request_info.url.SchemeIsHTTPOrHTTPS()) {
+    // Data uris should be accounted for in the generating resource, not
+    // here. Blobs for PLZNavigate shouldn't be counted as the http resource

[Charlie Harrison, 2017/04/24 19:18:26]

nit: s/PLZ/Plz

[jkarlin, 2017/04/25 15:56:49]

Done.

+    // was already counted. Blobs for other things like CacheStorage or
+    // IndexedDB are also ignored for now, as they're not normal HTTP loads.
+    return;
+  }
+
+  const auto& id_and_data =
+      ad_frames_data_.find(extra_request_info.frame_tree_node_id);
+  if (id_and_data == ad_frames_data_.end()) {
+    // This resouce is for a frame that hasn't yet committed. It must be the
+    // main document for the frame. Hold onto it and once it commits we'll run
+    // it in ProcessOngoingNavigationResource.
+    // TODO(jkarlin): Plumb the resource type through and DCHECK that the type
+    // is document.
+    auto it_and_success = ongoing_navigation_resources_.emplace(

[Charlie Harrison, 2017/04/24 19:18:25]

I thought map emplace wasn't allowed in Chromium yet?

[jkarlin, 2017/04/25 15:56:50]

I actually had assumed it was allowed (as it's allowed for vectors) but until
yesterday it wasn't allowed for maps. But now it is :)

+        std::piecewise_construct,

[Charlie Harrison, 2017/04/24 19:18:26]

#include <utility>

[jkarlin, 2017/04/25 15:56:50]

Done.

+        std::forward_as_tuple(extra_request_info.frame_tree_node_id),
+        std::forward_as_tuple(

[Charlie Harrison, 2017/04/24 19:18:26]

This seems a bit weird. Are we constructing a new ExtraRequestInfo here? Maybe
we should just copy the original.

[jkarlin, 2017/04/25 15:56:50]

We're creating a new one yes, but that's always required for a map. At least
with emplace we only construct once instead of twice. Also, since
ExtraRequestInfo is now non-copyable, we have little choice.

+            extra_request_info.url, extra_request_info.frame_tree_node_id,
+            extra_request_info.was_cached, extra_request_info.raw_body_bytes,
+            extra_request_info.original_network_content_length, nullptr));
+    DCHECK(it_and_success.second);
+    return;
+  }
+
+  page_bytes_ += extra_request_info.raw_body_bytes;
+  if (!extra_request_info.was_cached)
+    uncached_page_bytes_ += extra_request_info.raw_body_bytes;
+
+  // Determine if the frame (or its ancestor) is an ad, if so attribute the
+  // bytes to the highest ad ancestor.
+  AdFrameData* ancestor_data = id_and_data->second;
+
+  if (ancestor_data) {
+    ancestor_data->frame_bytes += extra_request_info.raw_body_bytes;
+    if (!extra_request_info.was_cached) {
+      ancestor_data->frame_bytes_uncached += extra_request_info.raw_body_bytes;
+    }
+  }
+}
+
+void AdsPageLoadMetricsObserver::RecordHistograms() {
+  if (page_bytes_ == 0)
+    return;
+
+  size_t total_ad_frame_bytes = 0;
+  size_t uncached_ad_frame_bytes = 0;
+
+  UMA_HISTOGRAM_COUNTS_1000("PageLoad.Clients.Ads.Google.AdFrameCount",
+                            ad_frames_data_storage_.size());
+
+  // Don't post UMA for pages that don't have ads.
+  if (ad_frames_data_storage_.empty())
+    return;
+
+  for (const AdFrameData& ad_frame_data : ad_frames_data_storage_) {
+    total_ad_frame_bytes += ad_frame_data.frame_bytes;
+    uncached_ad_frame_bytes += ad_frame_data.frame_bytes_uncached;
+
+    PAGE_BYTES_HISTOGRAM("PageLoad.Clients.Ads.Google.Bytes.PerAdFrame.Total",
+                         ad_frame_data.frame_bytes);
+    PAGE_BYTES_HISTOGRAM("PageLoad.Clients.Ads.Google.Bytes.PerAdFrame.Network",
+                         ad_frame_data.frame_bytes_uncached);
+    if (ad_frame_data.frame_bytes > 0) {
+      UMA_HISTOGRAM_PERCENTAGE(
+          "PageLoad.Clients.Ads.Google.Bytes.PercentAdFrame.Network",
+          ad_frame_data.frame_bytes_uncached * 100 / ad_frame_data.frame_bytes);
+    }
+  }
+
+  UMA_HISTOGRAM_COUNTS_1000(
+      "PageLoad.Clients.Ads.Google.SubFrameCount.TopLevel",
+      top_level_subframe_count_);
+  UMA_HISTOGRAM_COUNTS_1000("PageLoad.Clients.Ads.Google.AdFrameCount.TopLevel",
+                            top_level_ad_frame_count_);
+
+  DCHECK_LT(0, top_level_subframe_count_);  // Because ad frames isn't empty.
+  UMA_HISTOGRAM_PERCENTAGE(
+      "PageLoad.Clients.Ads.Google.PercentSubFramesAreAdFrames.TopLevel",
+      top_level_ad_frame_count_ * 100 / top_level_subframe_count_);
+
+  PAGE_BYTES_HISTOGRAM("PageLoad.Clients.Ads.Google.Bytes.NonAdFrames.Total",
+                       page_bytes_ - total_ad_frame_bytes);
+
+  PAGE_BYTES_HISTOGRAM("PageLoad.Clients.Ads.Google.Bytes.Total", page_bytes_);
+  PAGE_BYTES_HISTOGRAM("PageLoad.Clients.Ads.Google.Bytes.Network",
+                       uncached_page_bytes_);
+  if (page_bytes_) {
+    UMA_HISTOGRAM_PERCENTAGE(
+        "PageLoad.Clients.Ads.Google.Bytes.PercentPage.AllAdFrames.Total",
+        total_ad_frame_bytes * 100 / page_bytes_);
+  }
+  if (uncached_page_bytes_ > 0) {
+    UMA_HISTOGRAM_PERCENTAGE(
+        "PageLoad.Clients.Ads.Google.Bytes."
+        "PercentPage.AllAdFrames.Network",
+        uncached_ad_frame_bytes * 100 / uncached_page_bytes_);
+  }
+
+  PAGE_BYTES_HISTOGRAM("PageLoad.Clients.Ads.Google.Bytes.AllAdFrames.Total",
+                       total_ad_frame_bytes);
+  PAGE_BYTES_HISTOGRAM("PageLoad.Clients.Ads.Google.Bytes.AllAdFrames.Network",
+                       uncached_ad_frame_bytes);
+
+  if (total_ad_frame_bytes) {
+    UMA_HISTOGRAM_PERCENTAGE(
+        "PageLoad.Clients.Ads.Google.Bytes.PercentAllAdFrames.Network",
+        uncached_ad_frame_bytes * 100 / total_ad_frame_bytes);
+  }
+}
+
+void AdsPageLoadMetricsObserver::ProcessOngoingNavigationResource(
+    FrameTreeNodeId frame_tree_node_id) {
+  const auto& frame_id_and_request =
+      ongoing_navigation_resources_.find(frame_tree_node_id);
+  if (frame_id_and_request == ongoing_navigation_resources_.end())
+    return;
+
+  const page_load_metrics::ExtraRequestInfo& data =
+      frame_id_and_request->second;
+  ProcessLoadedResource(data);
+  ongoing_navigation_resources_.erase(frame_id_and_request);
+}