Add flag "distillability-dev" for distillability development

components/dom_distiller/content/renderer/distillability_agent.cc

 // Copyright 2015 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
+#include "base/json/json_writer.h"
 #include "base/metrics/histogram.h"
 #include "base/strings/string_util.h"
 
 #include "components/dom_distiller/content/common/distillability_service.mojom.h"
 #include "components/dom_distiller/content/renderer/distillability_agent.h"
 #include "components/dom_distiller/core/distillable_page_detector.h"
+#include "components/dom_distiller/core/dom_distiller_features.h"
 #include "components/dom_distiller/core/experiments.h"
 #include "components/dom_distiller/core/page_features.h"
 #include "components/dom_distiller/core/url_utils.h"
 #include "content/public/renderer/render_frame.h"
 #include "services/shell/public/cpp/interface_provider.h"
 #include "third_party/WebKit/public/platform/WebDistillability.h"
 #include "third_party/WebKit/public/web/WebDocument.h"
 #include "third_party/WebKit/public/web/WebElement.h"
 #include "third_party/WebKit/public/web/WebLocalFrame.h"
 
@@ skipping 45 matching lines @@
 
 bool IsBlacklisted(const GURL& url) {
   for (size_t i = 0; i < arraysize(kBlacklist); ++i) {
     if (base::LowerCaseEqualsASCII(url.host(), kBlacklist[i])) {
       return true;
     }
   }
   return false;
 }
 
+void DumpDistillability(content::RenderFrame* render_frame,
+                        const WebDistillabilityFeatures& features,
+                        const std::vector<double>& derived,
+                        double score,
+                        bool distillable,
+                        double long_score,
+                        bool long_page,
+                        bool blacklisted) {
+  {
+    base::DictionaryValue dict;
+    std::string msg;
+
+    std::unique_ptr<base::DictionaryValue> raw_features(
+        new base::DictionaryValue);
+    raw_features->SetInteger("isMobileFriendly", features.isMobileFriendly);
+    raw_features->SetInteger("openGraph", features.openGraph);
+    raw_features->SetInteger("elementCount", features.elementCount);
+    raw_features->SetInteger("anchorCount", features.anchorCount);
+    raw_features->SetInteger("formCount", features.formCount);
+    raw_features->SetInteger("textInputCount", features.textInputCount);
+    raw_features->SetInteger("passwordInputCount", features.passwordInputCount);
+    raw_features->SetDouble("mozScore", features.mozScore);
+    raw_features->SetDouble("mozScoreAllSqrt", features.mozScoreAllSqrt);
+    raw_features->SetDouble("mozScoreAllLinear", features.mozScoreAllLinear);
+    dict.Set("features", std::move(raw_features));
+
+    std::unique_ptr<base::ListValue> derived_features(new base::ListValue());
+    for (unsigned i = 0; i < derived.size(); i++) {
+      derived_features->AppendDouble(derived[i]);
+    }
+    dict.Set("derived_features", std::move(derived_features));
+
+    dict.SetDouble("score", score);
+    dict.SetInteger("distillable", distillable);
+    dict.SetDouble("long_score", long_score);
+    dict.SetInteger("long_page", long_page);
+    dict.SetInteger("blacklisted", blacklisted);
+    base::JSONWriter::WriteWithOptions(dict,
+        base::JSONWriter::OPTIONS_PRETTY_PRINT, &msg);
+    msg = "adaboost_classification = " + msg;
+
+    render_frame->AddMessageToConsole(content::CONSOLE_MESSAGE_LEVEL_DEBUG,
+        msg);
+  }
+}
+
 bool IsDistillablePageAdaboost(WebDocument& doc,
                                const DistillablePageDetector* detector,
                                const DistillablePageDetector* long_page,
-                               bool is_last) {
+                               bool is_last,
+                               content::RenderFrame* render_frame) {
+  bool isDevMode = IsDistillabilityDevSet();
   WebDistillabilityFeatures features = doc.distillabilityFeatures();
   GURL parsed_url(doc.url());
   if (!parsed_url.is_valid()) {
     return false;
   }
   std::vector<double> derived = CalculateDerivedFeatures(
     features.openGraph,
     parsed_url,
     features.elementCount,
     features.anchorCount,
     features.formCount,
     features.mozScore,
     features.mozScoreAllSqrt,
     features.mozScoreAllLinear
   );
   double score = detector->Score(derived) - detector->GetThreshold();
   double long_score = long_page->Score(derived) - long_page->GetThreshold();
   bool distillable = score > 0;
   bool long_article = long_score > 0;
   bool blacklisted = IsBlacklisted(parsed_url);
 
+  if (isDevMode) {
+    DumpDistillability(render_frame, features, derived, score, distillable,
+        long_score, long_article, blacklisted);
+  }
+
   if (!features.isMobileFriendly) {
     int score_int = std::round(score * 100);
     if (score > 0) {
       UMA_HISTOGRAM_COUNTS_1000("DomDistiller.DistillabilityScoreNMF.Positive",
           score_int);
     } else {
       UMA_HISTOGRAM_COUNTS_1000("DomDistiller.DistillabilityScoreNMF.Negative",
           -score_int);
     }
     if (distillable) {
@@ skipping 39 matching lines @@
 
   if (blacklisted) {
     return false;
   }
   if (features.isMobileFriendly) {
     return false;
   }
   return distillable && long_article;
 }
 
-bool IsDistillablePage(WebDocument& doc, bool is_last) {
+bool IsDistillablePage(WebDocument& doc, bool is_last,
+    content::RenderFrame* render_frame) {
   switch (GetDistillerHeuristicsType()) {
     case DistillerHeuristicsType::ALWAYS_TRUE:
       return true;
     case DistillerHeuristicsType::OG_ARTICLE:
       return doc.distillabilityFeatures().openGraph;
     case DistillerHeuristicsType::ADABOOST_MODEL:
       return IsDistillablePageAdaboost(doc,
           DistillablePageDetector::GetNewModel(),
-          DistillablePageDetector::GetLongPageModel(), is_last);
+          DistillablePageDetector::GetLongPageModel(), is_last, render_frame);
     case DistillerHeuristicsType::NONE:
     default:
       return false;
   }
 }
 
 }  // namespace
 
 DistillabilityAgent::DistillabilityAgent(
     content::RenderFrame* render_frame)
@@ skipping 17 matching lines @@
   bool is_loaded = layout_type == WebMeaningfulLayout::FinishedLoading;
   if (!NeedToUpdate(is_loaded)) return;
 
   bool is_last = IsLast(is_loaded);
   // Connect to Mojo service on browser to notify page distillability.
   mojom::DistillabilityServicePtr distillability_service;
   render_frame()->GetRemoteInterfaces()->GetInterface(
       &distillability_service);
   DCHECK(distillability_service);
   distillability_service->NotifyIsDistillable(
-      IsDistillablePage(doc, is_last), is_last);
+      IsDistillablePage(doc, is_last, render_frame()), is_last);
 }
 
 DistillabilityAgent::~DistillabilityAgent() {}
 
 void DistillabilityAgent::OnDestruct() {
   delete this;
 }
 
 }  // namespace dom_distiller