Add flag "distillability-dev" for distillability development

components/dom_distiller/content/renderer/distillability_agent.cc

 // Copyright 2015 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "base/metrics/histogram.h"
+#include "base/strings/string_number_conversions.h"
 #include "base/strings/string_util.h"
 
 #include "components/dom_distiller/content/common/distillability_service.mojom.h"
 #include "components/dom_distiller/content/renderer/distillability_agent.h"
 #include "components/dom_distiller/core/distillable_page_detector.h"
+#include "components/dom_distiller/core/dom_distiller_features.h"
 #include "components/dom_distiller/core/experiments.h"
 #include "components/dom_distiller/core/page_features.h"
 #include "components/dom_distiller/core/url_utils.h"
 #include "content/public/common/service_registry.h"
 #include "content/public/renderer/render_frame.h"
 
 #include "third_party/WebKit/public/platform/WebDistillability.h"
 #include "third_party/WebKit/public/web/WebDocument.h"
 #include "third_party/WebKit/public/web/WebElement.h"
 #include "third_party/WebKit/public/web/WebLocalFrame.h"
@@ skipping 37 matching lines @@
 
 bool IsBlacklisted(const GURL& url) {
   for (size_t i = 0; i < arraysize(kBlacklist); ++i) {
     if (base::LowerCaseEqualsASCII(url.host(), kBlacklist[i])) {
       return true;
     }
   }
   return false;
 }
 
+std::string DoubleToString(double v) {
+  // base::DoubleToString() returns format like ".1", which is invalid
+  // in python json parser.
+  std::stringstream ss;

[esprehn, 2016/05/23 20:19:42]

this is terrible, can we get an argument added to the base::DoubleToString
method?

[wychen, 2016/05/23 21:40:24]

JSON writer takes care of this.

+  ss << v;
+  return ss.str();
+}
+
+void dumpDistillability(content::RenderFrame* render_frame,

[esprehn, 2016/05/23 20:19:42]

Dump I think? your name is wrong

[wychen, 2016/05/23 21:40:24]

Done.

+                        const std::vector<double>& derived,
+                        double score,
+                        bool distillable,
+                        double long_score,
+                        bool long_page,
+                        bool blacklisted) {
+  std::string msg = "adaboost_classification = {\n";
+  msg += "\"derived_features\": [";
+  for (unsigned i = 0; i < derived.size(); i++) {
+    if (i != 0) {
+      msg += ", ";
+    }
+    msg += DoubleToString(derived[i]);
+  }
+  msg += "],\n";
+  msg += "\"score\": " + DoubleToString(score) + ",\n";
+  msg += "\"distillable\": " + base::IntToString(distillable) + ",\n";
+  msg += "\"long_score\": " + DoubleToString(long_score) + ",\n";
+  msg += "\"long_page\": " + base::IntToString(long_page) + ",\n";

[esprehn, 2016/05/23 20:19:42]

Can you use base::Value and base::JSONWriter::Write instead? This is pretty hard
to read, and looks error prone

[wychen, 2016/05/23 21:40:24]

Awesome idea! Done. The key order is sorted though.

+  msg += "\"blacklisted\": " + base::IntToString(blacklisted) + "\n";
+  msg += "}";
+  render_frame->AddMessageToConsole(content::CONSOLE_MESSAGE_LEVEL_DEBUG, msg);
+}
+
 bool IsDistillablePageAdaboost(WebDocument& doc,
                                const DistillablePageDetector* detector,
                                const DistillablePageDetector* long_page,
-                               bool is_last) {
-  WebDistillabilityFeatures features = doc.distillabilityFeatures();
+                               bool is_last,
+                               content::RenderFrame* render_frame) {
+  bool isDevMode = IsDistillabilityDevSet();
+  WebDistillabilityFeatures features = doc.distillabilityFeatures(isDevMode);
   GURL parsed_url(doc.url());
   if (!parsed_url.is_valid()) {
     return false;
   }
   std::vector<double> derived = CalculateDerivedFeatures(
     features.openGraph,
     parsed_url,
     features.elementCount,
     features.anchorCount,
     features.formCount,
     features.mozScore,
     features.mozScoreAllSqrt,
     features.mozScoreAllLinear
   );
-  bool distillable = detector->Classify(derived);
-  bool long_article = long_page->Classify(derived);
+  double score = detector->Score(derived) - detector->GetThreshold();
+  double long_score = long_page->Score(derived) - long_page->GetThreshold();
+  bool distillable = score > 0;
+  bool long_article = long_score > 0;
   bool blacklisted = IsBlacklisted(parsed_url);
 
+  if (isDevMode) {
+    dumpDistillability(render_frame, derived, score, distillable, long_score,
+        long_article, blacklisted);
+  }
+
   int bucket = static_cast<unsigned>(features.isMobileFriendly) |
       (static_cast<unsigned>(distillable) << 1);
   if (is_last) {
     UMA_HISTOGRAM_ENUMERATION("DomDistiller.PageDistillableAfterLoading",
         bucket, 4);
   } else {
     UMA_HISTOGRAM_ENUMERATION("DomDistiller.PageDistillableAfterParsing",
         bucket, 4);
   }
 
   if (blacklisted) {
     return false;
   }
   if (features.isMobileFriendly) {
     return false;
   }
   return distillable && long_article;
 }
 
-bool IsDistillablePage(WebDocument& doc, bool is_last) {
+bool IsDistillablePage(WebDocument& doc, bool is_last,
+    content::RenderFrame* render_frame) {
   switch (GetDistillerHeuristicsType()) {
     case DistillerHeuristicsType::ALWAYS_TRUE:
       return true;
     case DistillerHeuristicsType::OG_ARTICLE:
-      return doc.distillabilityFeatures().openGraph;
+      return doc.distillabilityFeatures(false).openGraph;
     case DistillerHeuristicsType::ADABOOST_MODEL:
       return IsDistillablePageAdaboost(doc,
           DistillablePageDetector::GetNewModel(),
-          DistillablePageDetector::GetLongPageModel(), is_last);
+          DistillablePageDetector::GetLongPageModel(), is_last, render_frame);
     case DistillerHeuristicsType::NONE:
     default:
       return false;
   }
 }
 
 }  // namespace
 
 DistillabilityAgent::DistillabilityAgent(
     content::RenderFrame* render_frame)
@@ skipping 17 matching lines @@
   bool is_loaded = layout_type == WebMeaningfulLayout::FinishedLoading;
   if (!NeedToUpdate(is_loaded)) return;
 
   bool is_last = IsLast(is_loaded);
   // Connect to Mojo service on browser to notify page distillability.
   DistillabilityServicePtr distillability_service;
   render_frame()->GetServiceRegistry()->ConnectToRemoteService(
       mojo::GetProxy(&distillability_service));
   DCHECK(distillability_service);
   distillability_service->NotifyIsDistillable(
-      IsDistillablePage(doc, is_last), is_last);
+      IsDistillablePage(doc, is_last, render_frame()), is_last);
 }
 
 DistillabilityAgent::~DistillabilityAgent() {}
 
 }  // namespace dom_distiller