add options to specify original domain(s)

components/dom_distiller/standalone/content_extractor.cc

 // Copyright 2014 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include <sstream>
 
 #include "base/command_line.h"
 #include "base/files/scoped_temp_dir.h"
+#include "base/id_map.h"
 #include "base/message_loop/message_loop.h"
 #include "base/path_service.h"
 #include "base/run_loop.h"
 #include "base/strings/string_number_conversions.h"
 #include "base/strings/string_split.h"
 #include "components/dom_distiller/content/distiller_page_web_contents.h"
 #include "components/dom_distiller/core/article_entry.h"
 #include "components/dom_distiller/core/distilled_page_prefs.h"
 #include "components/dom_distiller/core/distiller.h"
 #include "components/dom_distiller/core/dom_distiller_service.h"
@@ skipping 13 matching lines @@
 #include "net/dns/mock_host_resolver.h"
 #include "third_party/dom_distiller_js/dom_distiller.pb.h"
 #include "ui/base/resource/resource_bundle.h"
 
 using content::ContentBrowserTest;
 
 namespace dom_distiller {
 
 namespace {
 
+typedef base::hash_map<std::string, std::string> UrlToDomainMap;
+
+}
+
+// Factory for creating a Distiller that creates different DomDistillerOptions
+// for different URLs, i.e. a specific kOriginalDomain option for each URL.
+class TestDistillerFactoryImpl : public DistillerFactory {
+ public:
+  TestDistillerFactoryImpl(
+      scoped_ptr<DistillerURLFetcherFactory> distiller_url_fetcher_factory,
+      const dom_distiller::proto::DomDistillerOptions& dom_distiller_options,
+      const UrlToDomainMap& url_to_domain_map)
+      : distiller_url_fetcher_factory_(distiller_url_fetcher_factory.Pass()),
+        dom_distiller_options_(dom_distiller_options),
+        url_to_domain_map_(url_to_domain_map) {
+  }
+
+  ~TestDistillerFactoryImpl() override {}
+
+  scoped_ptr<Distiller> CreateDistillerForUrl(const GURL& url) override {
+    dom_distiller::proto::DomDistillerOptions options;
+    options = dom_distiller_options_;
+    UrlToDomainMap::const_iterator it = url_to_domain_map_.find(url.spec());
+    if (it != url_to_domain_map_.end()) options.set_original_domain(it->second);
+    scoped_ptr<DistillerImpl> distiller(new DistillerImpl(
+        *distiller_url_fetcher_factory_, options));
+    return distiller.Pass();
+  }
+
+ private:
+  scoped_ptr<DistillerURLFetcherFactory> distiller_url_fetcher_factory_;
+  dom_distiller::proto::DomDistillerOptions dom_distiller_options_;
+  UrlToDomainMap url_to_domain_map_;
+};
+
+namespace {
+
 // The url to distill.
 const char* kUrlSwitch = "url";
 
 // A space-separated list of urls to distill.
 const char* kUrlsSwitch = "urls";
 
 // Indicates that DNS resolution should be disabled for this test.
 const char* kDisableDnsSwitch = "disable-dns";
 
 // Will write the distilled output to the given file instead of to stdout.
 const char* kOutputFile = "output-file";
 
 // Indicates to output a serialized protocol buffer instead of human-readable
 // output.
 const char* kShouldOutputBinary = "output-binary";
 
 // Indicates to output only the text of the article and not the enclosing html.
 const char* kExtractTextOnly = "extract-text-only";
 
 // Indicates to include debug output.
 const char* kDebugLevel = "debug-level";
 
+// The original domain of the page if |kUrlSwitch| is a file.
+const char* kOriginalDomain = "original-domain";
+
+// A semi-colon-separated (i.e. ';') list of original domains corresponding to

[cjhopman, 2014/10/29 17:13:41]

Do space-separated instead (like --urls)

[kuan, 2014/10/29 17:26:53]

hm.. i changed to use ';' in case the domain can't be extracted from the url,
which would result in an empty string - see
https://critique.corp.google.com/#review/78763318 -> driver.go -> word
"schemeAndRest".  i don't want scorer to crash for any mistake.

+// "kUrlsSwitch".
+const char* kOriginalDomains = "original-domains";
+
 // Maximum number of concurrent started extractor requests.
 const int kMaxExtractorTasks = 8;
 
 scoped_ptr<DomDistillerService> CreateDomDistillerService(
     content::BrowserContext* context,
-    const base::FilePath& db_path) {
+    const base::FilePath& db_path,
+    const UrlToDomainMap& url_to_domain_map) {
   scoped_refptr<base::SequencedTaskRunner> background_task_runner =
       content::BrowserThread::GetBlockingPool()->GetSequencedTaskRunner(
           content::BrowserThread::GetBlockingPool()->GetSequenceToken());
 
   // TODO(cjhopman): use an in-memory database instead of an on-disk one with
   // temporary directory.
   scoped_ptr<leveldb_proto::ProtoDatabaseImpl<ArticleEntry> > db(
       new leveldb_proto::ProtoDatabaseImpl<ArticleEntry>(
           background_task_runner));
   scoped_ptr<DomDistillerStore> dom_distiller_store(
@@ skipping 10 matching lines @@
   }
   int debug_level = 0;
   if (base::CommandLine::ForCurrentProcess()->HasSwitch(kDebugLevel) &&
       base::StringToInt(
           base::CommandLine::ForCurrentProcess()->GetSwitchValueASCII(
               kDebugLevel),
           &debug_level)) {
     options.set_debug_level(debug_level);
   }
   scoped_ptr<DistillerFactory> distiller_factory(
-      new DistillerFactoryImpl(distiller_url_fetcher_factory.Pass(), options));
+      new TestDistillerFactoryImpl(distiller_url_fetcher_factory.Pass(),
+                                   options,
+                                   url_to_domain_map));
 
   // Setting up PrefService for DistilledPagePrefs.
   user_prefs::TestingPrefServiceSyncable* pref_service =
       new user_prefs::TestingPrefServiceSyncable();
   DistilledPagePrefs::RegisterProfilePrefs(pref_service->registry());
 
   return scoped_ptr<DomDistillerService>(new DomDistillerService(
       dom_distiller_store.Pass(),
       distiller_factory.Pass(),
       distiller_page_factory.Pass(),
@@ skipping 59 matching lines @@
         service->ViewUrl(this,
                          service->CreateDefaultDistillerPage(render_view_size),
                          url_);
   }
 
   DistilledArticleProto GetArticleCopy() {
     return *article_proto_;
   }
 
   static ScopedVector<ContentExtractionRequest> CreateForCommandLine(
-      const CommandLine& command_line) {
+      const CommandLine& command_line,
+      UrlToDomainMap* url_to_domain_map) {
     ScopedVector<ContentExtractionRequest> requests;
     if (command_line.HasSwitch(kUrlSwitch)) {
       GURL url;
       std::string url_string = command_line.GetSwitchValueASCII(kUrlSwitch);
       url = GURL(url_string);
       if (url.is_valid()) {
         requests.push_back(new ContentExtractionRequest(url));
+        if (command_line.HasSwitch(kOriginalDomain)) {
+          (*url_to_domain_map)[url.spec()] =
+              command_line.GetSwitchValueASCII(kOriginalDomain);
+        }
       }
     } else if (command_line.HasSwitch(kUrlsSwitch)) {
       std::string urls_string = command_line.GetSwitchValueASCII(kUrlsSwitch);
       std::vector<std::string> urls;
       base::SplitString(urls_string, ' ', &urls);
+      // Check for original-domains switch, which must exactly pair up with
+      // |kUrlsSwitch| i.e. number of domains must be same as that of urls.
+      std::vector<std::string> domains;
+      if (command_line.HasSwitch(kOriginalDomains)) {
+        std::string domains_string =
+            command_line.GetSwitchValueASCII( kOriginalDomains);
+        base::SplitString(domains_string, ';', &domains);
+        if (domains.size() != urls.size()) domains.clear();
+      }
       for (size_t i = 0; i < urls.size(); ++i) {
         GURL url(urls[i]);
         if (url.is_valid()) {
           requests.push_back(new ContentExtractionRequest(url));
+          // Only regard non-empty domain.
+          if (!domains.empty() && !domains[i].empty()) {
+              (*url_to_domain_map)[url.spec()] = domains[i];
+          }
         } else {
           ADD_FAILURE() << "Bad url";
         }
       }
     }
     if (requests.empty()) {
       ADD_FAILURE() << "No valid url provided";
     }
 
     return requests.Pass();
@@ skipping 41 matching lines @@
   }
 
   virtual void TearDownOnMainThread() override {
     DisableDNSLookupForThisTest();
   }
 
  protected:
   // Creates the DomDistillerService and creates and starts the extraction
   // request.
   void Start() {
+    const CommandLine& command_line = *CommandLine::ForCurrentProcess();
+    UrlToDomainMap url_to_domain_map;
+    requests_ = ContentExtractionRequest::CreateForCommandLine(
+        command_line, &url_to_domain_map);
     content::BrowserContext* context =
         shell()->web_contents()->GetBrowserContext();
     service_ = CreateDomDistillerService(context,
-                                         db_dir_.path());
-    const CommandLine& command_line = *CommandLine::ForCurrentProcess();
-    requests_ = ContentExtractionRequest::CreateForCommandLine(command_line);
+                                         db_dir_.path(),
+                                         url_to_domain_map);
     PumpQueue();
   }
 
   void PumpQueue() {
     while (pending_tasks_ < max_tasks_ && next_request_ < requests_.size()) {
       requests_[next_request_]->Start(
           service_.get(),
           shell()->web_contents()->GetContainerBounds().size(),
           base::Bind(&ContentExtractor::FinishRequest, base::Unretained(this)));
       ++next_request_;
@@ skipping 68 matching lines @@
   std::string output_data_;
   scoped_ptr<google::protobuf::io::StringOutputStream> protobuf_output_stream_;
 };
 
 IN_PROC_BROWSER_TEST_F(ContentExtractor, MANUAL_ExtractUrl) {
   Start();
   base::RunLoop().Run();
 }
 
 }  // namespace dom_distiller