Provide original URLs for next page detection in dom_distiller

components/dom_distiller/standalone/content_extractor.cc

 // Copyright 2014 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include <sstream>
 
 #include "base/command_line.h"
 #include "base/files/scoped_temp_dir.h"
 #include "base/id_map.h"
 #include "base/message_loop/message_loop.h"
@@ skipping 22 matching lines @@
 #include "net/dns/mock_host_resolver.h"
 #include "third_party/dom_distiller_js/dom_distiller.pb.h"
 #include "ui/base/resource/resource_bundle.h"
 
 using content::ContentBrowserTest;
 
 namespace dom_distiller {
 
 namespace {
 
-typedef base::hash_map<std::string, std::string> UrlToDomainMap;
+typedef base::hash_map<std::string, std::string> FileToUrlMap;
 
 }
 
 // Factory for creating a Distiller that creates different DomDistillerOptions
-// for different URLs, i.e. a specific kOriginalDomain option for each URL.
+// for different URLs, i.e. a specific kOriginalUrl option for each URL.
 class TestDistillerFactoryImpl : public DistillerFactory {
  public:
   TestDistillerFactoryImpl(
       scoped_ptr<DistillerURLFetcherFactory> distiller_url_fetcher_factory,
       const dom_distiller::proto::DomDistillerOptions& dom_distiller_options,
-      const UrlToDomainMap& url_to_domain_map)
+      const FileToUrlMap& file_to_url_map)
       : distiller_url_fetcher_factory_(distiller_url_fetcher_factory.Pass()),
         dom_distiller_options_(dom_distiller_options),
-        url_to_domain_map_(url_to_domain_map) {
+        file_to_url_map_(file_to_url_map) {
   }
 
   ~TestDistillerFactoryImpl() override {}
 
   scoped_ptr<Distiller> CreateDistillerForUrl(const GURL& url) override {
     dom_distiller::proto::DomDistillerOptions options;
     options = dom_distiller_options_;
-    UrlToDomainMap::const_iterator it = url_to_domain_map_.find(url.spec());
-    if (it != url_to_domain_map_.end()) options.set_original_domain(it->second);
+    FileToUrlMap::const_iterator it = file_to_url_map_.find(url.spec());
+    if (it != file_to_url_map_.end()) {
+      options.set_original_url(it->second);
+    }
     scoped_ptr<DistillerImpl> distiller(new DistillerImpl(
         *distiller_url_fetcher_factory_, options));
     return distiller.Pass();
   }
 
  private:
   scoped_ptr<DistillerURLFetcherFactory> distiller_url_fetcher_factory_;
   dom_distiller::proto::DomDistillerOptions dom_distiller_options_;
-  UrlToDomainMap url_to_domain_map_;
+  FileToUrlMap file_to_url_map_;
 };
 
 namespace {
 
 // The url to distill.
 const char* kUrlSwitch = "url";
 
 // A space-separated list of urls to distill.
 const char* kUrlsSwitch = "urls";
 
 // Indicates that DNS resolution should be disabled for this test.
 const char* kDisableDnsSwitch = "disable-dns";
 
 // Will write the distilled output to the given file instead of to stdout.
 const char* kOutputFile = "output-file";
 
 // Indicates to output a serialized protocol buffer instead of human-readable
 // output.
 const char* kShouldOutputBinary = "output-binary";
 
 // Indicates to output only the text of the article and not the enclosing html.
 const char* kExtractTextOnly = "extract-text-only";
 
 // Indicates to include debug output.
 const char* kDebugLevel = "debug-level";
 
-// The original domain of the page if |kUrlSwitch| is a file.
-const char* kOriginalDomain = "original-domain";
+// The original URL of the page if |kUrlSwitch| is a file.
+const char* kOriginalUrl = "original-url";
 
-// A semi-colon-separated (i.e. ';') list of original domains corresponding to
+// A semi-colon-separated (i.e. ';') list of original URLs corresponding to
 // "kUrlsSwitch".
-const char* kOriginalDomains = "original-domains";
+const char* kOriginalUrls = "original-urls";
 
 // Maximum number of concurrent started extractor requests.
 const int kMaxExtractorTasks = 8;
 
 scoped_ptr<DomDistillerService> CreateDomDistillerService(
     content::BrowserContext* context,
     const base::FilePath& db_path,
-    const UrlToDomainMap& url_to_domain_map) {
+    const FileToUrlMap& file_to_url_map) {
   scoped_refptr<base::SequencedTaskRunner> background_task_runner =
       content::BrowserThread::GetBlockingPool()->GetSequencedTaskRunner(
           content::BrowserThread::GetBlockingPool()->GetSequenceToken());
 
   // TODO(cjhopman): use an in-memory database instead of an on-disk one with
   // temporary directory.
   scoped_ptr<leveldb_proto::ProtoDatabaseImpl<ArticleEntry> > db(
       new leveldb_proto::ProtoDatabaseImpl<ArticleEntry>(
           background_task_runner));
   scoped_ptr<DomDistillerStore> dom_distiller_store(
@@ skipping 12 matching lines @@
   if (base::CommandLine::ForCurrentProcess()->HasSwitch(kDebugLevel) &&
       base::StringToInt(
           base::CommandLine::ForCurrentProcess()->GetSwitchValueASCII(
               kDebugLevel),
           &debug_level)) {
     options.set_debug_level(debug_level);
   }
   scoped_ptr<DistillerFactory> distiller_factory(
       new TestDistillerFactoryImpl(distiller_url_fetcher_factory.Pass(),
                                    options,
-                                   url_to_domain_map));
+                                   file_to_url_map));
 
   // Setting up PrefService for DistilledPagePrefs.
   user_prefs::TestingPrefServiceSyncable* pref_service =
       new user_prefs::TestingPrefServiceSyncable();
   DistilledPagePrefs::RegisterProfilePrefs(pref_service->registry());
 
   return scoped_ptr<DomDistillerService>(new DomDistillerService(
       dom_distiller_store.Pass(),
       distiller_factory.Pass(),
       distiller_page_factory.Pass(),
@@ skipping 61 matching lines @@
                          service->CreateDefaultDistillerPage(render_view_size),
                          url_);
   }
 
   DistilledArticleProto GetArticleCopy() {
     return *article_proto_;
   }
 
   static ScopedVector<ContentExtractionRequest> CreateForCommandLine(
       const base::CommandLine& command_line,
-      UrlToDomainMap* url_to_domain_map) {
+      FileToUrlMap* file_to_url_map) {
     ScopedVector<ContentExtractionRequest> requests;
     if (command_line.HasSwitch(kUrlSwitch)) {
       GURL url;
       std::string url_string = command_line.GetSwitchValueASCII(kUrlSwitch);
       url = GURL(url_string);
       if (url.is_valid()) {
         requests.push_back(new ContentExtractionRequest(url));
-        if (command_line.HasSwitch(kOriginalDomain)) {
-          (*url_to_domain_map)[url.spec()] =
-              command_line.GetSwitchValueASCII(kOriginalDomain);
+        if (command_line.HasSwitch(kOriginalUrl)) {
+          (*file_to_url_map)[url.spec()] =
+              command_line.GetSwitchValueASCII(kOriginalUrl);
         }
       }
     } else if (command_line.HasSwitch(kUrlsSwitch)) {
       std::string urls_string = command_line.GetSwitchValueASCII(kUrlsSwitch);
       std::vector<std::string> urls;
       base::SplitString(urls_string, ' ', &urls);
-      // Check for original-domains switch, which must exactly pair up with
-      // |kUrlsSwitch| i.e. number of domains must be same as that of urls.
-      std::vector<std::string> domains;
-      if (command_line.HasSwitch(kOriginalDomains)) {
-        std::string domains_string =
-            command_line.GetSwitchValueASCII( kOriginalDomains);
-        base::SplitString(domains_string, ';', &domains);
-        if (domains.size() != urls.size()) domains.clear();
+      // Check for original-urls switch, which must exactly pair up with
+      // |kUrlsSwitch| i.e. number of original urls must be same as that of
+      // urls.
+      std::vector<std::string> original_urls;
+      if (command_line.HasSwitch(kOriginalUrls)) {
+        std::string original_urls_string =
+            command_line.GetSwitchValueASCII(kOriginalUrls);
+        base::SplitString(original_urls_string, ' ', &original_urls);
+        if (original_urls.size() != urls.size()) original_urls.clear();
       }
       for (size_t i = 0; i < urls.size(); ++i) {
         GURL url(urls[i]);
         if (url.is_valid()) {
           requests.push_back(new ContentExtractionRequest(url));
-          // Only regard non-empty domain.
-          if (!domains.empty() && !domains[i].empty()) {
-              (*url_to_domain_map)[url.spec()] = domains[i];
+          // Only regard non-empty original urls.
+          if (!original_urls.empty() && !original_urls[i].empty()) {
+              (*file_to_url_map)[url.spec()] = original_urls[i];
           }
         } else {
           ADD_FAILURE() << "Bad url";
         }
       }
     }
     if (requests.empty()) {
       ADD_FAILURE() << "No valid url provided";
     }
 
@@ skipping 40 matching lines @@
   }
 
   void TearDownOnMainThread() override { DisableDNSLookupForThisTest(); }
 
  protected:
   // Creates the DomDistillerService and creates and starts the extraction
   // request.
   void Start() {
     const base::CommandLine& command_line =
         *base::CommandLine::ForCurrentProcess();
-    UrlToDomainMap url_to_domain_map;
+    FileToUrlMap file_to_url_map;
     requests_ = ContentExtractionRequest::CreateForCommandLine(
-        command_line, &url_to_domain_map);
+        command_line, &file_to_url_map);
     content::BrowserContext* context =
         shell()->web_contents()->GetBrowserContext();
     service_ = CreateDomDistillerService(context,
                                          db_dir_.path(),
-                                         url_to_domain_map);
+                                         file_to_url_map);
     PumpQueue();
   }
 
   void PumpQueue() {
     while (pending_tasks_ < max_tasks_ && next_request_ < requests_.size()) {
       requests_[next_request_]->Start(
           service_.get(),
           shell()->web_contents()->GetContainerBounds().size(),
           base::Bind(&ContentExtractor::FinishRequest, base::Unretained(this)));
       ++next_request_;
@@ skipping 69 matching lines @@
   std::string output_data_;
   scoped_ptr<google::protobuf::io::StringOutputStream> protobuf_output_stream_;
 };
 
 IN_PROC_BROWSER_TEST_F(ContentExtractor, MANUAL_ExtractUrl) {
   Start();
   base::RunLoop().Run();
 }
 
 }  // namespace dom_distiller