| OLD | NEW |
| 1 // Copyright 2014 The Chromium Authors. All rights reserved. | 1 // Copyright 2014 The Chromium Authors. All rights reserved. |
| 2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
| 3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
| 4 | 4 |
| 5 #include <stddef.h> | 5 #include <stddef.h> |
| 6 #include <sstream> | 6 #include <sstream> |
| 7 #include <utility> | 7 #include <utility> |
| 8 | 8 |
| 9 #include "base/command_line.h" | 9 #include "base/command_line.h" |
| 10 #include "base/files/scoped_temp_dir.h" | 10 #include "base/files/scoped_temp_dir.h" |
| (...skipping 38 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 49 | 49 |
| 50 typedef base::hash_map<std::string, std::string> FileToUrlMap; | 50 typedef base::hash_map<std::string, std::string> FileToUrlMap; |
| 51 | 51 |
| 52 } | 52 } |
| 53 | 53 |
| 54 // Factory for creating a Distiller that creates different DomDistillerOptions | 54 // Factory for creating a Distiller that creates different DomDistillerOptions |
| 55 // for different URLs, i.e. a specific kOriginalUrl option for each URL. | 55 // for different URLs, i.e. a specific kOriginalUrl option for each URL. |
| 56 class TestDistillerFactoryImpl : public DistillerFactory { | 56 class TestDistillerFactoryImpl : public DistillerFactory { |
| 57 public: | 57 public: |
| 58 TestDistillerFactoryImpl( | 58 TestDistillerFactoryImpl( |
| 59 scoped_ptr<DistillerURLFetcherFactory> distiller_url_fetcher_factory, | 59 std::unique_ptr<DistillerURLFetcherFactory> distiller_url_fetcher_factory, |
| 60 const dom_distiller::proto::DomDistillerOptions& dom_distiller_options, | 60 const dom_distiller::proto::DomDistillerOptions& dom_distiller_options, |
| 61 const FileToUrlMap& file_to_url_map) | 61 const FileToUrlMap& file_to_url_map) |
| 62 : distiller_url_fetcher_factory_( | 62 : distiller_url_fetcher_factory_( |
| 63 std::move(distiller_url_fetcher_factory)), | 63 std::move(distiller_url_fetcher_factory)), |
| 64 dom_distiller_options_(dom_distiller_options), | 64 dom_distiller_options_(dom_distiller_options), |
| 65 file_to_url_map_(file_to_url_map) {} | 65 file_to_url_map_(file_to_url_map) {} |
| 66 | 66 |
| 67 ~TestDistillerFactoryImpl() override {} | 67 ~TestDistillerFactoryImpl() override {} |
| 68 | 68 |
| 69 scoped_ptr<Distiller> CreateDistillerForUrl(const GURL& url) override { | 69 std::unique_ptr<Distiller> CreateDistillerForUrl(const GURL& url) override { |
| 70 dom_distiller::proto::DomDistillerOptions options; | 70 dom_distiller::proto::DomDistillerOptions options; |
| 71 options = dom_distiller_options_; | 71 options = dom_distiller_options_; |
| 72 FileToUrlMap::const_iterator it = file_to_url_map_.find(url.spec()); | 72 FileToUrlMap::const_iterator it = file_to_url_map_.find(url.spec()); |
| 73 if (it != file_to_url_map_.end()) { | 73 if (it != file_to_url_map_.end()) { |
| 74 options.set_original_url(it->second); | 74 options.set_original_url(it->second); |
| 75 } | 75 } |
| 76 scoped_ptr<DistillerImpl> distiller(new DistillerImpl( | 76 std::unique_ptr<DistillerImpl> distiller( |
| 77 *distiller_url_fetcher_factory_, options)); | 77 new DistillerImpl(*distiller_url_fetcher_factory_, options)); |
| 78 return std::move(distiller); | 78 return std::move(distiller); |
| 79 } | 79 } |
| 80 | 80 |
| 81 private: | 81 private: |
| 82 scoped_ptr<DistillerURLFetcherFactory> distiller_url_fetcher_factory_; | 82 std::unique_ptr<DistillerURLFetcherFactory> distiller_url_fetcher_factory_; |
| 83 dom_distiller::proto::DomDistillerOptions dom_distiller_options_; | 83 dom_distiller::proto::DomDistillerOptions dom_distiller_options_; |
| 84 FileToUrlMap file_to_url_map_; | 84 FileToUrlMap file_to_url_map_; |
| 85 }; | 85 }; |
| 86 | 86 |
| 87 namespace { | 87 namespace { |
| 88 | 88 |
| 89 // The url to distill. | 89 // The url to distill. |
| 90 const char* kUrlSwitch = "url"; | 90 const char* kUrlSwitch = "url"; |
| 91 | 91 |
| 92 // A space-separated list of urls to distill. | 92 // A space-separated list of urls to distill. |
| (...skipping 21 matching lines...) Expand all Loading... |
| 114 // A semi-colon-separated (i.e. ';') list of original URLs corresponding to | 114 // A semi-colon-separated (i.e. ';') list of original URLs corresponding to |
| 115 // "kUrlsSwitch". | 115 // "kUrlsSwitch". |
| 116 const char* kOriginalUrls = "original-urls"; | 116 const char* kOriginalUrls = "original-urls"; |
| 117 | 117 |
| 118 // The pagination algorithm to use, one of "next", "pagenum". | 118 // The pagination algorithm to use, one of "next", "pagenum". |
| 119 const char* kPaginationAlgo = "pagination-algo"; | 119 const char* kPaginationAlgo = "pagination-algo"; |
| 120 | 120 |
| 121 // Maximum number of concurrent started extractor requests. | 121 // Maximum number of concurrent started extractor requests. |
| 122 const int kMaxExtractorTasks = 8; | 122 const int kMaxExtractorTasks = 8; |
| 123 | 123 |
| 124 scoped_ptr<DomDistillerService> CreateDomDistillerService( | 124 std::unique_ptr<DomDistillerService> CreateDomDistillerService( |
| 125 content::BrowserContext* context, | 125 content::BrowserContext* context, |
| 126 const base::FilePath& db_path, | 126 const base::FilePath& db_path, |
| 127 const FileToUrlMap& file_to_url_map) { | 127 const FileToUrlMap& file_to_url_map) { |
| 128 scoped_refptr<base::SequencedTaskRunner> background_task_runner = | 128 scoped_refptr<base::SequencedTaskRunner> background_task_runner = |
| 129 content::BrowserThread::GetBlockingPool()->GetSequencedTaskRunner( | 129 content::BrowserThread::GetBlockingPool()->GetSequencedTaskRunner( |
| 130 content::BrowserThread::GetBlockingPool()->GetSequenceToken()); | 130 content::BrowserThread::GetBlockingPool()->GetSequenceToken()); |
| 131 | 131 |
| 132 // TODO(cjhopman): use an in-memory database instead of an on-disk one with | 132 // TODO(cjhopman): use an in-memory database instead of an on-disk one with |
| 133 // temporary directory. | 133 // temporary directory. |
| 134 scoped_ptr<leveldb_proto::ProtoDatabaseImpl<ArticleEntry> > db( | 134 std::unique_ptr<leveldb_proto::ProtoDatabaseImpl<ArticleEntry>> db( |
| 135 new leveldb_proto::ProtoDatabaseImpl<ArticleEntry>( | 135 new leveldb_proto::ProtoDatabaseImpl<ArticleEntry>( |
| 136 background_task_runner)); | 136 background_task_runner)); |
| 137 scoped_ptr<DomDistillerStore> dom_distiller_store( | 137 std::unique_ptr<DomDistillerStore> dom_distiller_store( |
| 138 new DomDistillerStore(std::move(db), db_path)); | 138 new DomDistillerStore(std::move(db), db_path)); |
| 139 | 139 |
| 140 scoped_ptr<DistillerPageFactory> distiller_page_factory( | 140 std::unique_ptr<DistillerPageFactory> distiller_page_factory( |
| 141 new DistillerPageWebContentsFactory(context)); | 141 new DistillerPageWebContentsFactory(context)); |
| 142 scoped_ptr<DistillerURLFetcherFactory> distiller_url_fetcher_factory( | 142 std::unique_ptr<DistillerURLFetcherFactory> distiller_url_fetcher_factory( |
| 143 new DistillerURLFetcherFactory( | 143 new DistillerURLFetcherFactory( |
| 144 content::BrowserContext::GetDefaultStoragePartition(context)-> | 144 content::BrowserContext::GetDefaultStoragePartition(context) |
| 145 GetURLRequestContext())); | 145 ->GetURLRequestContext())); |
| 146 | 146 |
| 147 dom_distiller::proto::DomDistillerOptions options; | 147 dom_distiller::proto::DomDistillerOptions options; |
| 148 if (base::CommandLine::ForCurrentProcess()->HasSwitch(kExtractTextOnly)) { | 148 if (base::CommandLine::ForCurrentProcess()->HasSwitch(kExtractTextOnly)) { |
| 149 options.set_extract_text_only(true); | 149 options.set_extract_text_only(true); |
| 150 } | 150 } |
| 151 int debug_level = 0; | 151 int debug_level = 0; |
| 152 if (base::CommandLine::ForCurrentProcess()->HasSwitch(kDebugLevel) && | 152 if (base::CommandLine::ForCurrentProcess()->HasSwitch(kDebugLevel) && |
| 153 base::StringToInt( | 153 base::StringToInt( |
| 154 base::CommandLine::ForCurrentProcess()->GetSwitchValueASCII( | 154 base::CommandLine::ForCurrentProcess()->GetSwitchValueASCII( |
| 155 kDebugLevel), | 155 kDebugLevel), |
| 156 &debug_level)) { | 156 &debug_level)) { |
| 157 options.set_debug_level(debug_level); | 157 options.set_debug_level(debug_level); |
| 158 } | 158 } |
| 159 // Options for pagination algorithm: | 159 // Options for pagination algorithm: |
| 160 // - "next": detect anchors with "next" text | 160 // - "next": detect anchors with "next" text |
| 161 // - "pagenum": detect anchors with numeric page numbers | 161 // - "pagenum": detect anchors with numeric page numbers |
| 162 // Default is "next". | 162 // Default is "next". |
| 163 if (base::CommandLine::ForCurrentProcess()->HasSwitch(kPaginationAlgo)) { | 163 if (base::CommandLine::ForCurrentProcess()->HasSwitch(kPaginationAlgo)) { |
| 164 options.set_pagination_algo( | 164 options.set_pagination_algo( |
| 165 base::CommandLine::ForCurrentProcess()->GetSwitchValueASCII( | 165 base::CommandLine::ForCurrentProcess()->GetSwitchValueASCII( |
| 166 kPaginationAlgo)); | 166 kPaginationAlgo)); |
| 167 } | 167 } |
| 168 | 168 |
| 169 scoped_ptr<DistillerFactory> distiller_factory(new TestDistillerFactoryImpl( | 169 std::unique_ptr<DistillerFactory> distiller_factory( |
| 170 std::move(distiller_url_fetcher_factory), options, file_to_url_map)); | 170 new TestDistillerFactoryImpl(std::move(distiller_url_fetcher_factory), |
| 171 options, file_to_url_map)); |
| 171 | 172 |
| 172 // Setting up PrefService for DistilledPagePrefs. | 173 // Setting up PrefService for DistilledPagePrefs. |
| 173 user_prefs::TestingPrefServiceSyncable* pref_service = | 174 user_prefs::TestingPrefServiceSyncable* pref_service = |
| 174 new user_prefs::TestingPrefServiceSyncable(); | 175 new user_prefs::TestingPrefServiceSyncable(); |
| 175 DistilledPagePrefs::RegisterProfilePrefs(pref_service->registry()); | 176 DistilledPagePrefs::RegisterProfilePrefs(pref_service->registry()); |
| 176 | 177 |
| 177 return scoped_ptr<DomDistillerService>(new DomDistillerService( | 178 return std::unique_ptr<DomDistillerService>(new DomDistillerService( |
| 178 std::move(dom_distiller_store), std::move(distiller_factory), | 179 std::move(dom_distiller_store), std::move(distiller_factory), |
| 179 std::move(distiller_page_factory), | 180 std::move(distiller_page_factory), |
| 180 scoped_ptr<DistilledPagePrefs>(new DistilledPagePrefs(pref_service)))); | 181 std::unique_ptr<DistilledPagePrefs>( |
| 182 new DistilledPagePrefs(pref_service)))); |
| 181 } | 183 } |
| 182 | 184 |
| 183 void AddComponentsTestResources() { | 185 void AddComponentsTestResources() { |
| 184 base::FilePath pak_file; | 186 base::FilePath pak_file; |
| 185 base::FilePath pak_dir; | 187 base::FilePath pak_dir; |
| 186 PathService::Get(base::DIR_MODULE, &pak_dir); | 188 PathService::Get(base::DIR_MODULE, &pak_dir); |
| 187 pak_file = | 189 pak_file = |
| 188 pak_dir.Append(FILE_PATH_LITERAL("components_tests_resources.pak")); | 190 pak_dir.Append(FILE_PATH_LITERAL("components_tests_resources.pak")); |
| 189 ui::ResourceBundle::GetSharedInstance().AddDataPackFromPath( | 191 ui::ResourceBundle::GetSharedInstance().AddDataPackFromPath( |
| 190 pak_file, ui::SCALE_FACTOR_NONE); | 192 pak_file, ui::SCALE_FACTOR_NONE); |
| (...skipping 113 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 304 void OnArticleUpdated(ArticleDistillationUpdate article_update) override {} | 306 void OnArticleUpdated(ArticleDistillationUpdate article_update) override {} |
| 305 | 307 |
| 306 void OnArticleReady(const DistilledArticleProto* article_proto) override { | 308 void OnArticleReady(const DistilledArticleProto* article_proto) override { |
| 307 article_proto_ = article_proto; | 309 article_proto_ = article_proto; |
| 308 CHECK(article_proto->pages_size()) << "Failed extracting " << url_; | 310 CHECK(article_proto->pages_size()) << "Failed extracting " << url_; |
| 309 base::ThreadTaskRunnerHandle::Get()->PostTask(FROM_HERE, | 311 base::ThreadTaskRunnerHandle::Get()->PostTask(FROM_HERE, |
| 310 finished_callback_); | 312 finished_callback_); |
| 311 } | 313 } |
| 312 | 314 |
| 313 const DistilledArticleProto* article_proto_; | 315 const DistilledArticleProto* article_proto_; |
| 314 scoped_ptr<ViewerHandle> viewer_handle_; | 316 std::unique_ptr<ViewerHandle> viewer_handle_; |
| 315 GURL url_; | 317 GURL url_; |
| 316 base::Closure finished_callback_; | 318 base::Closure finished_callback_; |
| 317 }; | 319 }; |
| 318 | 320 |
| 319 class ContentExtractor : public ContentBrowserTest { | 321 class ContentExtractor : public ContentBrowserTest { |
| 320 public: | 322 public: |
| 321 ContentExtractor() | 323 ContentExtractor() |
| 322 : pending_tasks_(0), | 324 : pending_tasks_(0), |
| 323 max_tasks_(kMaxExtractorTasks), | 325 max_tasks_(kMaxExtractorTasks), |
| 324 next_request_(0), | 326 next_request_(0), |
| (...skipping 95 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 420 service_.reset(); | 422 service_.reset(); |
| 421 base::ThreadTaskRunnerHandle::Get()->PostTask( | 423 base::ThreadTaskRunnerHandle::Get()->PostTask( |
| 422 FROM_HERE, base::MessageLoop::QuitWhenIdleClosure()); | 424 FROM_HERE, base::MessageLoop::QuitWhenIdleClosure()); |
| 423 } | 425 } |
| 424 | 426 |
| 425 size_t pending_tasks_; | 427 size_t pending_tasks_; |
| 426 size_t max_tasks_; | 428 size_t max_tasks_; |
| 427 size_t next_request_; | 429 size_t next_request_; |
| 428 | 430 |
| 429 base::ScopedTempDir db_dir_; | 431 base::ScopedTempDir db_dir_; |
| 430 scoped_ptr<net::ScopedDefaultHostResolverProc> mock_host_resolver_override_; | 432 std::unique_ptr<net::ScopedDefaultHostResolverProc> |
| 431 scoped_ptr<DomDistillerService> service_; | 433 mock_host_resolver_override_; |
| 434 std::unique_ptr<DomDistillerService> service_; |
| 432 ScopedVector<ContentExtractionRequest> requests_; | 435 ScopedVector<ContentExtractionRequest> requests_; |
| 433 | 436 |
| 434 std::string output_data_; | 437 std::string output_data_; |
| 435 scoped_ptr<google::protobuf::io::StringOutputStream> protobuf_output_stream_; | 438 std::unique_ptr<google::protobuf::io::StringOutputStream> |
| 439 protobuf_output_stream_; |
| 436 }; | 440 }; |
| 437 | 441 |
| 438 IN_PROC_BROWSER_TEST_F(ContentExtractor, MANUAL_ExtractUrl) { | 442 IN_PROC_BROWSER_TEST_F(ContentExtractor, MANUAL_ExtractUrl) { |
| 439 SetDistillerJavaScriptWorldId(content::ISOLATED_WORLD_ID_CONTENT_END); | 443 SetDistillerJavaScriptWorldId(content::ISOLATED_WORLD_ID_CONTENT_END); |
| 440 Start(); | 444 Start(); |
| 441 base::RunLoop().Run(); | 445 base::RunLoop().Run(); |
| 442 } | 446 } |
| 443 | 447 |
| 444 } // namespace dom_distiller | 448 } // namespace dom_distiller |
| OLD | NEW |