OLD | NEW |
1 // Copyright 2014 The Chromium Authors. All rights reserved. | 1 // Copyright 2014 The Chromium Authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
4 | 4 |
5 #include <stddef.h> | 5 #include <stddef.h> |
6 #include <sstream> | 6 #include <sstream> |
7 #include <utility> | 7 #include <utility> |
8 | 8 |
9 #include "base/command_line.h" | 9 #include "base/command_line.h" |
10 #include "base/files/scoped_temp_dir.h" | 10 #include "base/files/scoped_temp_dir.h" |
(...skipping 38 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
49 | 49 |
50 typedef base::hash_map<std::string, std::string> FileToUrlMap; | 50 typedef base::hash_map<std::string, std::string> FileToUrlMap; |
51 | 51 |
52 } | 52 } |
53 | 53 |
54 // Factory for creating a Distiller that creates different DomDistillerOptions | 54 // Factory for creating a Distiller that creates different DomDistillerOptions |
55 // for different URLs, i.e. a specific kOriginalUrl option for each URL. | 55 // for different URLs, i.e. a specific kOriginalUrl option for each URL. |
56 class TestDistillerFactoryImpl : public DistillerFactory { | 56 class TestDistillerFactoryImpl : public DistillerFactory { |
57 public: | 57 public: |
58 TestDistillerFactoryImpl( | 58 TestDistillerFactoryImpl( |
59 scoped_ptr<DistillerURLFetcherFactory> distiller_url_fetcher_factory, | 59 std::unique_ptr<DistillerURLFetcherFactory> distiller_url_fetcher_factory, |
60 const dom_distiller::proto::DomDistillerOptions& dom_distiller_options, | 60 const dom_distiller::proto::DomDistillerOptions& dom_distiller_options, |
61 const FileToUrlMap& file_to_url_map) | 61 const FileToUrlMap& file_to_url_map) |
62 : distiller_url_fetcher_factory_( | 62 : distiller_url_fetcher_factory_( |
63 std::move(distiller_url_fetcher_factory)), | 63 std::move(distiller_url_fetcher_factory)), |
64 dom_distiller_options_(dom_distiller_options), | 64 dom_distiller_options_(dom_distiller_options), |
65 file_to_url_map_(file_to_url_map) {} | 65 file_to_url_map_(file_to_url_map) {} |
66 | 66 |
67 ~TestDistillerFactoryImpl() override {} | 67 ~TestDistillerFactoryImpl() override {} |
68 | 68 |
69 scoped_ptr<Distiller> CreateDistillerForUrl(const GURL& url) override { | 69 std::unique_ptr<Distiller> CreateDistillerForUrl(const GURL& url) override { |
70 dom_distiller::proto::DomDistillerOptions options; | 70 dom_distiller::proto::DomDistillerOptions options; |
71 options = dom_distiller_options_; | 71 options = dom_distiller_options_; |
72 FileToUrlMap::const_iterator it = file_to_url_map_.find(url.spec()); | 72 FileToUrlMap::const_iterator it = file_to_url_map_.find(url.spec()); |
73 if (it != file_to_url_map_.end()) { | 73 if (it != file_to_url_map_.end()) { |
74 options.set_original_url(it->second); | 74 options.set_original_url(it->second); |
75 } | 75 } |
76 scoped_ptr<DistillerImpl> distiller(new DistillerImpl( | 76 std::unique_ptr<DistillerImpl> distiller( |
77 *distiller_url_fetcher_factory_, options)); | 77 new DistillerImpl(*distiller_url_fetcher_factory_, options)); |
78 return std::move(distiller); | 78 return std::move(distiller); |
79 } | 79 } |
80 | 80 |
81 private: | 81 private: |
82 scoped_ptr<DistillerURLFetcherFactory> distiller_url_fetcher_factory_; | 82 std::unique_ptr<DistillerURLFetcherFactory> distiller_url_fetcher_factory_; |
83 dom_distiller::proto::DomDistillerOptions dom_distiller_options_; | 83 dom_distiller::proto::DomDistillerOptions dom_distiller_options_; |
84 FileToUrlMap file_to_url_map_; | 84 FileToUrlMap file_to_url_map_; |
85 }; | 85 }; |
86 | 86 |
87 namespace { | 87 namespace { |
88 | 88 |
89 // The url to distill. | 89 // The url to distill. |
90 const char* kUrlSwitch = "url"; | 90 const char* kUrlSwitch = "url"; |
91 | 91 |
92 // A space-separated list of urls to distill. | 92 // A space-separated list of urls to distill. |
(...skipping 21 matching lines...) Expand all Loading... |
114 // A semi-colon-separated (i.e. ';') list of original URLs corresponding to | 114 // A semi-colon-separated (i.e. ';') list of original URLs corresponding to |
115 // "kUrlsSwitch". | 115 // "kUrlsSwitch". |
116 const char* kOriginalUrls = "original-urls"; | 116 const char* kOriginalUrls = "original-urls"; |
117 | 117 |
118 // The pagination algorithm to use, one of "next", "pagenum". | 118 // The pagination algorithm to use, one of "next", "pagenum". |
119 const char* kPaginationAlgo = "pagination-algo"; | 119 const char* kPaginationAlgo = "pagination-algo"; |
120 | 120 |
121 // Maximum number of concurrent started extractor requests. | 121 // Maximum number of concurrent started extractor requests. |
122 const int kMaxExtractorTasks = 8; | 122 const int kMaxExtractorTasks = 8; |
123 | 123 |
124 scoped_ptr<DomDistillerService> CreateDomDistillerService( | 124 std::unique_ptr<DomDistillerService> CreateDomDistillerService( |
125 content::BrowserContext* context, | 125 content::BrowserContext* context, |
126 const base::FilePath& db_path, | 126 const base::FilePath& db_path, |
127 const FileToUrlMap& file_to_url_map) { | 127 const FileToUrlMap& file_to_url_map) { |
128 scoped_refptr<base::SequencedTaskRunner> background_task_runner = | 128 scoped_refptr<base::SequencedTaskRunner> background_task_runner = |
129 content::BrowserThread::GetBlockingPool()->GetSequencedTaskRunner( | 129 content::BrowserThread::GetBlockingPool()->GetSequencedTaskRunner( |
130 content::BrowserThread::GetBlockingPool()->GetSequenceToken()); | 130 content::BrowserThread::GetBlockingPool()->GetSequenceToken()); |
131 | 131 |
132 // TODO(cjhopman): use an in-memory database instead of an on-disk one with | 132 // TODO(cjhopman): use an in-memory database instead of an on-disk one with |
133 // temporary directory. | 133 // temporary directory. |
134 scoped_ptr<leveldb_proto::ProtoDatabaseImpl<ArticleEntry> > db( | 134 std::unique_ptr<leveldb_proto::ProtoDatabaseImpl<ArticleEntry>> db( |
135 new leveldb_proto::ProtoDatabaseImpl<ArticleEntry>( | 135 new leveldb_proto::ProtoDatabaseImpl<ArticleEntry>( |
136 background_task_runner)); | 136 background_task_runner)); |
137 scoped_ptr<DomDistillerStore> dom_distiller_store( | 137 std::unique_ptr<DomDistillerStore> dom_distiller_store( |
138 new DomDistillerStore(std::move(db), db_path)); | 138 new DomDistillerStore(std::move(db), db_path)); |
139 | 139 |
140 scoped_ptr<DistillerPageFactory> distiller_page_factory( | 140 std::unique_ptr<DistillerPageFactory> distiller_page_factory( |
141 new DistillerPageWebContentsFactory(context)); | 141 new DistillerPageWebContentsFactory(context)); |
142 scoped_ptr<DistillerURLFetcherFactory> distiller_url_fetcher_factory( | 142 std::unique_ptr<DistillerURLFetcherFactory> distiller_url_fetcher_factory( |
143 new DistillerURLFetcherFactory( | 143 new DistillerURLFetcherFactory( |
144 content::BrowserContext::GetDefaultStoragePartition(context)-> | 144 content::BrowserContext::GetDefaultStoragePartition(context) |
145 GetURLRequestContext())); | 145 ->GetURLRequestContext())); |
146 | 146 |
147 dom_distiller::proto::DomDistillerOptions options; | 147 dom_distiller::proto::DomDistillerOptions options; |
148 if (base::CommandLine::ForCurrentProcess()->HasSwitch(kExtractTextOnly)) { | 148 if (base::CommandLine::ForCurrentProcess()->HasSwitch(kExtractTextOnly)) { |
149 options.set_extract_text_only(true); | 149 options.set_extract_text_only(true); |
150 } | 150 } |
151 int debug_level = 0; | 151 int debug_level = 0; |
152 if (base::CommandLine::ForCurrentProcess()->HasSwitch(kDebugLevel) && | 152 if (base::CommandLine::ForCurrentProcess()->HasSwitch(kDebugLevel) && |
153 base::StringToInt( | 153 base::StringToInt( |
154 base::CommandLine::ForCurrentProcess()->GetSwitchValueASCII( | 154 base::CommandLine::ForCurrentProcess()->GetSwitchValueASCII( |
155 kDebugLevel), | 155 kDebugLevel), |
156 &debug_level)) { | 156 &debug_level)) { |
157 options.set_debug_level(debug_level); | 157 options.set_debug_level(debug_level); |
158 } | 158 } |
159 // Options for pagination algorithm: | 159 // Options for pagination algorithm: |
160 // - "next": detect anchors with "next" text | 160 // - "next": detect anchors with "next" text |
161 // - "pagenum": detect anchors with numeric page numbers | 161 // - "pagenum": detect anchors with numeric page numbers |
162 // Default is "next". | 162 // Default is "next". |
163 if (base::CommandLine::ForCurrentProcess()->HasSwitch(kPaginationAlgo)) { | 163 if (base::CommandLine::ForCurrentProcess()->HasSwitch(kPaginationAlgo)) { |
164 options.set_pagination_algo( | 164 options.set_pagination_algo( |
165 base::CommandLine::ForCurrentProcess()->GetSwitchValueASCII( | 165 base::CommandLine::ForCurrentProcess()->GetSwitchValueASCII( |
166 kPaginationAlgo)); | 166 kPaginationAlgo)); |
167 } | 167 } |
168 | 168 |
169 scoped_ptr<DistillerFactory> distiller_factory(new TestDistillerFactoryImpl( | 169 std::unique_ptr<DistillerFactory> distiller_factory( |
170 std::move(distiller_url_fetcher_factory), options, file_to_url_map)); | 170 new TestDistillerFactoryImpl(std::move(distiller_url_fetcher_factory), |
| 171 options, file_to_url_map)); |
171 | 172 |
172 // Setting up PrefService for DistilledPagePrefs. | 173 // Setting up PrefService for DistilledPagePrefs. |
173 user_prefs::TestingPrefServiceSyncable* pref_service = | 174 user_prefs::TestingPrefServiceSyncable* pref_service = |
174 new user_prefs::TestingPrefServiceSyncable(); | 175 new user_prefs::TestingPrefServiceSyncable(); |
175 DistilledPagePrefs::RegisterProfilePrefs(pref_service->registry()); | 176 DistilledPagePrefs::RegisterProfilePrefs(pref_service->registry()); |
176 | 177 |
177 return scoped_ptr<DomDistillerService>(new DomDistillerService( | 178 return std::unique_ptr<DomDistillerService>(new DomDistillerService( |
178 std::move(dom_distiller_store), std::move(distiller_factory), | 179 std::move(dom_distiller_store), std::move(distiller_factory), |
179 std::move(distiller_page_factory), | 180 std::move(distiller_page_factory), |
180 scoped_ptr<DistilledPagePrefs>(new DistilledPagePrefs(pref_service)))); | 181 std::unique_ptr<DistilledPagePrefs>( |
| 182 new DistilledPagePrefs(pref_service)))); |
181 } | 183 } |
182 | 184 |
183 void AddComponentsTestResources() { | 185 void AddComponentsTestResources() { |
184 base::FilePath pak_file; | 186 base::FilePath pak_file; |
185 base::FilePath pak_dir; | 187 base::FilePath pak_dir; |
186 PathService::Get(base::DIR_MODULE, &pak_dir); | 188 PathService::Get(base::DIR_MODULE, &pak_dir); |
187 pak_file = | 189 pak_file = |
188 pak_dir.Append(FILE_PATH_LITERAL("components_tests_resources.pak")); | 190 pak_dir.Append(FILE_PATH_LITERAL("components_tests_resources.pak")); |
189 ui::ResourceBundle::GetSharedInstance().AddDataPackFromPath( | 191 ui::ResourceBundle::GetSharedInstance().AddDataPackFromPath( |
190 pak_file, ui::SCALE_FACTOR_NONE); | 192 pak_file, ui::SCALE_FACTOR_NONE); |
(...skipping 113 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
304 void OnArticleUpdated(ArticleDistillationUpdate article_update) override {} | 306 void OnArticleUpdated(ArticleDistillationUpdate article_update) override {} |
305 | 307 |
306 void OnArticleReady(const DistilledArticleProto* article_proto) override { | 308 void OnArticleReady(const DistilledArticleProto* article_proto) override { |
307 article_proto_ = article_proto; | 309 article_proto_ = article_proto; |
308 CHECK(article_proto->pages_size()) << "Failed extracting " << url_; | 310 CHECK(article_proto->pages_size()) << "Failed extracting " << url_; |
309 base::ThreadTaskRunnerHandle::Get()->PostTask(FROM_HERE, | 311 base::ThreadTaskRunnerHandle::Get()->PostTask(FROM_HERE, |
310 finished_callback_); | 312 finished_callback_); |
311 } | 313 } |
312 | 314 |
313 const DistilledArticleProto* article_proto_; | 315 const DistilledArticleProto* article_proto_; |
314 scoped_ptr<ViewerHandle> viewer_handle_; | 316 std::unique_ptr<ViewerHandle> viewer_handle_; |
315 GURL url_; | 317 GURL url_; |
316 base::Closure finished_callback_; | 318 base::Closure finished_callback_; |
317 }; | 319 }; |
318 | 320 |
319 class ContentExtractor : public ContentBrowserTest { | 321 class ContentExtractor : public ContentBrowserTest { |
320 public: | 322 public: |
321 ContentExtractor() | 323 ContentExtractor() |
322 : pending_tasks_(0), | 324 : pending_tasks_(0), |
323 max_tasks_(kMaxExtractorTasks), | 325 max_tasks_(kMaxExtractorTasks), |
324 next_request_(0), | 326 next_request_(0), |
(...skipping 95 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
420 service_.reset(); | 422 service_.reset(); |
421 base::ThreadTaskRunnerHandle::Get()->PostTask( | 423 base::ThreadTaskRunnerHandle::Get()->PostTask( |
422 FROM_HERE, base::MessageLoop::QuitWhenIdleClosure()); | 424 FROM_HERE, base::MessageLoop::QuitWhenIdleClosure()); |
423 } | 425 } |
424 | 426 |
425 size_t pending_tasks_; | 427 size_t pending_tasks_; |
426 size_t max_tasks_; | 428 size_t max_tasks_; |
427 size_t next_request_; | 429 size_t next_request_; |
428 | 430 |
429 base::ScopedTempDir db_dir_; | 431 base::ScopedTempDir db_dir_; |
430 scoped_ptr<net::ScopedDefaultHostResolverProc> mock_host_resolver_override_; | 432 std::unique_ptr<net::ScopedDefaultHostResolverProc> |
431 scoped_ptr<DomDistillerService> service_; | 433 mock_host_resolver_override_; |
| 434 std::unique_ptr<DomDistillerService> service_; |
432 ScopedVector<ContentExtractionRequest> requests_; | 435 ScopedVector<ContentExtractionRequest> requests_; |
433 | 436 |
434 std::string output_data_; | 437 std::string output_data_; |
435 scoped_ptr<google::protobuf::io::StringOutputStream> protobuf_output_stream_; | 438 std::unique_ptr<google::protobuf::io::StringOutputStream> |
| 439 protobuf_output_stream_; |
436 }; | 440 }; |
437 | 441 |
438 IN_PROC_BROWSER_TEST_F(ContentExtractor, MANUAL_ExtractUrl) { | 442 IN_PROC_BROWSER_TEST_F(ContentExtractor, MANUAL_ExtractUrl) { |
439 SetDistillerJavaScriptWorldId(content::ISOLATED_WORLD_ID_CONTENT_END); | 443 SetDistillerJavaScriptWorldId(content::ISOLATED_WORLD_ID_CONTENT_END); |
440 Start(); | 444 Start(); |
441 base::RunLoop().Run(); | 445 base::RunLoop().Run(); |
442 } | 446 } |
443 | 447 |
444 } // namespace dom_distiller | 448 } // namespace dom_distiller |
OLD | NEW |