OLD | NEW |
| (Empty) |
1 // Copyright 2016 The Chromium Authors. All rights reserved. | |
2 // Use of this source code is governed by a BSD-style license that can be | |
3 // found in the LICENSE file. | |
4 | |
5 #include "chrome/browser/android/offline_pages/background_loader_offliner.h" | |
6 | |
7 #include "base/bind.h" | |
8 #include "base/json/json_writer.h" | |
9 #include "base/metrics/histogram_macros.h" | |
10 #include "base/sys_info.h" | |
11 #include "base/threading/thread_task_runner_handle.h" | |
12 #include "base/time/time.h" | |
13 #include "chrome/browser/android/offline_pages/offline_page_mhtml_archiver.h" | |
14 #include "chrome/browser/android/offline_pages/offliner_helper.h" | |
15 #include "chrome/browser/profiles/profile.h" | |
16 #include "components/offline_pages/core/background/offliner_policy.h" | |
17 #include "components/offline_pages/core/background/save_page_request.h" | |
18 #include "components/offline_pages/core/client_namespace_constants.h" | |
19 #include "components/offline_pages/core/offline_page_feature.h" | |
20 #include "components/offline_pages/core/offline_page_model.h" | |
21 #include "content/public/browser/browser_context.h" | |
22 #include "content/public/browser/mhtml_extra_parts.h" | |
23 #include "content/public/browser/navigation_handle.h" | |
24 #include "content/public/browser/render_frame_host.h" | |
25 #include "content/public/browser/web_contents.h" | |
26 #include "content/public/browser/web_contents_user_data.h" | |
27 | |
28 namespace offline_pages { | |
29 | |
30 namespace { | |
31 const char kContentType[] = "text/plain"; | |
32 const char kContentTransferEncodingBinary[] = | |
33 "Content-Transfer-Encoding: binary"; | |
34 const char kXHeaderForSignals[] = "X-Chrome-Loading-Metrics-Data: 1"; | |
35 | |
36 class OfflinerData : public content::WebContentsUserData<OfflinerData> { | |
37 public: | |
38 static void AddToWebContents(content::WebContents* webcontents, | |
39 BackgroundLoaderOffliner* offliner) { | |
40 DCHECK(offliner); | |
41 webcontents->SetUserData(UserDataKey(), std::unique_ptr<OfflinerData>( | |
42 new OfflinerData(offliner))); | |
43 } | |
44 | |
45 explicit OfflinerData(BackgroundLoaderOffliner* offliner) { | |
46 offliner_ = offliner; | |
47 } | |
48 BackgroundLoaderOffliner* offliner() { return offliner_; } | |
49 | |
50 private: | |
51 // The offliner that the WebContents is attached to. The offliner owns the | |
52 // Delegate which owns the WebContents that this data is attached to. | |
53 // Therefore, its lifetime should exceed that of the WebContents, so this | |
54 // should always be non-null. | |
55 BackgroundLoaderOffliner* offliner_; | |
56 }; | |
57 | |
58 std::string AddHistogramSuffix(const ClientId& client_id, | |
59 const char* histogram_name) { | |
60 if (client_id.name_space.empty()) { | |
61 NOTREACHED(); | |
62 return histogram_name; | |
63 } | |
64 std::string adjusted_histogram_name(histogram_name); | |
65 adjusted_histogram_name += "." + client_id.name_space; | |
66 return adjusted_histogram_name; | |
67 } | |
68 | |
69 void RecordErrorCauseUMA(const ClientId& client_id, net::Error error_code) { | |
70 UMA_HISTOGRAM_SPARSE_SLOWLY( | |
71 AddHistogramSuffix(client_id, | |
72 "OfflinePages.Background.BackgroundLoadingFailedCode"), | |
73 std::abs(error_code)); | |
74 } | |
75 | |
76 void HandleApplicationStateChangeCancel( | |
77 const Offliner::CompletionCallback& completion_callback, | |
78 const SavePageRequest& canceled_request) { | |
79 completion_callback.Run(canceled_request, | |
80 Offliner::RequestStatus::FOREGROUND_CANCELED); | |
81 } | |
82 | |
83 } // namespace | |
84 | |
85 BackgroundLoaderOffliner::BackgroundLoaderOffliner( | |
86 content::BrowserContext* browser_context, | |
87 const OfflinerPolicy* policy, | |
88 OfflinePageModel* offline_page_model) | |
89 : browser_context_(browser_context), | |
90 offline_page_model_(offline_page_model), | |
91 policy_(policy), | |
92 is_low_end_device_(base::SysInfo::IsLowEndDevice()), | |
93 save_state_(NONE), | |
94 page_load_state_(SUCCESS), | |
95 network_bytes_(0LL), | |
96 is_low_bar_met_(false), | |
97 did_snapshot_on_last_retry_(false), | |
98 weak_ptr_factory_(this) { | |
99 DCHECK(offline_page_model_); | |
100 DCHECK(browser_context_); | |
101 } | |
102 | |
103 BackgroundLoaderOffliner::~BackgroundLoaderOffliner() {} | |
104 | |
105 // static | |
106 BackgroundLoaderOffliner* BackgroundLoaderOffliner::FromWebContents( | |
107 content::WebContents* contents) { | |
108 OfflinerData* data = OfflinerData::FromWebContents(contents); | |
109 if (data) | |
110 return data->offliner(); | |
111 return nullptr; | |
112 } | |
113 | |
114 bool BackgroundLoaderOffliner::LoadAndSave( | |
115 const SavePageRequest& request, | |
116 const CompletionCallback& completion_callback, | |
117 const ProgressCallback& progress_callback) { | |
118 DCHECK(completion_callback); | |
119 DCHECK(progress_callback); | |
120 DCHECK(offline_page_model_); | |
121 | |
122 if (pending_request_) { | |
123 DVLOG(1) << "Already have pending request"; | |
124 return false; | |
125 } | |
126 | |
127 ClientPolicyController* policy_controller = | |
128 offline_page_model_->GetPolicyController(); | |
129 if (policy_controller->IsDisabledWhenPrefetchDisabled( | |
130 request.client_id().name_space) && | |
131 (AreThirdPartyCookiesBlocked(browser_context_) || | |
132 IsNetworkPredictionDisabled(browser_context_))) { | |
133 DVLOG(1) << "WARNING: Unable to load when 3rd party cookies blocked or " | |
134 << "prediction disabled"; | |
135 // Record user metrics for third party cookies being disabled or network | |
136 // prediction being disabled. | |
137 if (AreThirdPartyCookiesBlocked(browser_context_)) { | |
138 UMA_HISTOGRAM_ENUMERATION( | |
139 "OfflinePages.Background.CctApiDisableStatus", | |
140 static_cast<int>(OfflinePagesCctApiPrerenderAllowedStatus:: | |
141 THIRD_PARTY_COOKIES_DISABLED), | |
142 static_cast<int>(OfflinePagesCctApiPrerenderAllowedStatus:: | |
143 NETWORK_PREDICTION_DISABLED) + | |
144 1); | |
145 } | |
146 if (IsNetworkPredictionDisabled(browser_context_)) { | |
147 UMA_HISTOGRAM_ENUMERATION( | |
148 "OfflinePages.Background.CctApiDisableStatus", | |
149 static_cast<int>(OfflinePagesCctApiPrerenderAllowedStatus:: | |
150 NETWORK_PREDICTION_DISABLED), | |
151 static_cast<int>(OfflinePagesCctApiPrerenderAllowedStatus:: | |
152 NETWORK_PREDICTION_DISABLED) + | |
153 1); | |
154 } | |
155 | |
156 return false; | |
157 } | |
158 | |
159 // Record UMA that the load was allowed to proceed. | |
160 if (request.client_id().name_space == kCCTNamespace) { | |
161 UMA_HISTOGRAM_ENUMERATION( | |
162 "OfflinePages.Background.CctApiDisableStatus", | |
163 static_cast<int>( | |
164 OfflinePagesCctApiPrerenderAllowedStatus::PRERENDER_ALLOWED), | |
165 static_cast<int>(OfflinePagesCctApiPrerenderAllowedStatus:: | |
166 NETWORK_PREDICTION_DISABLED) + | |
167 1); | |
168 } | |
169 | |
170 if (!OfflinePageModel::CanSaveURL(request.url())) { | |
171 DVLOG(1) << "Not able to save page for requested url: " << request.url(); | |
172 return false; | |
173 } | |
174 | |
175 ResetLoader(); | |
176 AttachObservers(); | |
177 | |
178 MarkLoadStartTime(); | |
179 | |
180 // Track copy of pending request. | |
181 pending_request_.reset(new SavePageRequest(request)); | |
182 completion_callback_ = completion_callback; | |
183 progress_callback_ = progress_callback; | |
184 | |
185 // Listen for app foreground/background change. | |
186 app_listener_.reset(new base::android::ApplicationStatusListener( | |
187 base::Bind(&BackgroundLoaderOffliner::OnApplicationStateChange, | |
188 weak_ptr_factory_.GetWeakPtr()))); | |
189 | |
190 // Load page attempt. | |
191 loader_.get()->LoadPage(request.url()); | |
192 | |
193 snapshot_controller_ = SnapshotController::CreateForBackgroundOfflining( | |
194 base::ThreadTaskRunnerHandle::Get(), this); | |
195 | |
196 return true; | |
197 } | |
198 | |
199 bool BackgroundLoaderOffliner::Cancel(const CancelCallback& callback) { | |
200 DCHECK(pending_request_); | |
201 // We ignore the case where pending_request_ is not set, but given the checks | |
202 // in RequestCoordinator this should not happen. | |
203 if (!pending_request_) | |
204 return false; | |
205 | |
206 // TODO(chili): We are not able to cancel a pending | |
207 // OfflinePageModel::SaveSnapshot() operation. We will notify caller that | |
208 // cancel completed once the SavePage operation returns. | |
209 if (save_state_ != NONE) { | |
210 save_state_ = DELETE_AFTER_SAVE; | |
211 cancel_callback_ = callback; | |
212 return true; | |
213 } | |
214 | |
215 // Post the cancel callback right after this call concludes. | |
216 base::ThreadTaskRunnerHandle::Get()->PostTask( | |
217 FROM_HERE, base::Bind(callback, *pending_request_.get())); | |
218 ResetState(); | |
219 return true; | |
220 } | |
221 | |
222 bool BackgroundLoaderOffliner::HandleTimeout(int64_t request_id) { | |
223 if (pending_request_) { | |
224 DCHECK(request_id == pending_request_->request_id()); | |
225 if (is_low_bar_met_ && (pending_request_->started_attempt_count() + 1 >= | |
226 policy_->GetMaxStartedTries() || | |
227 pending_request_->completed_attempt_count() + 1 >= | |
228 policy_->GetMaxCompletedTries())) { | |
229 // If we are already in the middle of a save operation, let it finish | |
230 // but do not return SAVED_ON_LAST_RETRY | |
231 if (save_state_ == NONE) { | |
232 did_snapshot_on_last_retry_ = true; | |
233 StartSnapshot(); | |
234 } | |
235 return true; | |
236 } | |
237 } | |
238 return false; | |
239 } | |
240 | |
241 void BackgroundLoaderOffliner::MarkLoadStartTime() { | |
242 load_start_time_ = base::TimeTicks::Now(); | |
243 } | |
244 | |
245 void BackgroundLoaderOffliner::DocumentAvailableInMainFrame() { | |
246 snapshot_controller_->DocumentAvailableInMainFrame(); | |
247 is_low_bar_met_ = true; | |
248 | |
249 // Add this signal to signal_data_. | |
250 AddLoadingSignal("DocumentAvailableInMainFrame"); | |
251 } | |
252 | |
253 void BackgroundLoaderOffliner::DocumentOnLoadCompletedInMainFrame() { | |
254 if (!pending_request_.get()) { | |
255 DVLOG(1) << "DidStopLoading called even though no pending request."; | |
256 return; | |
257 } | |
258 | |
259 // Add this signal to signal_data_. | |
260 AddLoadingSignal("DocumentOnLoadCompletedInMainFrame"); | |
261 | |
262 snapshot_controller_->DocumentOnLoadCompletedInMainFrame(); | |
263 } | |
264 | |
265 void BackgroundLoaderOffliner::RenderProcessGone( | |
266 base::TerminationStatus status) { | |
267 if (pending_request_) { | |
268 SavePageRequest request(*pending_request_.get()); | |
269 switch (status) { | |
270 case base::TERMINATION_STATUS_OOM: | |
271 case base::TERMINATION_STATUS_PROCESS_CRASHED: | |
272 case base::TERMINATION_STATUS_STILL_RUNNING: | |
273 completion_callback_.Run( | |
274 request, Offliner::RequestStatus::LOADING_FAILED_NO_NEXT); | |
275 break; | |
276 case base::TERMINATION_STATUS_PROCESS_WAS_KILLED: | |
277 default: | |
278 completion_callback_.Run(request, | |
279 Offliner::RequestStatus::LOADING_FAILED); | |
280 } | |
281 ResetState(); | |
282 } | |
283 } | |
284 | |
285 void BackgroundLoaderOffliner::WebContentsDestroyed() { | |
286 if (pending_request_) { | |
287 SavePageRequest request(*pending_request_.get()); | |
288 completion_callback_.Run(request, Offliner::RequestStatus::LOADING_FAILED); | |
289 ResetState(); | |
290 } | |
291 } | |
292 | |
293 void BackgroundLoaderOffliner::DidFinishNavigation( | |
294 content::NavigationHandle* navigation_handle) { | |
295 if (!navigation_handle->IsInMainFrame()) | |
296 return; | |
297 // If there was an error of any kind (certificate, client, DNS, etc), | |
298 // Mark as error page. Resetting here causes RecordNavigationMetrics to crash. | |
299 if (navigation_handle->IsErrorPage()) { | |
300 RecordErrorCauseUMA(pending_request_->client_id(), | |
301 navigation_handle->GetNetErrorCode()); | |
302 switch (navigation_handle->GetNetErrorCode()) { | |
303 case net::ERR_INTERNET_DISCONNECTED: | |
304 page_load_state_ = DELAY_RETRY; | |
305 break; | |
306 default: | |
307 page_load_state_ = RETRIABLE; | |
308 } | |
309 } | |
310 } | |
311 | |
312 void BackgroundLoaderOffliner::SetSnapshotControllerForTest( | |
313 std::unique_ptr<SnapshotController> controller) { | |
314 snapshot_controller_ = std::move(controller); | |
315 } | |
316 | |
317 void BackgroundLoaderOffliner::OnNetworkBytesChanged(int64_t bytes) { | |
318 if (pending_request_ && save_state_ != SAVING) { | |
319 network_bytes_ += bytes; | |
320 progress_callback_.Run(*pending_request_, network_bytes_); | |
321 } | |
322 } | |
323 | |
324 void BackgroundLoaderOffliner::StartSnapshot() { | |
325 if (!pending_request_.get()) { | |
326 DVLOG(1) << "Pending request was cleared during delay."; | |
327 return; | |
328 } | |
329 | |
330 // Add this signal to signal_data_. | |
331 AddLoadingSignal("Snapshotting"); | |
332 | |
333 SavePageRequest request(*pending_request_.get()); | |
334 // If there was an error navigating to page, return loading failed. | |
335 if (page_load_state_ != SUCCESS) { | |
336 Offliner::RequestStatus status; | |
337 switch (page_load_state_) { | |
338 case RETRIABLE: | |
339 status = Offliner::RequestStatus::LOADING_FAILED; | |
340 break; | |
341 case NONRETRIABLE: | |
342 status = Offliner::RequestStatus::LOADING_FAILED_NO_RETRY; | |
343 break; | |
344 case DELAY_RETRY: | |
345 status = Offliner::RequestStatus::LOADING_FAILED_NO_NEXT; | |
346 break; | |
347 default: | |
348 // We should've already checked for Success before entering here. | |
349 NOTREACHED(); | |
350 status = Offliner::RequestStatus::LOADING_FAILED; | |
351 } | |
352 | |
353 completion_callback_.Run(request, status); | |
354 ResetState(); | |
355 return; | |
356 } | |
357 | |
358 save_state_ = SAVING; | |
359 content::WebContents* web_contents( | |
360 content::WebContentsObserver::web_contents()); | |
361 | |
362 // Add loading signal into the MHTML that will be generated if the command | |
363 // line flag is set for it. | |
364 if (IsOfflinePagesLoadSignalCollectingEnabled()) { | |
365 // Stash loading signals for writing when we write out the MHTML. | |
366 std::string headers = base::StringPrintf( | |
367 "%s\r\n%s\r\n\r\n", kContentTransferEncodingBinary, kXHeaderForSignals); | |
368 std::string body; | |
369 base::JSONWriter::Write(signal_data_, &body); | |
370 std::string content_type = kContentType; | |
371 std::string content_location = base::StringPrintf( | |
372 "cid:signal-data-%" PRId64 "@mhtml.blink", request.request_id()); | |
373 | |
374 content::MHTMLExtraParts* extra_parts = | |
375 content::MHTMLExtraParts::FromWebContents(web_contents); | |
376 DCHECK(extra_parts); | |
377 if (extra_parts != nullptr) { | |
378 extra_parts->AddExtraMHTMLPart(content_type, content_location, headers, | |
379 body); | |
380 } | |
381 } | |
382 | |
383 std::unique_ptr<OfflinePageArchiver> archiver( | |
384 new OfflinePageMHTMLArchiver(web_contents)); | |
385 | |
386 OfflinePageModel::SavePageParams params; | |
387 params.url = web_contents->GetLastCommittedURL(); | |
388 params.client_id = request.client_id(); | |
389 params.proposed_offline_id = request.request_id(); | |
390 params.is_background = true; | |
391 | |
392 // Pass in the original URL if it's different from last committed | |
393 // when redirects occur. | |
394 if (!request.original_url().is_empty()) | |
395 params.original_url = request.original_url(); | |
396 else if (params.url != request.url()) | |
397 params.original_url = request.url(); | |
398 | |
399 offline_page_model_->SavePage( | |
400 params, std::move(archiver), | |
401 base::Bind(&BackgroundLoaderOffliner::OnPageSaved, | |
402 weak_ptr_factory_.GetWeakPtr())); | |
403 } | |
404 | |
405 void BackgroundLoaderOffliner::OnPageSaved(SavePageResult save_result, | |
406 int64_t offline_id) { | |
407 if (!pending_request_) | |
408 return; | |
409 | |
410 SavePageRequest request(*pending_request_.get()); | |
411 bool did_snapshot_on_last_retry = did_snapshot_on_last_retry_; | |
412 ResetState(); | |
413 | |
414 if (save_state_ == DELETE_AFTER_SAVE) { | |
415 // Delete the saved page off disk and from the OPM. | |
416 std::vector<int64_t> offline_ids; | |
417 offline_ids.push_back(offline_id); | |
418 offline_page_model_->DeletePagesByOfflineId( | |
419 offline_ids, | |
420 base::Bind(&BackgroundLoaderOffliner::DeleteOfflinePageCallback, | |
421 weak_ptr_factory_.GetWeakPtr(), request)); | |
422 save_state_ = NONE; | |
423 return; | |
424 } | |
425 | |
426 save_state_ = NONE; | |
427 | |
428 Offliner::RequestStatus save_status; | |
429 if (save_result == SavePageResult::ALREADY_EXISTS) { | |
430 save_status = RequestStatus::SAVED; | |
431 } else if (save_result == SavePageResult::SUCCESS) { | |
432 if (did_snapshot_on_last_retry) | |
433 save_status = RequestStatus::SAVED_ON_LAST_RETRY; | |
434 else | |
435 save_status = RequestStatus::SAVED; | |
436 } else { | |
437 save_status = RequestStatus::SAVE_FAILED; | |
438 } | |
439 | |
440 completion_callback_.Run(request, save_status); | |
441 } | |
442 | |
443 void BackgroundLoaderOffliner::DeleteOfflinePageCallback( | |
444 const SavePageRequest& request, | |
445 DeletePageResult result) { | |
446 cancel_callback_.Run(request); | |
447 } | |
448 | |
449 void BackgroundLoaderOffliner::ResetState() { | |
450 pending_request_.reset(); | |
451 snapshot_controller_.reset(); | |
452 page_load_state_ = SUCCESS; | |
453 network_bytes_ = 0LL; | |
454 is_low_bar_met_ = false; | |
455 did_snapshot_on_last_retry_ = false; | |
456 content::WebContentsObserver::Observe(nullptr); | |
457 loader_.reset(); | |
458 } | |
459 | |
460 void BackgroundLoaderOffliner::ResetLoader() { | |
461 loader_.reset( | |
462 new background_loader::BackgroundLoaderContents(browser_context_)); | |
463 } | |
464 | |
465 void BackgroundLoaderOffliner::AttachObservers() { | |
466 content::WebContents* contents = loader_->web_contents(); | |
467 content::WebContentsObserver::Observe(contents); | |
468 OfflinerData::AddToWebContents(contents, this); | |
469 } | |
470 | |
471 void BackgroundLoaderOffliner::OnApplicationStateChange( | |
472 base::android::ApplicationState application_state) { | |
473 if (pending_request_ && is_low_end_device_ && | |
474 application_state == | |
475 base::android::APPLICATION_STATE_HAS_RUNNING_ACTIVITIES) { | |
476 DVLOG(1) << "App became active, canceling current offlining request"; | |
477 // No need to check the return value or complete early, as false would | |
478 // indicate that there was no request, in which case the state change is | |
479 // ignored. | |
480 Cancel( | |
481 base::Bind(HandleApplicationStateChangeCancel, completion_callback_)); | |
482 } | |
483 } | |
484 | |
485 void BackgroundLoaderOffliner::AddLoadingSignal(const char* signal_name) { | |
486 base::TimeTicks current_time = base::TimeTicks::Now(); | |
487 base::TimeDelta delay_so_far = current_time - load_start_time_; | |
488 // We would prefer to use int64_t here, but JSON does not support that type. | |
489 // Given the choice between int and double, we choose to implicitly convert to | |
490 // a double since it maintains more precision (we can get a longer time in | |
491 // milliseconds than we can with a 2 bit int, 53 bits vs 32). | |
492 double delay = delay_so_far.InMilliseconds(); | |
493 signal_data_.SetDouble(signal_name, delay); | |
494 } | |
495 | |
496 } // namespace offline_pages | |
497 | |
498 DEFINE_WEB_CONTENTS_USER_DATA_KEY(offline_pages::OfflinerData); | |
OLD | NEW |