| OLD | NEW |
| (Empty) |
| 1 // Copyright 2016 The Chromium Authors. All rights reserved. | |
| 2 // Use of this source code is governed by a BSD-style license that can be | |
| 3 // found in the LICENSE file. | |
| 4 | |
| 5 #include "chrome/browser/android/offline_pages/background_loader_offliner.h" | |
| 6 | |
| 7 #include "base/bind.h" | |
| 8 #include "base/json/json_writer.h" | |
| 9 #include "base/metrics/histogram_macros.h" | |
| 10 #include "base/sys_info.h" | |
| 11 #include "base/threading/thread_task_runner_handle.h" | |
| 12 #include "base/time/time.h" | |
| 13 #include "chrome/browser/android/offline_pages/offline_page_mhtml_archiver.h" | |
| 14 #include "chrome/browser/android/offline_pages/offliner_helper.h" | |
| 15 #include "chrome/browser/profiles/profile.h" | |
| 16 #include "components/offline_pages/core/background/offliner_policy.h" | |
| 17 #include "components/offline_pages/core/background/save_page_request.h" | |
| 18 #include "components/offline_pages/core/client_namespace_constants.h" | |
| 19 #include "components/offline_pages/core/offline_page_feature.h" | |
| 20 #include "components/offline_pages/core/offline_page_model.h" | |
| 21 #include "content/public/browser/browser_context.h" | |
| 22 #include "content/public/browser/mhtml_extra_parts.h" | |
| 23 #include "content/public/browser/navigation_handle.h" | |
| 24 #include "content/public/browser/render_frame_host.h" | |
| 25 #include "content/public/browser/web_contents.h" | |
| 26 #include "content/public/browser/web_contents_user_data.h" | |
| 27 | |
| 28 namespace offline_pages { | |
| 29 | |
| 30 namespace { | |
| 31 const char kContentType[] = "text/plain"; | |
| 32 const char kContentTransferEncodingBinary[] = | |
| 33 "Content-Transfer-Encoding: binary"; | |
| 34 const char kXHeaderForSignals[] = "X-Chrome-Loading-Metrics-Data: 1"; | |
| 35 | |
| 36 class OfflinerData : public content::WebContentsUserData<OfflinerData> { | |
| 37 public: | |
| 38 static void AddToWebContents(content::WebContents* webcontents, | |
| 39 BackgroundLoaderOffliner* offliner) { | |
| 40 DCHECK(offliner); | |
| 41 webcontents->SetUserData(UserDataKey(), std::unique_ptr<OfflinerData>( | |
| 42 new OfflinerData(offliner))); | |
| 43 } | |
| 44 | |
| 45 explicit OfflinerData(BackgroundLoaderOffliner* offliner) { | |
| 46 offliner_ = offliner; | |
| 47 } | |
| 48 BackgroundLoaderOffliner* offliner() { return offliner_; } | |
| 49 | |
| 50 private: | |
| 51 // The offliner that the WebContents is attached to. The offliner owns the | |
| 52 // Delegate which owns the WebContents that this data is attached to. | |
| 53 // Therefore, its lifetime should exceed that of the WebContents, so this | |
| 54 // should always be non-null. | |
| 55 BackgroundLoaderOffliner* offliner_; | |
| 56 }; | |
| 57 | |
| 58 std::string AddHistogramSuffix(const ClientId& client_id, | |
| 59 const char* histogram_name) { | |
| 60 if (client_id.name_space.empty()) { | |
| 61 NOTREACHED(); | |
| 62 return histogram_name; | |
| 63 } | |
| 64 std::string adjusted_histogram_name(histogram_name); | |
| 65 adjusted_histogram_name += "." + client_id.name_space; | |
| 66 return adjusted_histogram_name; | |
| 67 } | |
| 68 | |
| 69 void RecordErrorCauseUMA(const ClientId& client_id, net::Error error_code) { | |
| 70 UMA_HISTOGRAM_SPARSE_SLOWLY( | |
| 71 AddHistogramSuffix(client_id, | |
| 72 "OfflinePages.Background.BackgroundLoadingFailedCode"), | |
| 73 std::abs(error_code)); | |
| 74 } | |
| 75 | |
| 76 void HandleApplicationStateChangeCancel( | |
| 77 const Offliner::CompletionCallback& completion_callback, | |
| 78 const SavePageRequest& canceled_request) { | |
| 79 completion_callback.Run(canceled_request, | |
| 80 Offliner::RequestStatus::FOREGROUND_CANCELED); | |
| 81 } | |
| 82 | |
| 83 } // namespace | |
| 84 | |
| 85 BackgroundLoaderOffliner::BackgroundLoaderOffliner( | |
| 86 content::BrowserContext* browser_context, | |
| 87 const OfflinerPolicy* policy, | |
| 88 OfflinePageModel* offline_page_model) | |
| 89 : browser_context_(browser_context), | |
| 90 offline_page_model_(offline_page_model), | |
| 91 policy_(policy), | |
| 92 is_low_end_device_(base::SysInfo::IsLowEndDevice()), | |
| 93 save_state_(NONE), | |
| 94 page_load_state_(SUCCESS), | |
| 95 network_bytes_(0LL), | |
| 96 is_low_bar_met_(false), | |
| 97 did_snapshot_on_last_retry_(false), | |
| 98 weak_ptr_factory_(this) { | |
| 99 DCHECK(offline_page_model_); | |
| 100 DCHECK(browser_context_); | |
| 101 } | |
| 102 | |
| 103 BackgroundLoaderOffliner::~BackgroundLoaderOffliner() {} | |
| 104 | |
| 105 // static | |
| 106 BackgroundLoaderOffliner* BackgroundLoaderOffliner::FromWebContents( | |
| 107 content::WebContents* contents) { | |
| 108 OfflinerData* data = OfflinerData::FromWebContents(contents); | |
| 109 if (data) | |
| 110 return data->offliner(); | |
| 111 return nullptr; | |
| 112 } | |
| 113 | |
| 114 bool BackgroundLoaderOffliner::LoadAndSave( | |
| 115 const SavePageRequest& request, | |
| 116 const CompletionCallback& completion_callback, | |
| 117 const ProgressCallback& progress_callback) { | |
| 118 DCHECK(completion_callback); | |
| 119 DCHECK(progress_callback); | |
| 120 DCHECK(offline_page_model_); | |
| 121 | |
| 122 if (pending_request_) { | |
| 123 DVLOG(1) << "Already have pending request"; | |
| 124 return false; | |
| 125 } | |
| 126 | |
| 127 ClientPolicyController* policy_controller = | |
| 128 offline_page_model_->GetPolicyController(); | |
| 129 if (policy_controller->IsDisabledWhenPrefetchDisabled( | |
| 130 request.client_id().name_space) && | |
| 131 (AreThirdPartyCookiesBlocked(browser_context_) || | |
| 132 IsNetworkPredictionDisabled(browser_context_))) { | |
| 133 DVLOG(1) << "WARNING: Unable to load when 3rd party cookies blocked or " | |
| 134 << "prediction disabled"; | |
| 135 // Record user metrics for third party cookies being disabled or network | |
| 136 // prediction being disabled. | |
| 137 if (AreThirdPartyCookiesBlocked(browser_context_)) { | |
| 138 UMA_HISTOGRAM_ENUMERATION( | |
| 139 "OfflinePages.Background.CctApiDisableStatus", | |
| 140 static_cast<int>(OfflinePagesCctApiPrerenderAllowedStatus:: | |
| 141 THIRD_PARTY_COOKIES_DISABLED), | |
| 142 static_cast<int>(OfflinePagesCctApiPrerenderAllowedStatus:: | |
| 143 NETWORK_PREDICTION_DISABLED) + | |
| 144 1); | |
| 145 } | |
| 146 if (IsNetworkPredictionDisabled(browser_context_)) { | |
| 147 UMA_HISTOGRAM_ENUMERATION( | |
| 148 "OfflinePages.Background.CctApiDisableStatus", | |
| 149 static_cast<int>(OfflinePagesCctApiPrerenderAllowedStatus:: | |
| 150 NETWORK_PREDICTION_DISABLED), | |
| 151 static_cast<int>(OfflinePagesCctApiPrerenderAllowedStatus:: | |
| 152 NETWORK_PREDICTION_DISABLED) + | |
| 153 1); | |
| 154 } | |
| 155 | |
| 156 return false; | |
| 157 } | |
| 158 | |
| 159 // Record UMA that the load was allowed to proceed. | |
| 160 if (request.client_id().name_space == kCCTNamespace) { | |
| 161 UMA_HISTOGRAM_ENUMERATION( | |
| 162 "OfflinePages.Background.CctApiDisableStatus", | |
| 163 static_cast<int>( | |
| 164 OfflinePagesCctApiPrerenderAllowedStatus::PRERENDER_ALLOWED), | |
| 165 static_cast<int>(OfflinePagesCctApiPrerenderAllowedStatus:: | |
| 166 NETWORK_PREDICTION_DISABLED) + | |
| 167 1); | |
| 168 } | |
| 169 | |
| 170 if (!OfflinePageModel::CanSaveURL(request.url())) { | |
| 171 DVLOG(1) << "Not able to save page for requested url: " << request.url(); | |
| 172 return false; | |
| 173 } | |
| 174 | |
| 175 ResetLoader(); | |
| 176 AttachObservers(); | |
| 177 | |
| 178 MarkLoadStartTime(); | |
| 179 | |
| 180 // Track copy of pending request. | |
| 181 pending_request_.reset(new SavePageRequest(request)); | |
| 182 completion_callback_ = completion_callback; | |
| 183 progress_callback_ = progress_callback; | |
| 184 | |
| 185 // Listen for app foreground/background change. | |
| 186 app_listener_.reset(new base::android::ApplicationStatusListener( | |
| 187 base::Bind(&BackgroundLoaderOffliner::OnApplicationStateChange, | |
| 188 weak_ptr_factory_.GetWeakPtr()))); | |
| 189 | |
| 190 // Load page attempt. | |
| 191 loader_.get()->LoadPage(request.url()); | |
| 192 | |
| 193 snapshot_controller_ = SnapshotController::CreateForBackgroundOfflining( | |
| 194 base::ThreadTaskRunnerHandle::Get(), this); | |
| 195 | |
| 196 return true; | |
| 197 } | |
| 198 | |
| 199 bool BackgroundLoaderOffliner::Cancel(const CancelCallback& callback) { | |
| 200 DCHECK(pending_request_); | |
| 201 // We ignore the case where pending_request_ is not set, but given the checks | |
| 202 // in RequestCoordinator this should not happen. | |
| 203 if (!pending_request_) | |
| 204 return false; | |
| 205 | |
| 206 // TODO(chili): We are not able to cancel a pending | |
| 207 // OfflinePageModel::SaveSnapshot() operation. We will notify caller that | |
| 208 // cancel completed once the SavePage operation returns. | |
| 209 if (save_state_ != NONE) { | |
| 210 save_state_ = DELETE_AFTER_SAVE; | |
| 211 cancel_callback_ = callback; | |
| 212 return true; | |
| 213 } | |
| 214 | |
| 215 // Post the cancel callback right after this call concludes. | |
| 216 base::ThreadTaskRunnerHandle::Get()->PostTask( | |
| 217 FROM_HERE, base::Bind(callback, *pending_request_.get())); | |
| 218 ResetState(); | |
| 219 return true; | |
| 220 } | |
| 221 | |
| 222 bool BackgroundLoaderOffliner::HandleTimeout(int64_t request_id) { | |
| 223 if (pending_request_) { | |
| 224 DCHECK(request_id == pending_request_->request_id()); | |
| 225 if (is_low_bar_met_ && (pending_request_->started_attempt_count() + 1 >= | |
| 226 policy_->GetMaxStartedTries() || | |
| 227 pending_request_->completed_attempt_count() + 1 >= | |
| 228 policy_->GetMaxCompletedTries())) { | |
| 229 // If we are already in the middle of a save operation, let it finish | |
| 230 // but do not return SAVED_ON_LAST_RETRY | |
| 231 if (save_state_ == NONE) { | |
| 232 did_snapshot_on_last_retry_ = true; | |
| 233 StartSnapshot(); | |
| 234 } | |
| 235 return true; | |
| 236 } | |
| 237 } | |
| 238 return false; | |
| 239 } | |
| 240 | |
| 241 void BackgroundLoaderOffliner::MarkLoadStartTime() { | |
| 242 load_start_time_ = base::TimeTicks::Now(); | |
| 243 } | |
| 244 | |
| 245 void BackgroundLoaderOffliner::DocumentAvailableInMainFrame() { | |
| 246 snapshot_controller_->DocumentAvailableInMainFrame(); | |
| 247 is_low_bar_met_ = true; | |
| 248 | |
| 249 // Add this signal to signal_data_. | |
| 250 AddLoadingSignal("DocumentAvailableInMainFrame"); | |
| 251 } | |
| 252 | |
| 253 void BackgroundLoaderOffliner::DocumentOnLoadCompletedInMainFrame() { | |
| 254 if (!pending_request_.get()) { | |
| 255 DVLOG(1) << "DidStopLoading called even though no pending request."; | |
| 256 return; | |
| 257 } | |
| 258 | |
| 259 // Add this signal to signal_data_. | |
| 260 AddLoadingSignal("DocumentOnLoadCompletedInMainFrame"); | |
| 261 | |
| 262 snapshot_controller_->DocumentOnLoadCompletedInMainFrame(); | |
| 263 } | |
| 264 | |
| 265 void BackgroundLoaderOffliner::RenderProcessGone( | |
| 266 base::TerminationStatus status) { | |
| 267 if (pending_request_) { | |
| 268 SavePageRequest request(*pending_request_.get()); | |
| 269 switch (status) { | |
| 270 case base::TERMINATION_STATUS_OOM: | |
| 271 case base::TERMINATION_STATUS_PROCESS_CRASHED: | |
| 272 case base::TERMINATION_STATUS_STILL_RUNNING: | |
| 273 completion_callback_.Run( | |
| 274 request, Offliner::RequestStatus::LOADING_FAILED_NO_NEXT); | |
| 275 break; | |
| 276 case base::TERMINATION_STATUS_PROCESS_WAS_KILLED: | |
| 277 default: | |
| 278 completion_callback_.Run(request, | |
| 279 Offliner::RequestStatus::LOADING_FAILED); | |
| 280 } | |
| 281 ResetState(); | |
| 282 } | |
| 283 } | |
| 284 | |
| 285 void BackgroundLoaderOffliner::WebContentsDestroyed() { | |
| 286 if (pending_request_) { | |
| 287 SavePageRequest request(*pending_request_.get()); | |
| 288 completion_callback_.Run(request, Offliner::RequestStatus::LOADING_FAILED); | |
| 289 ResetState(); | |
| 290 } | |
| 291 } | |
| 292 | |
| 293 void BackgroundLoaderOffliner::DidFinishNavigation( | |
| 294 content::NavigationHandle* navigation_handle) { | |
| 295 if (!navigation_handle->IsInMainFrame()) | |
| 296 return; | |
| 297 // If there was an error of any kind (certificate, client, DNS, etc), | |
| 298 // Mark as error page. Resetting here causes RecordNavigationMetrics to crash. | |
| 299 if (navigation_handle->IsErrorPage()) { | |
| 300 RecordErrorCauseUMA(pending_request_->client_id(), | |
| 301 navigation_handle->GetNetErrorCode()); | |
| 302 switch (navigation_handle->GetNetErrorCode()) { | |
| 303 case net::ERR_INTERNET_DISCONNECTED: | |
| 304 page_load_state_ = DELAY_RETRY; | |
| 305 break; | |
| 306 default: | |
| 307 page_load_state_ = RETRIABLE; | |
| 308 } | |
| 309 } | |
| 310 } | |
| 311 | |
| 312 void BackgroundLoaderOffliner::SetSnapshotControllerForTest( | |
| 313 std::unique_ptr<SnapshotController> controller) { | |
| 314 snapshot_controller_ = std::move(controller); | |
| 315 } | |
| 316 | |
| 317 void BackgroundLoaderOffliner::OnNetworkBytesChanged(int64_t bytes) { | |
| 318 if (pending_request_ && save_state_ != SAVING) { | |
| 319 network_bytes_ += bytes; | |
| 320 progress_callback_.Run(*pending_request_, network_bytes_); | |
| 321 } | |
| 322 } | |
| 323 | |
| 324 void BackgroundLoaderOffliner::StartSnapshot() { | |
| 325 if (!pending_request_.get()) { | |
| 326 DVLOG(1) << "Pending request was cleared during delay."; | |
| 327 return; | |
| 328 } | |
| 329 | |
| 330 // Add this signal to signal_data_. | |
| 331 AddLoadingSignal("Snapshotting"); | |
| 332 | |
| 333 SavePageRequest request(*pending_request_.get()); | |
| 334 // If there was an error navigating to page, return loading failed. | |
| 335 if (page_load_state_ != SUCCESS) { | |
| 336 Offliner::RequestStatus status; | |
| 337 switch (page_load_state_) { | |
| 338 case RETRIABLE: | |
| 339 status = Offliner::RequestStatus::LOADING_FAILED; | |
| 340 break; | |
| 341 case NONRETRIABLE: | |
| 342 status = Offliner::RequestStatus::LOADING_FAILED_NO_RETRY; | |
| 343 break; | |
| 344 case DELAY_RETRY: | |
| 345 status = Offliner::RequestStatus::LOADING_FAILED_NO_NEXT; | |
| 346 break; | |
| 347 default: | |
| 348 // We should've already checked for Success before entering here. | |
| 349 NOTREACHED(); | |
| 350 status = Offliner::RequestStatus::LOADING_FAILED; | |
| 351 } | |
| 352 | |
| 353 completion_callback_.Run(request, status); | |
| 354 ResetState(); | |
| 355 return; | |
| 356 } | |
| 357 | |
| 358 save_state_ = SAVING; | |
| 359 content::WebContents* web_contents( | |
| 360 content::WebContentsObserver::web_contents()); | |
| 361 | |
| 362 // Add loading signal into the MHTML that will be generated if the command | |
| 363 // line flag is set for it. | |
| 364 if (IsOfflinePagesLoadSignalCollectingEnabled()) { | |
| 365 // Stash loading signals for writing when we write out the MHTML. | |
| 366 std::string headers = base::StringPrintf( | |
| 367 "%s\r\n%s\r\n\r\n", kContentTransferEncodingBinary, kXHeaderForSignals); | |
| 368 std::string body; | |
| 369 base::JSONWriter::Write(signal_data_, &body); | |
| 370 std::string content_type = kContentType; | |
| 371 std::string content_location = base::StringPrintf( | |
| 372 "cid:signal-data-%" PRId64 "@mhtml.blink", request.request_id()); | |
| 373 | |
| 374 content::MHTMLExtraParts* extra_parts = | |
| 375 content::MHTMLExtraParts::FromWebContents(web_contents); | |
| 376 DCHECK(extra_parts); | |
| 377 if (extra_parts != nullptr) { | |
| 378 extra_parts->AddExtraMHTMLPart(content_type, content_location, headers, | |
| 379 body); | |
| 380 } | |
| 381 } | |
| 382 | |
| 383 std::unique_ptr<OfflinePageArchiver> archiver( | |
| 384 new OfflinePageMHTMLArchiver(web_contents)); | |
| 385 | |
| 386 OfflinePageModel::SavePageParams params; | |
| 387 params.url = web_contents->GetLastCommittedURL(); | |
| 388 params.client_id = request.client_id(); | |
| 389 params.proposed_offline_id = request.request_id(); | |
| 390 params.is_background = true; | |
| 391 | |
| 392 // Pass in the original URL if it's different from last committed | |
| 393 // when redirects occur. | |
| 394 if (!request.original_url().is_empty()) | |
| 395 params.original_url = request.original_url(); | |
| 396 else if (params.url != request.url()) | |
| 397 params.original_url = request.url(); | |
| 398 | |
| 399 offline_page_model_->SavePage( | |
| 400 params, std::move(archiver), | |
| 401 base::Bind(&BackgroundLoaderOffliner::OnPageSaved, | |
| 402 weak_ptr_factory_.GetWeakPtr())); | |
| 403 } | |
| 404 | |
| 405 void BackgroundLoaderOffliner::OnPageSaved(SavePageResult save_result, | |
| 406 int64_t offline_id) { | |
| 407 if (!pending_request_) | |
| 408 return; | |
| 409 | |
| 410 SavePageRequest request(*pending_request_.get()); | |
| 411 bool did_snapshot_on_last_retry = did_snapshot_on_last_retry_; | |
| 412 ResetState(); | |
| 413 | |
| 414 if (save_state_ == DELETE_AFTER_SAVE) { | |
| 415 // Delete the saved page off disk and from the OPM. | |
| 416 std::vector<int64_t> offline_ids; | |
| 417 offline_ids.push_back(offline_id); | |
| 418 offline_page_model_->DeletePagesByOfflineId( | |
| 419 offline_ids, | |
| 420 base::Bind(&BackgroundLoaderOffliner::DeleteOfflinePageCallback, | |
| 421 weak_ptr_factory_.GetWeakPtr(), request)); | |
| 422 save_state_ = NONE; | |
| 423 return; | |
| 424 } | |
| 425 | |
| 426 save_state_ = NONE; | |
| 427 | |
| 428 Offliner::RequestStatus save_status; | |
| 429 if (save_result == SavePageResult::ALREADY_EXISTS) { | |
| 430 save_status = RequestStatus::SAVED; | |
| 431 } else if (save_result == SavePageResult::SUCCESS) { | |
| 432 if (did_snapshot_on_last_retry) | |
| 433 save_status = RequestStatus::SAVED_ON_LAST_RETRY; | |
| 434 else | |
| 435 save_status = RequestStatus::SAVED; | |
| 436 } else { | |
| 437 save_status = RequestStatus::SAVE_FAILED; | |
| 438 } | |
| 439 | |
| 440 completion_callback_.Run(request, save_status); | |
| 441 } | |
| 442 | |
| 443 void BackgroundLoaderOffliner::DeleteOfflinePageCallback( | |
| 444 const SavePageRequest& request, | |
| 445 DeletePageResult result) { | |
| 446 cancel_callback_.Run(request); | |
| 447 } | |
| 448 | |
| 449 void BackgroundLoaderOffliner::ResetState() { | |
| 450 pending_request_.reset(); | |
| 451 snapshot_controller_.reset(); | |
| 452 page_load_state_ = SUCCESS; | |
| 453 network_bytes_ = 0LL; | |
| 454 is_low_bar_met_ = false; | |
| 455 did_snapshot_on_last_retry_ = false; | |
| 456 content::WebContentsObserver::Observe(nullptr); | |
| 457 loader_.reset(); | |
| 458 } | |
| 459 | |
| 460 void BackgroundLoaderOffliner::ResetLoader() { | |
| 461 loader_.reset( | |
| 462 new background_loader::BackgroundLoaderContents(browser_context_)); | |
| 463 } | |
| 464 | |
| 465 void BackgroundLoaderOffliner::AttachObservers() { | |
| 466 content::WebContents* contents = loader_->web_contents(); | |
| 467 content::WebContentsObserver::Observe(contents); | |
| 468 OfflinerData::AddToWebContents(contents, this); | |
| 469 } | |
| 470 | |
| 471 void BackgroundLoaderOffliner::OnApplicationStateChange( | |
| 472 base::android::ApplicationState application_state) { | |
| 473 if (pending_request_ && is_low_end_device_ && | |
| 474 application_state == | |
| 475 base::android::APPLICATION_STATE_HAS_RUNNING_ACTIVITIES) { | |
| 476 DVLOG(1) << "App became active, canceling current offlining request"; | |
| 477 // No need to check the return value or complete early, as false would | |
| 478 // indicate that there was no request, in which case the state change is | |
| 479 // ignored. | |
| 480 Cancel( | |
| 481 base::Bind(HandleApplicationStateChangeCancel, completion_callback_)); | |
| 482 } | |
| 483 } | |
| 484 | |
| 485 void BackgroundLoaderOffliner::AddLoadingSignal(const char* signal_name) { | |
| 486 base::TimeTicks current_time = base::TimeTicks::Now(); | |
| 487 base::TimeDelta delay_so_far = current_time - load_start_time_; | |
| 488 // We would prefer to use int64_t here, but JSON does not support that type. | |
| 489 // Given the choice between int and double, we choose to implicitly convert to | |
| 490 // a double since it maintains more precision (we can get a longer time in | |
| 491 // milliseconds than we can with a 2 bit int, 53 bits vs 32). | |
| 492 double delay = delay_so_far.InMilliseconds(); | |
| 493 signal_data_.SetDouble(signal_name, delay); | |
| 494 } | |
| 495 | |
| 496 } // namespace offline_pages | |
| 497 | |
| 498 DEFINE_WEB_CONTENTS_USER_DATA_KEY(offline_pages::OfflinerData); | |
| OLD | NEW |