OLD | NEW |
1 // Copyright 2016 The Chromium Authors. All rights reserved. | 1 // Copyright 2016 The Chromium Authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
4 | 4 |
5 #include "components/ntp_snippets/ntp_snippets_fetcher.h" | 5 #include "components/ntp_snippets/ntp_snippets_fetcher.h" |
6 | 6 |
7 #include <stdlib.h> | 7 #include <stdlib.h> |
8 | 8 |
9 #include "base/command_line.h" | 9 #include "base/command_line.h" |
10 #include "base/files/file_path.h" | 10 #include "base/files/file_path.h" |
(...skipping 90 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
101 return endpoint.empty() ? kChromeReaderServer : endpoint; | 101 return endpoint.empty() ? kChromeReaderServer : endpoint; |
102 } | 102 } |
103 | 103 |
104 bool UsesChromeContentSuggestionsAPI(const GURL& endpoint) { | 104 bool UsesChromeContentSuggestionsAPI(const GURL& endpoint) { |
105 if (endpoint == GURL(kChromeReaderServer)) { | 105 if (endpoint == GURL(kChromeReaderServer)) { |
106 return false; | 106 return false; |
107 } else if (endpoint != GURL(kContentSuggestionsServer) && | 107 } else if (endpoint != GURL(kContentSuggestionsServer) && |
108 endpoint != GURL(kContentSuggestionsDevServer) && | 108 endpoint != GURL(kContentSuggestionsDevServer) && |
109 endpoint != GURL(kContentSuggestionsAlphaServer)) { | 109 endpoint != GURL(kContentSuggestionsAlphaServer)) { |
110 LOG(WARNING) << "Unknown value for " << kContentSuggestionsBackend << ": " | 110 LOG(WARNING) << "Unknown value for " << kContentSuggestionsBackend << ": " |
111 << "assuming chromecontentsuggestions-style API"; | 111 << "assuming chromecontentsuggestions-style API"; |
112 } | 112 } |
113 return true; | 113 return true; |
114 } | 114 } |
115 | 115 |
116 // Creates snippets from dictionary values in |list| and adds them to | 116 // Creates snippets from dictionary values in |list| and adds them to |
117 // |snippets|. Returns true on success, false if anything went wrong. | 117 // |snippets|. Returns true on success, false if anything went wrong. |
118 bool AddSnippetsFromListValue(bool content_suggestions_api, | 118 bool AddSnippetsFromListValue(bool content_suggestions_api, |
119 const base::ListValue& list, | 119 const base::ListValue& list, |
120 NTPSnippet::PtrVector* snippets) { | 120 NTPSnippet::PtrVector* snippets) { |
121 for (const auto& value : list) { | 121 for (const auto& value : list) { |
(...skipping 42 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
164 token_service_(token_service), | 164 token_service_(token_service), |
165 waiting_for_refresh_token_(false), | 165 waiting_for_refresh_token_(false), |
166 url_request_context_getter_(url_request_context_getter), | 166 url_request_context_getter_(url_request_context_getter), |
167 category_factory_(category_factory), | 167 category_factory_(category_factory), |
168 parse_json_callback_(parse_json_callback), | 168 parse_json_callback_(parse_json_callback), |
169 fetch_url_(GetFetchEndpoint()), | 169 fetch_url_(GetFetchEndpoint()), |
170 fetch_api_(UsesChromeContentSuggestionsAPI(fetch_url_) | 170 fetch_api_(UsesChromeContentSuggestionsAPI(fetch_url_) |
171 ? CHROME_CONTENT_SUGGESTIONS_API | 171 ? CHROME_CONTENT_SUGGESTIONS_API |
172 : CHROME_READER_API), | 172 : CHROME_READER_API), |
173 is_stable_channel_(is_stable_channel), | 173 is_stable_channel_(is_stable_channel), |
| 174 interactive_request_(false), |
174 tick_clock_(new base::DefaultTickClock()), | 175 tick_clock_(new base::DefaultTickClock()), |
175 request_throttler_( | 176 request_throttler_( |
176 pref_service, | 177 pref_service, |
177 RequestThrottler::RequestType::CONTENT_SUGGESTION_FETCHER), | 178 RequestThrottler::RequestType::CONTENT_SUGGESTION_FETCHER), |
178 oauth_token_retried_(false), | 179 oauth_token_retried_(false), |
179 weak_ptr_factory_(this) { | 180 weak_ptr_factory_(this) { |
180 // Parse the variation parameters and set the defaults if missing. | 181 // Parse the variation parameters and set the defaults if missing. |
181 std::string personalization = variations::GetVariationParamValue( | 182 std::string personalization = variations::GetVariationParamValue( |
182 ntp_snippets::kStudyName, kPersonalizationName); | 183 ntp_snippets::kStudyName, kPersonalizationName); |
183 if (personalization == kPersonalizationNonPersonalString) { | 184 if (personalization == kPersonalizationNonPersonalString) { |
(...skipping 47 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
231 if (UsesHostRestrictions() && hosts_.empty()) { | 232 if (UsesHostRestrictions() && hosts_.empty()) { |
232 FetchFinished(OptionalSnippets(), FetchResult::EMPTY_HOSTS, | 233 FetchFinished(OptionalSnippets(), FetchResult::EMPTY_HOSTS, |
233 /*extra_message=*/std::string()); | 234 /*extra_message=*/std::string()); |
234 return; | 235 return; |
235 } | 236 } |
236 | 237 |
237 locale_ = PosixLocaleFromBCP47Language(language_code); | 238 locale_ = PosixLocaleFromBCP47Language(language_code); |
238 count_to_fetch_ = count; | 239 count_to_fetch_ = count; |
239 | 240 |
240 bool use_authentication = UsesAuthentication(); | 241 bool use_authentication = UsesAuthentication(); |
| 242 interactive_request_ = interactive_request; |
241 | 243 |
242 if (use_authentication && signin_manager_->IsAuthenticated()) { | 244 if (use_authentication && signin_manager_->IsAuthenticated()) { |
243 // Signed-in: get OAuth token --> fetch snippets. | 245 // Signed-in: get OAuth token --> fetch snippets. |
244 oauth_token_retried_ = false; | 246 oauth_token_retried_ = false; |
245 StartTokenRequest(); | 247 StartTokenRequest(); |
246 } else if (use_authentication && signin_manager_->AuthInProgress()) { | 248 } else if (use_authentication && signin_manager_->AuthInProgress()) { |
247 // Currently signing in: wait for auth to finish (the refresh token) --> | 249 // Currently signing in: wait for auth to finish (the refresh token) --> |
248 // get OAuth token --> fetch snippets. | 250 // get OAuth token --> fetch snippets. |
249 if (!waiting_for_refresh_token_) { | 251 if (!waiting_for_refresh_token_) { |
250 // Wait until we get a refresh token. | 252 // Wait until we get a refresh token. |
251 waiting_for_refresh_token_ = true; | 253 waiting_for_refresh_token_ = true; |
252 token_service_->AddObserver(this); | 254 token_service_->AddObserver(this); |
253 } | 255 } |
254 } else { | 256 } else { |
255 // Not signed in: fetch snippets (without authentication). | 257 // Not signed in: fetch snippets (without authentication). |
256 FetchSnippetsNonAuthenticated(); | 258 FetchSnippetsNonAuthenticated(); |
257 } | 259 } |
258 } | 260 } |
259 | 261 |
260 NTPSnippetsFetcher::RequestParams::RequestParams() | 262 NTPSnippetsFetcher::RequestParams::RequestParams() |
261 : fetch_api(), | 263 : fetch_api(), |
262 obfuscated_gaia_id(), | 264 obfuscated_gaia_id(), |
263 only_return_personalized_results(), | 265 only_return_personalized_results(), |
264 user_locale(), | 266 user_locale(), |
265 host_restricts(), | 267 host_restricts(), |
266 count_to_fetch() {} | 268 count_to_fetch(), |
| 269 interactive_request() {} |
267 | 270 |
268 NTPSnippetsFetcher::RequestParams::~RequestParams() = default; | 271 NTPSnippetsFetcher::RequestParams::~RequestParams() = default; |
269 | 272 |
270 std::string NTPSnippetsFetcher::RequestParams::BuildRequest() { | 273 std::string NTPSnippetsFetcher::RequestParams::BuildRequest() { |
271 auto request = base::MakeUnique<base::DictionaryValue>(); | 274 auto request = base::MakeUnique<base::DictionaryValue>(); |
272 switch (fetch_api) { | 275 switch (fetch_api) { |
273 case CHROME_READER_API: { | 276 case CHROME_READER_API: { |
274 auto content_params = base::MakeUnique<base::DictionaryValue>(); | 277 auto content_params = base::MakeUnique<base::DictionaryValue>(); |
275 content_params->SetBoolean("only_return_personalized_results", | 278 content_params->SetBoolean("only_return_personalized_results", |
276 only_return_personalized_results); | 279 only_return_personalized_results); |
(...skipping 43 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
320 case CHROME_CONTENT_SUGGESTIONS_API: { | 323 case CHROME_CONTENT_SUGGESTIONS_API: { |
321 if (!user_locale.empty()) { | 324 if (!user_locale.empty()) { |
322 request->SetString("uiLanguage", user_locale); | 325 request->SetString("uiLanguage", user_locale); |
323 } | 326 } |
324 | 327 |
325 auto regular_hosts = base::MakeUnique<base::ListValue>(); | 328 auto regular_hosts = base::MakeUnique<base::ListValue>(); |
326 for (const auto& host : host_restricts) { | 329 for (const auto& host : host_restricts) { |
327 regular_hosts->AppendString(host); | 330 regular_hosts->AppendString(host); |
328 } | 331 } |
329 request->Set("regularlyVisitedHostNames", std::move(regular_hosts)); | 332 request->Set("regularlyVisitedHostNames", std::move(regular_hosts)); |
| 333 request->SetString("priority", interactive_request |
| 334 ? "USER_ACTION" |
| 335 : "BACKGROUND_PREFETCH"); |
330 | 336 |
331 auto excluded = base::MakeUnique<base::ListValue>(); | 337 auto excluded = base::MakeUnique<base::ListValue>(); |
332 for (const auto& id : excluded_ids) { | 338 for (const auto& id : excluded_ids) { |
333 excluded->AppendString(id); | 339 excluded->AppendString(id); |
334 if (excluded->GetSize() >= kMaxExcludedIds) | 340 if (excluded->GetSize() >= kMaxExcludedIds) |
335 break; | 341 break; |
336 } | 342 } |
337 request->Set("excludedSuggestionIds", std::move(excluded)); | 343 request->Set("excludedSuggestionIds", std::move(excluded)); |
338 | 344 |
339 // TODO(sfiera): support authentication and personalization | 345 // TODO(sfiera): support authentication and personalization |
(...skipping 27 matching lines...) Expand all Loading... |
367 headers.SetHeader("Content-Type", "application/json; charset=UTF-8"); | 373 headers.SetHeader("Content-Type", "application/json; charset=UTF-8"); |
368 // Add X-Client-Data header with experiment IDs from field trials. | 374 // Add X-Client-Data header with experiment IDs from field trials. |
369 variations::AppendVariationHeaders(url, | 375 variations::AppendVariationHeaders(url, |
370 false, // incognito | 376 false, // incognito |
371 false, // uma_enabled | 377 false, // uma_enabled |
372 &headers); | 378 &headers); |
373 url_fetcher_->SetExtraRequestHeaders(headers.ToString()); | 379 url_fetcher_->SetExtraRequestHeaders(headers.ToString()); |
374 url_fetcher_->SetUploadData("application/json", request); | 380 url_fetcher_->SetUploadData("application/json", request); |
375 // Log the request for debugging network issues. | 381 // Log the request for debugging network issues. |
376 VLOG(1) << "Sending a NTP snippets request to " << url << ":" << std::endl | 382 VLOG(1) << "Sending a NTP snippets request to " << url << ":" << std::endl |
377 << headers.ToString() << std::endl << request; | 383 << headers.ToString() << std::endl |
| 384 << request; |
378 // Fetchers are sometimes cancelled because a network change was detected. | 385 // Fetchers are sometimes cancelled because a network change was detected. |
379 url_fetcher_->SetAutomaticallyRetryOnNetworkChanges(3); | 386 url_fetcher_->SetAutomaticallyRetryOnNetworkChanges(3); |
380 // Try to make fetching the files bit more robust even with poor connection. | 387 // Try to make fetching the files bit more robust even with poor connection. |
381 url_fetcher_->SetMaxRetriesOn5xx(3); | 388 url_fetcher_->SetMaxRetriesOn5xx(3); |
382 url_fetcher_->Start(); | 389 url_fetcher_->Start(); |
383 } | 390 } |
384 | 391 |
385 bool NTPSnippetsFetcher::UsesHostRestrictions() const { | 392 bool NTPSnippetsFetcher::UsesHostRestrictions() const { |
386 return use_host_restriction_ && | 393 return use_host_restriction_ && |
387 !base::CommandLine::ForCurrentProcess()->HasSwitch( | 394 !base::CommandLine::ForCurrentProcess()->HasSwitch( |
(...skipping 12 matching lines...) Expand all Loading... |
400 : google_apis::GetNonStableAPIKey(); | 407 : google_apis::GetNonStableAPIKey(); |
401 GURL url(base::StringPrintf(kSnippetsServerNonAuthorizedFormat, | 408 GURL url(base::StringPrintf(kSnippetsServerNonAuthorizedFormat, |
402 fetch_url_.spec().c_str(), key.c_str())); | 409 fetch_url_.spec().c_str(), key.c_str())); |
403 | 410 |
404 RequestParams params; | 411 RequestParams params; |
405 params.fetch_api = fetch_api_; | 412 params.fetch_api = fetch_api_; |
406 params.host_restricts = | 413 params.host_restricts = |
407 UsesHostRestrictions() ? hosts_ : std::set<std::string>(); | 414 UsesHostRestrictions() ? hosts_ : std::set<std::string>(); |
408 params.excluded_ids = excluded_ids_; | 415 params.excluded_ids = excluded_ids_; |
409 params.count_to_fetch = count_to_fetch_; | 416 params.count_to_fetch = count_to_fetch_; |
| 417 params.interactive_request = interactive_request_; |
410 FetchSnippetsImpl(url, std::string(), params.BuildRequest()); | 418 FetchSnippetsImpl(url, std::string(), params.BuildRequest()); |
411 } | 419 } |
412 | 420 |
413 void NTPSnippetsFetcher::FetchSnippetsAuthenticated( | 421 void NTPSnippetsFetcher::FetchSnippetsAuthenticated( |
414 const std::string& account_id, | 422 const std::string& account_id, |
415 const std::string& oauth_access_token) { | 423 const std::string& oauth_access_token) { |
416 RequestParams params; | 424 RequestParams params; |
417 params.fetch_api = fetch_api_; | 425 params.fetch_api = fetch_api_; |
418 params.obfuscated_gaia_id = account_id; | 426 params.obfuscated_gaia_id = account_id; |
419 params.only_return_personalized_results = | 427 params.only_return_personalized_results = |
420 personalization_ == Personalization::kPersonal; | 428 personalization_ == Personalization::kPersonal; |
421 params.user_locale = locale_; | 429 params.user_locale = locale_; |
422 params.host_restricts = | 430 params.host_restricts = |
423 UsesHostRestrictions() ? hosts_ : std::set<std::string>(); | 431 UsesHostRestrictions() ? hosts_ : std::set<std::string>(); |
424 params.excluded_ids = excluded_ids_; | 432 params.excluded_ids = excluded_ids_; |
425 params.count_to_fetch = count_to_fetch_; | 433 params.count_to_fetch = count_to_fetch_; |
| 434 params.interactive_request = interactive_request_; |
426 // TODO(jkrcal, treib): Add unit-tests for authenticated fetches. | 435 // TODO(jkrcal, treib): Add unit-tests for authenticated fetches. |
427 FetchSnippetsImpl(fetch_url_, | 436 FetchSnippetsImpl(fetch_url_, |
428 base::StringPrintf(kAuthorizationRequestHeaderFormat, | 437 base::StringPrintf(kAuthorizationRequestHeaderFormat, |
429 oauth_access_token.c_str()), | 438 oauth_access_token.c_str()), |
430 params.BuildRequest()); | 439 params.BuildRequest()); |
431 } | 440 } |
432 | 441 |
433 void NTPSnippetsFetcher::StartTokenRequest() { | 442 void NTPSnippetsFetcher::StartTokenRequest() { |
434 OAuth2TokenService::ScopeSet scopes; | 443 OAuth2TokenService::ScopeSet scopes; |
435 scopes.insert(fetch_api_ == CHROME_CONTENT_SUGGESTIONS_API | 444 scopes.insert(fetch_api_ == CHROME_CONTENT_SUGGESTIONS_API |
(...skipping 69 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
505 } else if (source->GetResponseCode() != net::HTTP_OK) { | 514 } else if (source->GetResponseCode() != net::HTTP_OK) { |
506 // TODO(jkrcal): https://crbug.com/609084 | 515 // TODO(jkrcal): https://crbug.com/609084 |
507 // We need to deal with the edge case again where the auth | 516 // We need to deal with the edge case again where the auth |
508 // token expires just before we send the request (in which case we need to | 517 // token expires just before we send the request (in which case we need to |
509 // fetch a new auth token). We should extract that into a common class | 518 // fetch a new auth token). We should extract that into a common class |
510 // instead of adding it to every single class that uses auth tokens. | 519 // instead of adding it to every single class that uses auth tokens. |
511 FetchFinished( | 520 FetchFinished( |
512 OptionalSnippets(), FetchResult::HTTP_ERROR, | 521 OptionalSnippets(), FetchResult::HTTP_ERROR, |
513 /*extra_message=*/base::StringPrintf(" %d", source->GetResponseCode())); | 522 /*extra_message=*/base::StringPrintf(" %d", source->GetResponseCode())); |
514 } else { | 523 } else { |
515 bool stores_result_to_string = source->GetResponseAsString( | 524 bool stores_result_to_string = |
516 &last_fetch_json_); | 525 source->GetResponseAsString(&last_fetch_json_); |
517 DCHECK(stores_result_to_string); | 526 DCHECK(stores_result_to_string); |
518 | 527 |
519 parse_json_callback_.Run( | 528 parse_json_callback_.Run(last_fetch_json_, |
520 last_fetch_json_, | 529 base::Bind(&NTPSnippetsFetcher::OnJsonParsed, |
521 base::Bind(&NTPSnippetsFetcher::OnJsonParsed, | 530 weak_ptr_factory_.GetWeakPtr()), |
522 weak_ptr_factory_.GetWeakPtr()), | 531 base::Bind(&NTPSnippetsFetcher::OnJsonError, |
523 base::Bind(&NTPSnippetsFetcher::OnJsonError, | 532 weak_ptr_factory_.GetWeakPtr())); |
524 weak_ptr_factory_.GetWeakPtr())); | |
525 } | 533 } |
526 } | 534 } |
527 | 535 |
528 bool NTPSnippetsFetcher::JsonToSnippets(const base::Value& parsed, | 536 bool NTPSnippetsFetcher::JsonToSnippets(const base::Value& parsed, |
529 NTPSnippet::CategoryMap* snippets) { | 537 NTPSnippet::CategoryMap* snippets) { |
530 const base::DictionaryValue* top_dict = nullptr; | 538 const base::DictionaryValue* top_dict = nullptr; |
531 if (!parsed.GetAsDictionary(&top_dict)) { | 539 if (!parsed.GetAsDictionary(&top_dict)) { |
532 return false; | 540 return false; |
533 } | 541 } |
534 | 542 |
(...skipping 49 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
584 /*extra_message=*/std::string()); | 592 /*extra_message=*/std::string()); |
585 } else { | 593 } else { |
586 LOG(WARNING) << "Received invalid snippets: " << last_fetch_json_; | 594 LOG(WARNING) << "Received invalid snippets: " << last_fetch_json_; |
587 FetchFinished(OptionalSnippets(), | 595 FetchFinished(OptionalSnippets(), |
588 FetchResult::INVALID_SNIPPET_CONTENT_ERROR, | 596 FetchResult::INVALID_SNIPPET_CONTENT_ERROR, |
589 /*extra_message=*/std::string()); | 597 /*extra_message=*/std::string()); |
590 } | 598 } |
591 } | 599 } |
592 | 600 |
593 void NTPSnippetsFetcher::OnJsonError(const std::string& error) { | 601 void NTPSnippetsFetcher::OnJsonError(const std::string& error) { |
594 LOG(WARNING) << "Received invalid JSON (" << error << "): " | 602 LOG(WARNING) << "Received invalid JSON (" << error |
595 << last_fetch_json_; | 603 << "): " << last_fetch_json_; |
596 FetchFinished( | 604 FetchFinished( |
597 OptionalSnippets(), FetchResult::JSON_PARSE_ERROR, | 605 OptionalSnippets(), FetchResult::JSON_PARSE_ERROR, |
598 /*extra_message=*/base::StringPrintf(" (error %s)", error.c_str())); | 606 /*extra_message=*/base::StringPrintf(" (error %s)", error.c_str())); |
599 } | 607 } |
600 | 608 |
601 void NTPSnippetsFetcher::FetchFinished(OptionalSnippets snippets, | 609 void NTPSnippetsFetcher::FetchFinished(OptionalSnippets snippets, |
602 FetchResult result, | 610 FetchResult result, |
603 const std::string& extra_message) { | 611 const std::string& extra_message) { |
604 DCHECK(result == FetchResult::SUCCESS || !snippets); | 612 DCHECK(result == FetchResult::SUCCESS || !snippets); |
605 last_status_ = FetchResultToString(result) + extra_message; | 613 last_status_ = FetchResultToString(result) + extra_message; |
606 | 614 |
607 // If the result is EMPTY_HOSTS or OAUTH_TOKEN_ERROR, we didn't actually send | 615 // If the result is EMPTY_HOSTS or OAUTH_TOKEN_ERROR, we didn't actually send |
608 // a network request, so don't record FetchTime in those cases. | 616 // a network request, so don't record FetchTime in those cases. |
609 if (result != FetchResult::EMPTY_HOSTS && | 617 if (result != FetchResult::EMPTY_HOSTS && |
610 result != FetchResult::OAUTH_TOKEN_ERROR) { | 618 result != FetchResult::OAUTH_TOKEN_ERROR) { |
611 UMA_HISTOGRAM_TIMES("NewTabPage.Snippets.FetchTime", | 619 UMA_HISTOGRAM_TIMES("NewTabPage.Snippets.FetchTime", |
612 tick_clock_->NowTicks() - fetch_start_time_); | 620 tick_clock_->NowTicks() - fetch_start_time_); |
613 } | 621 } |
614 UMA_HISTOGRAM_ENUMERATION("NewTabPage.Snippets.FetchResult", | 622 UMA_HISTOGRAM_ENUMERATION("NewTabPage.Snippets.FetchResult", |
615 static_cast<int>(result), | 623 static_cast<int>(result), |
616 static_cast<int>(FetchResult::RESULT_MAX)); | 624 static_cast<int>(FetchResult::RESULT_MAX)); |
617 | 625 |
618 DVLOG(1) << "Fetch finished: " << last_status_; | 626 DVLOG(1) << "Fetch finished: " << last_status_; |
619 if (!snippets_available_callback_.is_null()) | 627 if (!snippets_available_callback_.is_null()) |
620 snippets_available_callback_.Run(std::move(snippets)); | 628 snippets_available_callback_.Run(std::move(snippets)); |
621 } | 629 } |
622 | 630 |
623 } // namespace ntp_snippets | 631 } // namespace ntp_snippets |
OLD | NEW |