| OLD | NEW |
| 1 // Copyright 2016 The Chromium Authors. All rights reserved. | 1 // Copyright 2016 The Chromium Authors. All rights reserved. |
| 2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
| 3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
| 4 | 4 |
| 5 #include "components/ntp_snippets/remote/remote_suggestions_fetcher.h" | 5 #include "components/ntp_snippets/remote/remote_suggestions_fetcher.h" |
| 6 | 6 |
| 7 #include <cstdlib> | 7 #include <cstdlib> |
| 8 #include <utility> | 8 #include <utility> |
| 9 | 9 |
| 10 #include "base/files/file_path.h" | 10 #include "base/files/file_path.h" |
| (...skipping 22 matching lines...) Expand all Loading... |
| 33 | 33 |
| 34 using net::URLFetcher; | 34 using net::URLFetcher; |
| 35 using net::URLRequestContextGetter; | 35 using net::URLRequestContextGetter; |
| 36 using net::HttpRequestHeaders; | 36 using net::HttpRequestHeaders; |
| 37 using net::URLRequestStatus; | 37 using net::URLRequestStatus; |
| 38 using translate::LanguageModel; | 38 using translate::LanguageModel; |
| 39 | 39 |
| 40 namespace ntp_snippets { | 40 namespace ntp_snippets { |
| 41 | 41 |
| 42 using internal::JsonRequest; | 42 using internal::JsonRequest; |
| 43 using internal::FetchAPI; | |
| 44 using internal::FetchResult; | 43 using internal::FetchResult; |
| 45 | 44 |
| 46 namespace { | 45 namespace { |
| 47 | 46 |
| 48 const char kChromeReaderApiScope[] = | |
| 49 "https://www.googleapis.com/auth/webhistory"; | |
| 50 const char kContentSuggestionsApiScope[] = | 47 const char kContentSuggestionsApiScope[] = |
| 51 "https://www.googleapis.com/auth/chrome-content-suggestions"; | 48 "https://www.googleapis.com/auth/chrome-content-suggestions"; |
| 52 const char kSnippetsServerNonAuthorizedFormat[] = "%s?key=%s"; | 49 const char kSnippetsServerNonAuthorizedFormat[] = "%s?key=%s"; |
| 53 const char kAuthorizationRequestHeaderFormat[] = "Bearer %s"; | 50 const char kAuthorizationRequestHeaderFormat[] = "Bearer %s"; |
| 54 | 51 |
| 55 // Variation parameter for chrome-content-suggestions backend. | 52 // Variation parameter for chrome-content-suggestions backend. |
| 56 const char kContentSuggestionsBackend[] = "content_suggestions_backend"; | 53 const char kContentSuggestionsBackend[] = "content_suggestions_backend"; |
| 57 | 54 |
| 58 const int kFetchTimeHistogramResolution = 5; | 55 const int kFetchTimeHistogramResolution = 5; |
| 59 | 56 |
| (...skipping 37 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 97 case FetchResult::INVALID_SNIPPET_CONTENT_ERROR: | 94 case FetchResult::INVALID_SNIPPET_CONTENT_ERROR: |
| 98 case FetchResult::JSON_PARSE_ERROR: | 95 case FetchResult::JSON_PARSE_ERROR: |
| 99 return Status(StatusCode::TEMPORARY_ERROR, FetchResultToString(result)); | 96 return Status(StatusCode::TEMPORARY_ERROR, FetchResultToString(result)); |
| 100 case FetchResult::RESULT_MAX: | 97 case FetchResult::RESULT_MAX: |
| 101 break; | 98 break; |
| 102 } | 99 } |
| 103 NOTREACHED(); | 100 NOTREACHED(); |
| 104 return Status(StatusCode::PERMANENT_ERROR, std::string()); | 101 return Status(StatusCode::PERMANENT_ERROR, std::string()); |
| 105 } | 102 } |
| 106 | 103 |
| 107 bool UsesChromeContentSuggestionsAPI(const GURL& endpoint) { | |
| 108 if (endpoint == kChromeReaderServer) { | |
| 109 return false; | |
| 110 } | |
| 111 | |
| 112 if (endpoint != kContentSuggestionsServer && | |
| 113 endpoint != kContentSuggestionsStagingServer && | |
| 114 endpoint != kContentSuggestionsAlphaServer) { | |
| 115 LOG(WARNING) << "Unknown value for " << kContentSuggestionsBackend << ": " | |
| 116 << endpoint << "; assuming chromecontentsuggestions-style API"; | |
| 117 } | |
| 118 return true; | |
| 119 } | |
| 120 | |
| 121 // Creates suggestions from dictionary values in |list| and adds them to | 104 // Creates suggestions from dictionary values in |list| and adds them to |
| 122 // |suggestions|. Returns true on success, false if anything went wrong. | 105 // |suggestions|. Returns true on success, false if anything went wrong. |
| 123 // |remote_category_id| is only used if |content_suggestions_api| is true. | 106 // |remote_category_id| is only used if |content_suggestions_api| is true. |
| 124 bool AddSuggestionsFromListValue(bool content_suggestions_api, | 107 bool AddSuggestionsFromListValue(bool content_suggestions_api, |
| 125 int remote_category_id, | 108 int remote_category_id, |
| 126 const base::ListValue& list, | 109 const base::ListValue& list, |
| 127 RemoteSuggestion::PtrVector* suggestions, | 110 RemoteSuggestion::PtrVector* suggestions, |
| 128 const base::Time& fetch_time) { | 111 const base::Time& fetch_time) { |
| 129 for (const auto& value : list) { | 112 for (const auto& value : list) { |
| 130 const base::DictionaryValue* dict = nullptr; | 113 const base::DictionaryValue* dict = nullptr; |
| (...skipping 129 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 260 const GURL& api_endpoint, | 243 const GURL& api_endpoint, |
| 261 const std::string& api_key, | 244 const std::string& api_key, |
| 262 const UserClassifier* user_classifier) | 245 const UserClassifier* user_classifier) |
| 263 : OAuth2TokenService::Consumer("ntp_snippets"), | 246 : OAuth2TokenService::Consumer("ntp_snippets"), |
| 264 signin_manager_(signin_manager), | 247 signin_manager_(signin_manager), |
| 265 token_service_(token_service), | 248 token_service_(token_service), |
| 266 url_request_context_getter_(std::move(url_request_context_getter)), | 249 url_request_context_getter_(std::move(url_request_context_getter)), |
| 267 language_model_(language_model), | 250 language_model_(language_model), |
| 268 parse_json_callback_(parse_json_callback), | 251 parse_json_callback_(parse_json_callback), |
| 269 fetch_url_(api_endpoint), | 252 fetch_url_(api_endpoint), |
| 270 fetch_api_(UsesChromeContentSuggestionsAPI(fetch_url_) | |
| 271 ? FetchAPI::CHROME_CONTENT_SUGGESTIONS_API | |
| 272 : FetchAPI::CHROME_READER_API), | |
| 273 api_key_(api_key), | 253 api_key_(api_key), |
| 274 clock_(new base::DefaultClock()), | 254 clock_(new base::DefaultClock()), |
| 275 user_classifier_(user_classifier) {} | 255 user_classifier_(user_classifier) {} |
| 276 | 256 |
| 277 RemoteSuggestionsFetcher::~RemoteSuggestionsFetcher() { | 257 RemoteSuggestionsFetcher::~RemoteSuggestionsFetcher() { |
| 278 if (waiting_for_refresh_token_) { | 258 if (waiting_for_refresh_token_) { |
| 279 token_service_->RemoveObserver(this); | 259 token_service_->RemoveObserver(this); |
| 280 } | 260 } |
| 281 } | 261 } |
| 282 | 262 |
| 283 void RemoteSuggestionsFetcher::FetchSnippets( | 263 void RemoteSuggestionsFetcher::FetchSnippets( |
| 284 const RequestParams& params, | 264 const RequestParams& params, |
| 285 SnippetsAvailableCallback callback) { | 265 SnippetsAvailableCallback callback) { |
| 286 if (!params.interactive_request) { | 266 if (!params.interactive_request) { |
| 287 UMA_HISTOGRAM_SPARSE_SLOWLY( | 267 UMA_HISTOGRAM_SPARSE_SLOWLY( |
| 288 "NewTabPage.Snippets.FetchTimeLocal", | 268 "NewTabPage.Snippets.FetchTimeLocal", |
| 289 GetMinuteOfTheDay(/*local_time=*/true, | 269 GetMinuteOfTheDay(/*local_time=*/true, |
| 290 /*reduced_resolution=*/true, clock_.get())); | 270 /*reduced_resolution=*/true, clock_.get())); |
| 291 UMA_HISTOGRAM_SPARSE_SLOWLY( | 271 UMA_HISTOGRAM_SPARSE_SLOWLY( |
| 292 "NewTabPage.Snippets.FetchTimeUTC", | 272 "NewTabPage.Snippets.FetchTimeUTC", |
| 293 GetMinuteOfTheDay(/*local_time=*/false, | 273 GetMinuteOfTheDay(/*local_time=*/false, |
| 294 /*reduced_resolution=*/true, clock_.get())); | 274 /*reduced_resolution=*/true, clock_.get())); |
| 295 } | 275 } |
| 296 | 276 |
| 297 JsonRequest::Builder builder; | 277 JsonRequest::Builder builder; |
| 298 builder.SetFetchAPI(fetch_api_) | 278 builder.SetLanguageModel(language_model_) |
| 299 .SetFetchAPI(fetch_api_) | |
| 300 .SetLanguageModel(language_model_) | |
| 301 .SetParams(params) | 279 .SetParams(params) |
| 302 .SetParseJsonCallback(parse_json_callback_) | 280 .SetParseJsonCallback(parse_json_callback_) |
| 303 .SetClock(clock_.get()) | 281 .SetClock(clock_.get()) |
| 304 .SetUrlRequestContextGetter(url_request_context_getter_) | 282 .SetUrlRequestContextGetter(url_request_context_getter_) |
| 305 .SetUserClassifier(*user_classifier_); | 283 .SetUserClassifier(*user_classifier_); |
| 306 | 284 |
| 307 if (signin_manager_->IsAuthenticated()) { | 285 if (signin_manager_->IsAuthenticated()) { |
| 308 // Signed-in: get OAuth token --> fetch suggestions. | 286 // Signed-in: get OAuth token --> fetch suggestions. |
| 309 oauth_token_retried_ = false; | 287 oauth_token_retried_ = false; |
| 310 pending_requests_.emplace(std::move(builder), std::move(callback)); | 288 pending_requests_.emplace(std::move(builder), std::move(callback)); |
| (...skipping 47 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 358 SnippetsAvailableCallback callback) { | 336 SnippetsAvailableCallback callback) { |
| 359 std::unique_ptr<JsonRequest> request = builder.Build(); | 337 std::unique_ptr<JsonRequest> request = builder.Build(); |
| 360 JsonRequest* raw_request = request.get(); | 338 JsonRequest* raw_request = request.get(); |
| 361 raw_request->Start(base::BindOnce(&RemoteSuggestionsFetcher::JsonRequestDone, | 339 raw_request->Start(base::BindOnce(&RemoteSuggestionsFetcher::JsonRequestDone, |
| 362 base::Unretained(this), std::move(request), | 340 base::Unretained(this), std::move(request), |
| 363 std::move(callback))); | 341 std::move(callback))); |
| 364 } | 342 } |
| 365 | 343 |
| 366 void RemoteSuggestionsFetcher::StartTokenRequest() { | 344 void RemoteSuggestionsFetcher::StartTokenRequest() { |
| 367 OAuth2TokenService::ScopeSet scopes; | 345 OAuth2TokenService::ScopeSet scopes; |
| 368 scopes.insert(fetch_api_ == FetchAPI::CHROME_CONTENT_SUGGESTIONS_API | 346 scopes.insert(kContentSuggestionsApiScope); |
| 369 ? kContentSuggestionsApiScope | |
| 370 : kChromeReaderApiScope); | |
| 371 oauth_request_ = token_service_->StartRequest( | 347 oauth_request_ = token_service_->StartRequest( |
| 372 signin_manager_->GetAuthenticatedAccountId(), scopes, this); | 348 signin_manager_->GetAuthenticatedAccountId(), scopes, this); |
| 373 } | 349 } |
| 374 | 350 |
| 375 //////////////////////////////////////////////////////////////////////////////// | 351 //////////////////////////////////////////////////////////////////////////////// |
| 376 // OAuth2TokenService::Consumer overrides | 352 // OAuth2TokenService::Consumer overrides |
| 377 void RemoteSuggestionsFetcher::OnGetTokenSuccess( | 353 void RemoteSuggestionsFetcher::OnGetTokenSuccess( |
| 378 const OAuth2TokenService::Request* request, | 354 const OAuth2TokenService::Request* request, |
| 379 const std::string& access_token, | 355 const std::string& access_token, |
| 380 const base::Time& expiration_time) { | 356 const base::Time& expiration_time) { |
| (...skipping 69 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 450 last_fetch_json_ = request->GetResponseString(); | 426 last_fetch_json_ = request->GetResponseString(); |
| 451 | 427 |
| 452 UMA_HISTOGRAM_TIMES("NewTabPage.Snippets.FetchTime", | 428 UMA_HISTOGRAM_TIMES("NewTabPage.Snippets.FetchTime", |
| 453 request->GetFetchDuration()); | 429 request->GetFetchDuration()); |
| 454 | 430 |
| 455 if (!result) { | 431 if (!result) { |
| 456 FetchFinished(OptionalFetchedCategories(), std::move(callback), status_code, | 432 FetchFinished(OptionalFetchedCategories(), std::move(callback), status_code, |
| 457 error_details); | 433 error_details); |
| 458 return; | 434 return; |
| 459 } | 435 } |
| 436 |
| 460 FetchedCategoriesVector categories; | 437 FetchedCategoriesVector categories; |
| 461 | |
| 462 if (!JsonToSnippets(*result, &categories, fetch_time)) { | 438 if (!JsonToSnippets(*result, &categories, fetch_time)) { |
| 463 LOG(WARNING) << "Received invalid snippets: " << last_fetch_json_; | 439 LOG(WARNING) << "Received invalid snippets: " << last_fetch_json_; |
| 464 FetchFinished(OptionalFetchedCategories(), std::move(callback), | 440 FetchFinished(OptionalFetchedCategories(), std::move(callback), |
| 465 FetchResult::INVALID_SNIPPET_CONTENT_ERROR, std::string()); | 441 FetchResult::INVALID_SNIPPET_CONTENT_ERROR, std::string()); |
| 466 return; | 442 return; |
| 467 } | 443 } |
| 468 // Filter out unwanted categories if necessary. | 444 // Filter out unwanted categories if necessary. |
| 469 // TODO(fhorschig): As soon as the server supports filtering by category, | 445 // TODO(fhorschig): As soon as the server supports filtering by category, |
| 470 // adjust the request instead of over-fetching and filtering here. | 446 // adjust the request instead of over-fetching and filtering here. |
| 471 FilterCategories(&categories, request->exclusive_category()); | 447 FilterCategories(&categories, request->exclusive_category()); |
| (...skipping 23 matching lines...) Expand all Loading... |
| 495 | 471 |
| 496 bool RemoteSuggestionsFetcher::JsonToSnippets( | 472 bool RemoteSuggestionsFetcher::JsonToSnippets( |
| 497 const base::Value& parsed, | 473 const base::Value& parsed, |
| 498 FetchedCategoriesVector* categories, | 474 FetchedCategoriesVector* categories, |
| 499 const base::Time& fetch_time) { | 475 const base::Time& fetch_time) { |
| 500 const base::DictionaryValue* top_dict = nullptr; | 476 const base::DictionaryValue* top_dict = nullptr; |
| 501 if (!parsed.GetAsDictionary(&top_dict)) { | 477 if (!parsed.GetAsDictionary(&top_dict)) { |
| 502 return false; | 478 return false; |
| 503 } | 479 } |
| 504 | 480 |
| 505 switch (fetch_api_) { | 481 const base::ListValue* categories_value = nullptr; |
| 506 case FetchAPI::CHROME_READER_API: { | 482 if (!top_dict->GetList("categories", &categories_value)) { |
| 507 const int kUnusedRemoteCategoryId = -1; | 483 return false; |
| 508 categories->push_back(FetchedCategory( | 484 } |
| 509 Category::FromKnownCategory(KnownCategories::ARTICLES), | |
| 510 BuildArticleCategoryInfo(base::nullopt))); | |
| 511 | 485 |
| 512 const base::ListValue* recos = nullptr; | 486 for (const auto& v : *categories_value) { |
| 513 return top_dict->GetList("recos", &recos) && | 487 std::string utf8_title; |
| 514 AddSuggestionsFromListValue( | 488 int remote_category_id = -1; |
| 515 /*content_suggestions_api=*/false, kUnusedRemoteCategoryId, | 489 const base::DictionaryValue* category_value = nullptr; |
| 516 *recos, &categories->back().suggestions, fetch_time); | 490 if (!(v->GetAsDictionary(&category_value) && |
| 491 category_value->GetString("localizedTitle", &utf8_title) && |
| 492 category_value->GetInteger("id", &remote_category_id) && |
| 493 (remote_category_id > 0))) { |
| 494 return false; |
| 517 } | 495 } |
| 518 | 496 |
| 519 case FetchAPI::CHROME_CONTENT_SUGGESTIONS_API: { | 497 RemoteSuggestion::PtrVector suggestions; |
| 520 const base::ListValue* categories_value = nullptr; | 498 const base::ListValue* suggestions_list = nullptr; |
| 521 if (!top_dict->GetList("categories", &categories_value)) { | 499 // Absence of a list of suggestions is treated as an empty list, which |
| 500 // is permissible. |
| 501 if (category_value->GetList("suggestions", &suggestions_list)) { |
| 502 if (!AddSuggestionsFromListValue( |
| 503 /*content_suggestions_api=*/true, remote_category_id, |
| 504 *suggestions_list, &suggestions, fetch_time)) { |
| 522 return false; | 505 return false; |
| 523 } | 506 } |
| 507 } |
| 508 Category category = Category::FromRemoteCategory(remote_category_id); |
| 509 if (category.IsKnownCategory(KnownCategories::ARTICLES)) { |
| 510 categories->push_back(FetchedCategory( |
| 511 category, BuildArticleCategoryInfo(base::UTF8ToUTF16(utf8_title)))); |
| 512 } else { |
| 513 // TODO(tschumann): Right now, the backend does not yet populate this |
| 514 // field. Make it mandatory once the backends provide it. |
| 515 bool allow_fetching_more_results = false; |
| 516 category_value->GetBoolean("allowFetchingMoreResults", |
| 517 &allow_fetching_more_results); |
| 518 categories->push_back(FetchedCategory( |
| 519 category, BuildRemoteCategoryInfo(base::UTF8ToUTF16(utf8_title), |
| 520 allow_fetching_more_results))); |
| 521 } |
| 522 categories->back().suggestions = std::move(suggestions); |
| 523 } |
| 524 | 524 |
| 525 for (const auto& v : *categories_value) { | 525 return true; |
| 526 std::string utf8_title; | |
| 527 int remote_category_id = -1; | |
| 528 const base::DictionaryValue* category_value = nullptr; | |
| 529 if (!(v->GetAsDictionary(&category_value) && | |
| 530 category_value->GetString("localizedTitle", &utf8_title) && | |
| 531 category_value->GetInteger("id", &remote_category_id) && | |
| 532 (remote_category_id > 0))) { | |
| 533 return false; | |
| 534 } | |
| 535 | |
| 536 RemoteSuggestion::PtrVector suggestions; | |
| 537 const base::ListValue* suggestions_list = nullptr; | |
| 538 // Absence of a list of suggestions is treated as an empty list, which | |
| 539 // is permissible. | |
| 540 if (category_value->GetList("suggestions", &suggestions_list)) { | |
| 541 if (!AddSuggestionsFromListValue( | |
| 542 /*content_suggestions_api=*/true, remote_category_id, | |
| 543 *suggestions_list, &suggestions, fetch_time)) { | |
| 544 return false; | |
| 545 } | |
| 546 } | |
| 547 Category category = Category::FromRemoteCategory(remote_category_id); | |
| 548 if (category.IsKnownCategory(KnownCategories::ARTICLES)) { | |
| 549 categories->push_back(FetchedCategory( | |
| 550 category, | |
| 551 BuildArticleCategoryInfo(base::UTF8ToUTF16(utf8_title)))); | |
| 552 } else { | |
| 553 // TODO(tschumann): Right now, the backend does not yet populate this | |
| 554 // field. Make it mandatory once the backends provide it. | |
| 555 bool allow_fetching_more_results = false; | |
| 556 category_value->GetBoolean("allowFetchingMoreResults", | |
| 557 &allow_fetching_more_results); | |
| 558 categories->push_back(FetchedCategory( | |
| 559 category, BuildRemoteCategoryInfo(base::UTF8ToUTF16(utf8_title), | |
| 560 allow_fetching_more_results))); | |
| 561 } | |
| 562 categories->back().suggestions = std::move(suggestions); | |
| 563 } | |
| 564 return true; | |
| 565 } | |
| 566 } | |
| 567 NOTREACHED(); | |
| 568 return false; | |
| 569 } | 526 } |
| 570 | 527 |
| 571 } // namespace ntp_snippets | 528 } // namespace ntp_snippets |
| OLD | NEW |