OLD | NEW |
1 // Copyright 2016 The Chromium Authors. All rights reserved. | 1 // Copyright 2016 The Chromium Authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
4 | 4 |
5 #include "components/ntp_snippets/remote/ntp_snippets_fetcher.h" | 5 #include "components/ntp_snippets/remote/ntp_snippets_fetcher.h" |
6 | 6 |
7 #include <cstdlib> | 7 #include <cstdlib> |
8 | 8 |
9 #include "base/command_line.h" | 9 #include "base/command_line.h" |
10 #include "base/files/file_path.h" | 10 #include "base/files/file_path.h" |
11 #include "base/files/file_util.h" | 11 #include "base/files/file_util.h" |
12 #include "base/json/json_writer.h" | 12 #include "base/json/json_writer.h" |
13 #include "base/memory/ptr_util.h" | 13 #include "base/memory/ptr_util.h" |
14 #include "base/metrics/histogram_macros.h" | 14 #include "base/metrics/histogram_macros.h" |
15 #include "base/metrics/sparse_histogram.h" | 15 #include "base/metrics/sparse_histogram.h" |
16 #include "base/path_service.h" | 16 #include "base/path_service.h" |
17 #include "base/strings/string_number_conversions.h" | 17 #include "base/strings/string_number_conversions.h" |
18 #include "base/strings/string_util.h" | 18 #include "base/strings/string_util.h" |
19 #include "base/strings/stringprintf.h" | 19 #include "base/strings/stringprintf.h" |
20 #include "base/strings/utf_string_conversions.h" | 20 #include "base/strings/utf_string_conversions.h" |
21 #include "base/time/default_tick_clock.h" | 21 #include "base/time/default_tick_clock.h" |
22 #include "base/values.h" | 22 #include "base/values.h" |
23 #include "components/data_use_measurement/core/data_use_user_data.h" | 23 #include "components/data_use_measurement/core/data_use_user_data.h" |
24 #include "components/ntp_snippets/category_factory.h" | 24 #include "components/ntp_snippets/category_factory.h" |
25 #include "components/ntp_snippets/ntp_snippets_constants.h" | 25 #include "components/ntp_snippets/ntp_snippets_constants.h" |
26 #include "components/ntp_snippets/switches.h" | 26 #include "components/ntp_snippets/switches.h" |
| 27 #include "components/ntp_snippets/user_classifier.h" |
27 #include "components/signin/core/browser/profile_oauth2_token_service.h" | 28 #include "components/signin/core/browser/profile_oauth2_token_service.h" |
28 #include "components/signin/core/browser/signin_manager.h" | 29 #include "components/signin/core/browser/signin_manager.h" |
29 #include "components/signin/core/browser/signin_manager_base.h" | 30 #include "components/signin/core/browser/signin_manager_base.h" |
30 #include "components/variations/net/variations_http_headers.h" | 31 #include "components/variations/net/variations_http_headers.h" |
31 #include "components/variations/variations_associated_data.h" | 32 #include "components/variations/variations_associated_data.h" |
32 #include "net/base/load_flags.h" | 33 #include "net/base/load_flags.h" |
33 #include "net/http/http_request_headers.h" | 34 #include "net/http/http_request_headers.h" |
34 #include "net/http/http_response_headers.h" | 35 #include "net/http/http_response_headers.h" |
35 #include "net/http/http_status_code.h" | 36 #include "net/http/http_status_code.h" |
36 #include "net/url_request/url_fetcher.h" | 37 #include "net/url_request/url_fetcher.h" |
(...skipping 129 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
166 char locale[ULOC_FULLNAME_CAPACITY]; | 167 char locale[ULOC_FULLNAME_CAPACITY]; |
167 UErrorCode error = U_ZERO_ERROR; | 168 UErrorCode error = U_ZERO_ERROR; |
168 // Translate the input to a posix locale. | 169 // Translate the input to a posix locale. |
169 uloc_forLanguageTag(language_code.c_str(), locale, ULOC_FULLNAME_CAPACITY, | 170 uloc_forLanguageTag(language_code.c_str(), locale, ULOC_FULLNAME_CAPACITY, |
170 nullptr, &error); | 171 nullptr, &error); |
171 DLOG_IF(WARNING, U_ZERO_ERROR != error) | 172 DLOG_IF(WARNING, U_ZERO_ERROR != error) |
172 << "Error in translating language code to a locale string: " << error; | 173 << "Error in translating language code to a locale string: " << error; |
173 return locale; | 174 return locale; |
174 } | 175 } |
175 | 176 |
| 177 std::string GetUserClassString(UserClassifier::UserClass user_class) { |
| 178 switch (user_class) { |
| 179 case UserClassifier::UserClass::RARE_NTP_USER: |
| 180 return "RARE_NTP_USER"; |
| 181 case UserClassifier::UserClass::ACTIVE_NTP_USER: |
| 182 return "ACTIVE_NTP_USER"; |
| 183 case UserClassifier::UserClass::ACTIVE_SUGGESTIONS_CONSUMER: |
| 184 return "ACTIVE_SUGGESTIONS_CONSUMER"; |
| 185 } |
| 186 NOTREACHED(); |
| 187 return std::string(); |
| 188 } |
| 189 |
176 } // namespace | 190 } // namespace |
177 | 191 |
178 NTPSnippetsFetcher::FetchedCategory::FetchedCategory(Category c) | 192 NTPSnippetsFetcher::FetchedCategory::FetchedCategory(Category c) |
179 : category(c) {} | 193 : category(c) {} |
180 | 194 |
181 NTPSnippetsFetcher::FetchedCategory::FetchedCategory(FetchedCategory&&) = | 195 NTPSnippetsFetcher::FetchedCategory::FetchedCategory(FetchedCategory&&) = |
182 default; | 196 default; |
183 NTPSnippetsFetcher::FetchedCategory::~FetchedCategory() = default; | 197 NTPSnippetsFetcher::FetchedCategory::~FetchedCategory() = default; |
184 NTPSnippetsFetcher::FetchedCategory& NTPSnippetsFetcher::FetchedCategory:: | 198 NTPSnippetsFetcher::FetchedCategory& NTPSnippetsFetcher::FetchedCategory:: |
185 operator=(FetchedCategory&&) = default; | 199 operator=(FetchedCategory&&) = default; |
186 | 200 |
187 NTPSnippetsFetcher::NTPSnippetsFetcher( | 201 NTPSnippetsFetcher::NTPSnippetsFetcher( |
188 SigninManagerBase* signin_manager, | 202 SigninManagerBase* signin_manager, |
189 OAuth2TokenService* token_service, | 203 OAuth2TokenService* token_service, |
190 scoped_refptr<URLRequestContextGetter> url_request_context_getter, | 204 scoped_refptr<URLRequestContextGetter> url_request_context_getter, |
191 PrefService* pref_service, | 205 PrefService* pref_service, |
192 CategoryFactory* category_factory, | 206 CategoryFactory* category_factory, |
193 const ParseJSONCallback& parse_json_callback, | 207 const ParseJSONCallback& parse_json_callback, |
194 const std::string& api_key) | 208 const std::string& api_key, |
| 209 const UserClassifier* user_classifier) |
195 : OAuth2TokenService::Consumer("ntp_snippets"), | 210 : OAuth2TokenService::Consumer("ntp_snippets"), |
196 signin_manager_(signin_manager), | 211 signin_manager_(signin_manager), |
197 token_service_(token_service), | 212 token_service_(token_service), |
198 waiting_for_refresh_token_(false), | 213 waiting_for_refresh_token_(false), |
199 url_request_context_getter_(std::move(url_request_context_getter)), | 214 url_request_context_getter_(std::move(url_request_context_getter)), |
200 category_factory_(category_factory), | 215 category_factory_(category_factory), |
201 parse_json_callback_(parse_json_callback), | 216 parse_json_callback_(parse_json_callback), |
202 count_to_fetch_(0), | 217 count_to_fetch_(0), |
203 fetch_url_(GetFetchEndpoint()), | 218 fetch_url_(GetFetchEndpoint()), |
204 fetch_api_(UsesChromeContentSuggestionsAPI(fetch_url_) | 219 fetch_api_(UsesChromeContentSuggestionsAPI(fetch_url_) |
205 ? CHROME_CONTENT_SUGGESTIONS_API | 220 ? CHROME_CONTENT_SUGGESTIONS_API |
206 : CHROME_READER_API), | 221 : CHROME_READER_API), |
207 api_key_(api_key), | 222 api_key_(api_key), |
208 interactive_request_(false), | 223 interactive_request_(false), |
209 tick_clock_(new base::DefaultTickClock()), | 224 tick_clock_(new base::DefaultTickClock()), |
210 request_throttler_( | 225 user_classifier_(user_classifier), |
| 226 request_throttler_rare_ntp_user_( |
211 pref_service, | 227 pref_service, |
212 RequestThrottler::RequestType::CONTENT_SUGGESTION_FETCHER), | 228 RequestThrottler::RequestType:: |
| 229 CONTENT_SUGGESTION_FETCHER_RARE_NTP_USER), |
| 230 request_throttler_active_ntp_user_( |
| 231 pref_service, |
| 232 RequestThrottler::RequestType:: |
| 233 CONTENT_SUGGESTION_FETCHER_ACTIVE_NTP_USER), |
| 234 request_throttler_active_suggestions_consumer_( |
| 235 pref_service, |
| 236 RequestThrottler::RequestType:: |
| 237 CONTENT_SUGGESTION_FETCHER_ACTIVE_SUGGESTIONS_CONSUMER), |
213 oauth_token_retried_(false), | 238 oauth_token_retried_(false), |
214 weak_ptr_factory_(this) { | 239 weak_ptr_factory_(this) { |
215 // Parse the variation parameters and set the defaults if missing. | 240 // Parse the variation parameters and set the defaults if missing. |
216 std::string personalization = variations::GetVariationParamValue( | 241 std::string personalization = variations::GetVariationParamValue( |
217 ntp_snippets::kStudyName, kPersonalizationName); | 242 ntp_snippets::kStudyName, kPersonalizationName); |
218 if (personalization == kPersonalizationNonPersonalString) { | 243 if (personalization == kPersonalizationNonPersonalString) { |
219 personalization_ = Personalization::kNonPersonal; | 244 personalization_ = Personalization::kNonPersonal; |
220 } else if (personalization == kPersonalizationPersonalString) { | 245 } else if (personalization == kPersonalizationPersonalString) { |
221 personalization_ = Personalization::kPersonal; | 246 personalization_ = Personalization::kPersonal; |
222 } else { | 247 } else { |
(...skipping 26 matching lines...) Expand all Loading... |
249 const SnippetsAvailableCallback& callback) { | 274 const SnippetsAvailableCallback& callback) { |
250 snippets_available_callback_ = callback; | 275 snippets_available_callback_ = callback; |
251 } | 276 } |
252 | 277 |
253 void NTPSnippetsFetcher::FetchSnippetsFromHosts( | 278 void NTPSnippetsFetcher::FetchSnippetsFromHosts( |
254 const std::set<std::string>& hosts, | 279 const std::set<std::string>& hosts, |
255 const std::string& language_code, | 280 const std::string& language_code, |
256 const std::set<std::string>& excluded_ids, | 281 const std::set<std::string>& excluded_ids, |
257 int count, | 282 int count, |
258 bool interactive_request) { | 283 bool interactive_request) { |
259 if (!request_throttler_.DemandQuotaForRequest(interactive_request)) { | 284 if (!DemandQuotaForRequest(interactive_request)) { |
260 FetchFinished(OptionalSnippets(), | 285 FetchFinished(OptionalSnippets(), |
261 interactive_request | 286 interactive_request |
262 ? FetchResult::INTERACTIVE_QUOTA_ERROR | 287 ? FetchResult::INTERACTIVE_QUOTA_ERROR |
263 : FetchResult::NON_INTERACTIVE_QUOTA_ERROR, | 288 : FetchResult::NON_INTERACTIVE_QUOTA_ERROR, |
264 /*extra_message=*/std::string()); | 289 /*extra_message=*/std::string()); |
265 return; | 290 return; |
266 } | 291 } |
267 | 292 |
268 hosts_ = hosts; | 293 hosts_ = hosts; |
269 fetch_start_time_ = tick_clock_->NowTicks(); | 294 fetch_start_time_ = tick_clock_->NowTicks(); |
(...skipping 29 matching lines...) Expand all Loading... |
299 } | 324 } |
300 } | 325 } |
301 | 326 |
302 NTPSnippetsFetcher::RequestParams::RequestParams() | 327 NTPSnippetsFetcher::RequestParams::RequestParams() |
303 : fetch_api(), | 328 : fetch_api(), |
304 obfuscated_gaia_id(), | 329 obfuscated_gaia_id(), |
305 only_return_personalized_results(), | 330 only_return_personalized_results(), |
306 user_locale(), | 331 user_locale(), |
307 host_restricts(), | 332 host_restricts(), |
308 count_to_fetch(), | 333 count_to_fetch(), |
309 interactive_request() {} | 334 interactive_request(), |
| 335 user_class() {} |
310 | 336 |
311 NTPSnippetsFetcher::RequestParams::~RequestParams() = default; | 337 NTPSnippetsFetcher::RequestParams::~RequestParams() = default; |
312 | 338 |
313 std::string NTPSnippetsFetcher::RequestParams::BuildRequest() { | 339 std::string NTPSnippetsFetcher::RequestParams::BuildRequest() { |
314 auto request = base::MakeUnique<base::DictionaryValue>(); | 340 auto request = base::MakeUnique<base::DictionaryValue>(); |
315 switch (fetch_api) { | 341 switch (fetch_api) { |
316 case CHROME_READER_API: { | 342 case CHROME_READER_API: { |
317 auto content_params = base::MakeUnique<base::DictionaryValue>(); | 343 auto content_params = base::MakeUnique<base::DictionaryValue>(); |
318 content_params->SetBoolean("only_return_personalized_results", | 344 content_params->SetBoolean("only_return_personalized_results", |
319 only_return_personalized_results); | 345 only_return_personalized_results); |
(...skipping 54 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
374 ? "USER_ACTION" | 400 ? "USER_ACTION" |
375 : "BACKGROUND_PREFETCH"); | 401 : "BACKGROUND_PREFETCH"); |
376 | 402 |
377 auto excluded = base::MakeUnique<base::ListValue>(); | 403 auto excluded = base::MakeUnique<base::ListValue>(); |
378 for (const auto& id : excluded_ids) { | 404 for (const auto& id : excluded_ids) { |
379 excluded->AppendString(id); | 405 excluded->AppendString(id); |
380 if (excluded->GetSize() >= kMaxExcludedIds) | 406 if (excluded->GetSize() >= kMaxExcludedIds) |
381 break; | 407 break; |
382 } | 408 } |
383 request->Set("excludedSuggestionIds", std::move(excluded)); | 409 request->Set("excludedSuggestionIds", std::move(excluded)); |
| 410 request->SetString("user_activeness_class", user_class); |
384 | 411 |
385 // TODO(sfiera): support authentication and personalization | 412 // TODO(sfiera): support authentication and personalization |
386 // TODO(sfiera): support count_to_fetch | 413 // TODO(sfiera): support count_to_fetch |
387 break; | 414 break; |
388 } | 415 } |
389 } | 416 } |
390 | 417 |
391 std::string request_json; | 418 std::string request_json; |
392 bool success = base::JSONWriter::WriteWithOptions( | 419 bool success = base::JSONWriter::WriteWithOptions( |
393 *request, base::JSONWriter::OPTIONS_PRETTY_PRINT, &request_json); | 420 *request, base::JSONWriter::OPTIONS_PRETTY_PRINT, &request_json); |
(...skipping 52 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
446 fetch_url_.spec().c_str(), api_key_.c_str())); | 473 fetch_url_.spec().c_str(), api_key_.c_str())); |
447 | 474 |
448 RequestParams params; | 475 RequestParams params; |
449 params.fetch_api = fetch_api_; | 476 params.fetch_api = fetch_api_; |
450 params.host_restricts = | 477 params.host_restricts = |
451 UsesHostRestrictions() ? hosts_ : std::set<std::string>(); | 478 UsesHostRestrictions() ? hosts_ : std::set<std::string>(); |
452 params.excluded_ids = excluded_ids_; | 479 params.excluded_ids = excluded_ids_; |
453 params.count_to_fetch = count_to_fetch_; | 480 params.count_to_fetch = count_to_fetch_; |
454 params.interactive_request = interactive_request_; | 481 params.interactive_request = interactive_request_; |
455 params.user_locale = locale_; | 482 params.user_locale = locale_; |
| 483 params.user_class = GetUserClassString(user_classifier_->GetUserClass()); |
456 FetchSnippetsImpl(url, std::string(), params.BuildRequest()); | 484 FetchSnippetsImpl(url, std::string(), params.BuildRequest()); |
457 } | 485 } |
458 | 486 |
459 void NTPSnippetsFetcher::FetchSnippetsAuthenticated( | 487 void NTPSnippetsFetcher::FetchSnippetsAuthenticated( |
460 const std::string& account_id, | 488 const std::string& account_id, |
461 const std::string& oauth_access_token) { | 489 const std::string& oauth_access_token) { |
462 RequestParams params; | 490 RequestParams params; |
463 params.fetch_api = fetch_api_; | 491 params.fetch_api = fetch_api_; |
464 params.obfuscated_gaia_id = account_id; | 492 params.obfuscated_gaia_id = account_id; |
465 params.only_return_personalized_results = | 493 params.only_return_personalized_results = |
466 personalization_ == Personalization::kPersonal; | 494 personalization_ == Personalization::kPersonal; |
467 params.user_locale = locale_; | 495 params.user_locale = locale_; |
468 params.host_restricts = | 496 params.host_restricts = |
469 UsesHostRestrictions() ? hosts_ : std::set<std::string>(); | 497 UsesHostRestrictions() ? hosts_ : std::set<std::string>(); |
470 params.excluded_ids = excluded_ids_; | 498 params.excluded_ids = excluded_ids_; |
471 params.count_to_fetch = count_to_fetch_; | 499 params.count_to_fetch = count_to_fetch_; |
472 params.interactive_request = interactive_request_; | 500 params.interactive_request = interactive_request_; |
| 501 params.user_class = GetUserClassString(user_classifier_->GetUserClass()); |
473 // TODO(jkrcal, treib): Add unit-tests for authenticated fetches. | 502 // TODO(jkrcal, treib): Add unit-tests for authenticated fetches. |
474 FetchSnippetsImpl(fetch_url_, | 503 FetchSnippetsImpl(fetch_url_, |
475 base::StringPrintf(kAuthorizationRequestHeaderFormat, | 504 base::StringPrintf(kAuthorizationRequestHeaderFormat, |
476 oauth_access_token.c_str()), | 505 oauth_access_token.c_str()), |
477 params.BuildRequest()); | 506 params.BuildRequest()); |
478 } | 507 } |
479 | 508 |
480 void NTPSnippetsFetcher::StartTokenRequest() { | 509 void NTPSnippetsFetcher::StartTokenRequest() { |
481 OAuth2TokenService::ScopeSet scopes; | 510 OAuth2TokenService::ScopeSet scopes; |
482 scopes.insert(fetch_api_ == CHROME_CONTENT_SUGGESTIONS_API | 511 scopes.insert(fetch_api_ == CHROME_CONTENT_SUGGESTIONS_API |
(...skipping 180 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
663 } | 692 } |
664 UMA_HISTOGRAM_ENUMERATION("NewTabPage.Snippets.FetchResult", | 693 UMA_HISTOGRAM_ENUMERATION("NewTabPage.Snippets.FetchResult", |
665 static_cast<int>(result), | 694 static_cast<int>(result), |
666 static_cast<int>(FetchResult::RESULT_MAX)); | 695 static_cast<int>(FetchResult::RESULT_MAX)); |
667 | 696 |
668 DVLOG(1) << "Fetch finished: " << last_status_; | 697 DVLOG(1) << "Fetch finished: " << last_status_; |
669 if (!snippets_available_callback_.is_null()) | 698 if (!snippets_available_callback_.is_null()) |
670 snippets_available_callback_.Run(std::move(snippets)); | 699 snippets_available_callback_.Run(std::move(snippets)); |
671 } | 700 } |
672 | 701 |
| 702 bool NTPSnippetsFetcher::DemandQuotaForRequest(bool interactive_request) { |
| 703 switch (user_classifier_->GetUserClass()) { |
| 704 case UserClassifier::UserClass::RARE_NTP_USER: |
| 705 return request_throttler_rare_ntp_user_.DemandQuotaForRequest( |
| 706 interactive_request); |
| 707 case UserClassifier::UserClass::ACTIVE_NTP_USER: |
| 708 return request_throttler_active_ntp_user_.DemandQuotaForRequest( |
| 709 interactive_request); |
| 710 case UserClassifier::UserClass::ACTIVE_SUGGESTIONS_CONSUMER: |
| 711 return request_throttler_active_suggestions_consumer_ |
| 712 .DemandQuotaForRequest(interactive_request); |
| 713 } |
| 714 NOTREACHED(); |
| 715 return false; |
| 716 } |
| 717 |
673 } // namespace ntp_snippets | 718 } // namespace ntp_snippets |
OLD | NEW |