Connecting UserClassifier to NtpSnippetsFetcher

components/ntp_snippets/remote/ntp_snippets_fetcher.cc

 // Copyright 2016 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "components/ntp_snippets/remote/ntp_snippets_fetcher.h"
 
 #include <cstdlib>
 
 #include "base/command_line.h"
 #include "base/files/file_path.h"
 #include "base/files/file_util.h"
 #include "base/json/json_writer.h"
 #include "base/memory/ptr_util.h"
 #include "base/metrics/histogram_macros.h"
 #include "base/metrics/sparse_histogram.h"
 #include "base/path_service.h"
 #include "base/strings/string_number_conversions.h"
 #include "base/strings/string_util.h"
 #include "base/strings/stringprintf.h"
 #include "base/strings/utf_string_conversions.h"
 #include "base/time/default_tick_clock.h"
 #include "base/values.h"
 #include "components/data_use_measurement/core/data_use_user_data.h"
 #include "components/ntp_snippets/category_factory.h"
 #include "components/ntp_snippets/features.h"
 #include "components/ntp_snippets/ntp_snippets_constants.h"
+#include "components/ntp_snippets/user_classifier.h"
 #include "components/signin/core/browser/profile_oauth2_token_service.h"
 #include "components/signin/core/browser/signin_manager.h"
 #include "components/signin/core/browser/signin_manager_base.h"
 #include "components/variations/net/variations_http_headers.h"
 #include "components/variations/variations_associated_data.h"
 #include "net/base/load_flags.h"
 #include "net/http/http_request_headers.h"
 #include "net/http/http_response_headers.h"
 #include "net/http/http_status_code.h"
 #include "net/url_request/url_fetcher.h"
@@ skipping 25 matching lines @@
 
 // Constants for possible values of the "fetching_personalization" parameter.
 const char kPersonalizationPersonalString[] = "personal";
 const char kPersonalizationNonPersonalString[] = "non_personal";
 const char kPersonalizationBothString[] = "both";  // the default value
 
 const int kMaxExcludedIds = 100;
 
 // Variation parameter for sending LanguageModel info to the server.
 const char kSendTopLanguagesName[] = "send_top_languages";
-const char kSendTopLanguagesEnabled[] = "true";
-const char kSendTopLanguagesDisabled[] = "false";
+
+// Variation parameter for sending UserClassifier info to the server.
+const char kSendUserClassName[] = "send_user_class";
+
+const char kBooleanParameterEnabled[] = "true";
+const char kBooleanParameterDisabled[] = "false";
 
 std::string FetchResultToString(NTPSnippetsFetcher::FetchResult result) {
   switch (result) {
     case NTPSnippetsFetcher::FetchResult::SUCCESS:
       return "OK";
     case NTPSnippetsFetcher::FetchResult::DEPRECATED_EMPTY_HOSTS:
       return "Cannot fetch for empty hosts list.";
     case NTPSnippetsFetcher::FetchResult::URL_REQUEST_STATUS_ERROR:
       return "URLRequestStatus error";
     case NTPSnippetsFetcher::FetchResult::HTTP_ERROR:
@@ skipping 33 matching lines @@
   NOTREACHED();
   return true;
 }
 
 std::string GetFetchEndpoint() {
   std::string endpoint = variations::GetVariationParamValue(
       ntp_snippets::kStudyName, kContentSuggestionsBackend);
   return endpoint.empty() ? kChromeReaderServer : endpoint;
 }
 
+bool IsBooleanParameterEnabled(std::string param_name, bool default_value) {

[Marc Treib, 2016/10/10 08:01:01]

nit: const std::string&

[jkrcal, 2016/10/10 08:37:16]

Oh, again :|

By the way, I am sick of this parsing of variation params over and over again.
I'd like to put some common functions in variations:: when I find a bit of time
to discuss it with Alexei (and to do it).

+  std::string param_value = variations::GetVariationParamValueByFeature(
+        ntp_snippets::kArticleSuggestionsFeature, param_name);
+  if (param_value == kBooleanParameterEnabled)
+    return true;
+  if (param_value == kBooleanParameterDisabled)
+    return false;
+  if (!param_value.empty()) {
+    LOG(WARNING) << "Invalid value \"" << param_value
+                 << "\" for variation parameter " << param_name;
+  }
+  return default_value;
+}
+
 bool IsSendingTopLanguagesEnabled() {
-  std::string send_top_languages = variations::GetVariationParamValueByFeature(
-        ntp_snippets::kArticleSuggestionsFeature, kSendTopLanguagesName);
-  if (send_top_languages == kSendTopLanguagesEnabled)
-    return true;
-  if (!send_top_languages.empty() &&
-      send_top_languages != kSendTopLanguagesDisabled) {
-    LOG(WARNING) << "Invalid value \"" << send_top_languages
-                 << "\" for variation parameter " << kSendTopLanguagesName;
-  }
-  return false;
+  return IsBooleanParameterEnabled(kSendTopLanguagesName, false);
+}
+
+bool IsSendingUserClassEnabled() {
+  return IsBooleanParameterEnabled(kSendUserClassName, false);
 }
 
 bool UsesChromeContentSuggestionsAPI(const GURL& endpoint) {
   if (endpoint == GURL(kChromeReaderServer))
     return false;
 
   if (endpoint != GURL(kContentSuggestionsServer) &&
       endpoint != GURL(kContentSuggestionsDevServer) &&
       endpoint != GURL(kContentSuggestionsAlphaServer)) {
     LOG(WARNING) << "Unknown value for " << kContentSuggestionsBackend << ": "
@@ skipping 57 matching lines @@
 }
 
 void AppendLanguageInfoToList(base::ListValue* list,
                               const LanguageModel::LanguageInfo& info) {
   auto lang = base::MakeUnique<base::DictionaryValue>();
   lang->SetString("language", info.language_code);
   lang->SetDouble("frequency", info.frequency);
   list->Append(std::move(lang));
 }
 
+std::string GetUserClassString(UserClassifier::UserClass user_class) {
+  switch (user_class) {
+    case UserClassifier::UserClass::RARE_NTP_USER:
+      return "RARE_NTP_USER";
+    case UserClassifier::UserClass::ACTIVE_NTP_USER:
+      return "ACTIVE_NTP_USER";
+    case UserClassifier::UserClass::ACTIVE_SUGGESTIONS_CONSUMER:
+      return "ACTIVE_SUGGESTIONS_CONSUMER";
+  }
+  NOTREACHED();
+  return std::string();
+}
+
 }  // namespace
 
 NTPSnippetsFetcher::FetchedCategory::FetchedCategory(Category c)
     : category(c) {}
 
 NTPSnippetsFetcher::FetchedCategory::FetchedCategory(FetchedCategory&&) =
     default;
 NTPSnippetsFetcher::FetchedCategory::~FetchedCategory() = default;
 NTPSnippetsFetcher::FetchedCategory& NTPSnippetsFetcher::FetchedCategory::
 operator=(FetchedCategory&&) = default;
 
 NTPSnippetsFetcher::NTPSnippetsFetcher(
     SigninManagerBase* signin_manager,
     OAuth2TokenService* token_service,
     scoped_refptr<URLRequestContextGetter> url_request_context_getter,
     PrefService* pref_service,
     CategoryFactory* category_factory,
     LanguageModel* language_model,
     const ParseJSONCallback& parse_json_callback,
-    const std::string& api_key)
+    const std::string& api_key,
+    const UserClassifier* user_classifier)
     : OAuth2TokenService::Consumer("ntp_snippets"),
       signin_manager_(signin_manager),
       token_service_(token_service),
       waiting_for_refresh_token_(false),
       url_request_context_getter_(std::move(url_request_context_getter)),
       category_factory_(category_factory),
       language_model_(language_model),
       parse_json_callback_(parse_json_callback),
       count_to_fetch_(0),
       fetch_url_(GetFetchEndpoint()),
       fetch_api_(UsesChromeContentSuggestionsAPI(fetch_url_)
                      ? CHROME_CONTENT_SUGGESTIONS_API
                      : CHROME_READER_API),
       api_key_(api_key),
       interactive_request_(false),
       tick_clock_(new base::DefaultTickClock()),
-      request_throttler_(
+      user_classifier_(user_classifier),
+      request_throttler_rare_ntp_user_(
           pref_service,
-          RequestThrottler::RequestType::CONTENT_SUGGESTION_FETCHER),
+          RequestThrottler::RequestType::
+              CONTENT_SUGGESTION_FETCHER_RARE_NTP_USER),
+      request_throttler_active_ntp_user_(
+          pref_service,
+          RequestThrottler::RequestType::
+              CONTENT_SUGGESTION_FETCHER_ACTIVE_NTP_USER),
+      request_throttler_active_suggestions_consumer_(
+          pref_service,
+          RequestThrottler::RequestType::
+              CONTENT_SUGGESTION_FETCHER_ACTIVE_SUGGESTIONS_CONSUMER),
       oauth_token_retried_(false),
       weak_ptr_factory_(this) {
   // Parse the variation parameters and set the defaults if missing.
   std::string personalization = variations::GetVariationParamValue(
       ntp_snippets::kStudyName, kPersonalizationName);
   if (personalization == kPersonalizationNonPersonalString) {
     personalization_ = Personalization::kNonPersonal;
   } else if (personalization == kPersonalizationPersonalString) {
     personalization_ = Personalization::kPersonal;
   } else {
@@ skipping 14 matching lines @@
     const SnippetsAvailableCallback& callback) {
   snippets_available_callback_ = callback;
 }
 
 void NTPSnippetsFetcher::FetchSnippetsFromHosts(
     const std::set<std::string>& hosts,
     const std::string& language_code,
     const std::set<std::string>& excluded_ids,
     int count,
     bool interactive_request) {
-  if (!request_throttler_.DemandQuotaForRequest(interactive_request)) {
+  if (!DemandQuotaForRequest(interactive_request)) {
     FetchFinished(OptionalFetchedCategories(),
                   interactive_request
                       ? FetchResult::INTERACTIVE_QUOTA_ERROR
                       : FetchResult::NON_INTERACTIVE_QUOTA_ERROR,
                   /*extra_message=*/std::string());
     return;
   }
 
   hosts_ = hosts;
   fetch_start_time_ = tick_clock_->NowTicks();
@@ skipping 23 matching lines @@
   }
 }
 
 NTPSnippetsFetcher::RequestParams::RequestParams()
     : fetch_api(),
       obfuscated_gaia_id(),
       only_return_personalized_results(),
       user_locale(),
       host_restricts(),
       count_to_fetch(),
-      interactive_request() {}
+      interactive_request(),
+      user_class() {}
 
 NTPSnippetsFetcher::RequestParams::~RequestParams() = default;
 
 std::string NTPSnippetsFetcher::RequestParams::BuildRequest() {
   auto request = base::MakeUnique<base::DictionaryValue>();
   switch (fetch_api) {
     case CHROME_READER_API: {
       auto content_params = base::MakeUnique<base::DictionaryValue>();
       content_params->SetBoolean("only_return_personalized_results",
                                  only_return_personalized_results);
@@ skipping 55 matching lines @@
                                          : "BACKGROUND_PREFETCH");
 
       auto excluded = base::MakeUnique<base::ListValue>();
       for (const auto& id : excluded_ids) {
         excluded->AppendString(id);
         if (excluded->GetSize() >= kMaxExcludedIds)
           break;
       }
       request->Set("excludedSuggestionIds", std::move(excluded));
 
+      if (!user_class.empty())
+        request->SetString("user_activeness_class", user_class);

[Marc Treib, 2016/10/10 08:01:01]

What's up with the naming conventions here - "excludedSuggestionIds" vs
"user_activeness_class"? Are we sure all this works as expected?

[jkrcal, 2016/10/10 08:37:16]

Oh, thanks for spotting it! It's hard to test as the server currently still does
not recognize the param in any form :) Let me sneak in a fix for another new
param "top_languages" below, as well.

+
       if (ui_language.frequency == 0 && other_top_language.frequency == 0)
         break;
 
       auto language_list = base::MakeUnique<base::ListValue>();
       if (ui_language.frequency > 0)
         AppendLanguageInfoToList(language_list.get(), ui_language);
       if (other_top_language.frequency > 0)
         AppendLanguageInfoToList(language_list.get(), other_top_language);
       request->Set("top_languages", std::move(language_list));
 
@@ skipping 50 matching lines @@
 }
 
 void NTPSnippetsFetcher::SetUpCommonFetchingParameters(
     RequestParams* params) const {
   params->fetch_api = fetch_api_;
   params->host_restricts = hosts_;
   params->user_locale = locale_;
   params->excluded_ids = excluded_ids_;
   params->count_to_fetch = count_to_fetch_;
   params->interactive_request = interactive_request_;
+
+  if (IsSendingUserClassEnabled())
+    params->user_class = GetUserClassString(user_classifier_->GetUserClass());
+
   // TODO(jkrcal): add the initializers into the struct and remove it from here
   // and from the unit-tests (building the request).
   params->ui_language.frequency = 0;
   params->other_top_language.frequency = 0;
 
   // TODO(jkrcal): Add language model factory for iOS and add fakes to tests so
   // that |language_model_| is never nullptr. Remove this check and add a DCHECK
   // into the constructor.
   if (!language_model_ || !IsSendingTopLanguagesEnabled())
     return;
@@ skipping 226 matching lines @@
   }
   UMA_HISTOGRAM_ENUMERATION("NewTabPage.Snippets.FetchResult",
                             static_cast<int>(result),
                             static_cast<int>(FetchResult::RESULT_MAX));
 
   DVLOG(1) << "Fetch finished: " << last_status_;
   if (!snippets_available_callback_.is_null())
     snippets_available_callback_.Run(std::move(fetched_categories));
 }
 
+bool NTPSnippetsFetcher::DemandQuotaForRequest(bool interactive_request) {
+  switch (user_classifier_->GetUserClass()) {
+    case UserClassifier::UserClass::RARE_NTP_USER:
+      return request_throttler_rare_ntp_user_.DemandQuotaForRequest(
+          interactive_request);
+    case UserClassifier::UserClass::ACTIVE_NTP_USER:
+      return request_throttler_active_ntp_user_.DemandQuotaForRequest(
+          interactive_request);
+    case UserClassifier::UserClass::ACTIVE_SUGGESTIONS_CONSUMER:
+      return request_throttler_active_suggestions_consumer_
+          .DemandQuotaForRequest(interactive_request);
+  }
+  NOTREACHED();
+  return false;
+}
+
 }  // namespace ntp_snippets