Sending LanguageModel info to suggestions server

components/ntp_snippets/remote/ntp_snippets_fetcher.cc

 // Copyright 2016 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "components/ntp_snippets/remote/ntp_snippets_fetcher.h"
 
 #include <cstdlib>
 
 #include "base/command_line.h"
 #include "base/files/file_path.h"
 #include "base/files/file_util.h"
 #include "base/json/json_writer.h"
 #include "base/memory/ptr_util.h"
 #include "base/metrics/histogram_macros.h"
 #include "base/metrics/sparse_histogram.h"
 #include "base/path_service.h"
 #include "base/strings/string_number_conversions.h"
 #include "base/strings/string_util.h"
 #include "base/strings/stringprintf.h"
 #include "base/strings/utf_string_conversions.h"
 #include "base/time/default_tick_clock.h"
 #include "base/values.h"
 #include "components/data_use_measurement/core/data_use_user_data.h"
 #include "components/ntp_snippets/category_factory.h"
+#include "components/ntp_snippets/features.h"
 #include "components/ntp_snippets/ntp_snippets_constants.h"
 #include "components/signin/core/browser/profile_oauth2_token_service.h"
 #include "components/signin/core/browser/signin_manager.h"
 #include "components/signin/core/browser/signin_manager_base.h"
 #include "components/variations/net/variations_http_headers.h"
 #include "components/variations/variations_associated_data.h"
 #include "net/base/load_flags.h"
 #include "net/http/http_request_headers.h"
 #include "net/http/http_response_headers.h"
 #include "net/http/http_status_code.h"
 #include "net/url_request/url_fetcher.h"
 #include "third_party/icu/source/common/unicode/uloc.h"
 #include "third_party/icu/source/common/unicode/utypes.h"
 
 using net::URLFetcher;
 using net::URLRequestContextGetter;
 using net::HttpRequestHeaders;
 using net::URLRequestStatus;
+using translate::LanguageModel;
 
 namespace ntp_snippets {
 
 namespace {
 
 const char kChromeReaderApiScope[] =
     "https://www.googleapis.com/auth/webhistory";
 const char kContentSuggestionsApiScope[] =
     "https://www.googleapis.com/auth/chrome-content-suggestions";
 const char kSnippetsServerNonAuthorizedFormat[] = "%s?key=%s";
 const char kAuthorizationRequestHeaderFormat[] = "Bearer %s";
 
 // Variation parameter for personalizing fetching of snippets.
 const char kPersonalizationName[] = "fetching_personalization";
 
 // Variation parameter for chrome-content-suggestions backend.
 const char kContentSuggestionsBackend[] = "content_suggestions_backend";
 
 // Constants for possible values of the "fetching_personalization" parameter.
 const char kPersonalizationPersonalString[] = "personal";
 const char kPersonalizationNonPersonalString[] = "non_personal";
 const char kPersonalizationBothString[] = "both";  // the default value
 
 const int kMaxExcludedIds = 100;
 
+// Variation parameter for sending LanguageModel info to the server.
+const char kSendTopLanguagesName[] = "send_top_languages";
+const char kSendTopLanguagesEnabled[] = "true";
+const char kSendTopLanguagesDisabled[] = "false";
+
 std::string FetchResultToString(NTPSnippetsFetcher::FetchResult result) {
   switch (result) {
     case NTPSnippetsFetcher::FetchResult::SUCCESS:
       return "OK";
     case NTPSnippetsFetcher::FetchResult::DEPRECATED_EMPTY_HOSTS:
       return "Cannot fetch for empty hosts list.";
     case NTPSnippetsFetcher::FetchResult::URL_REQUEST_STATUS_ERROR:
       return "URLRequestStatus error";
     case NTPSnippetsFetcher::FetchResult::HTTP_ERROR:
       return "HTTP error";
@@ skipping 32 matching lines @@
   NOTREACHED();
   return true;
 }
 
 std::string GetFetchEndpoint() {
   std::string endpoint = variations::GetVariationParamValue(
       ntp_snippets::kStudyName, kContentSuggestionsBackend);
   return endpoint.empty() ? kChromeReaderServer : endpoint;
 }
 
+bool IsSendingTopLanguagesEnabled() {
+  std::string send_top_languages = variations::GetVariationParamValueByFeature(
+        ntp_snippets::kArticleSuggestionsFeature, kSendTopLanguagesName);
+  if (send_top_languages == kSendTopLanguagesEnabled)
+    return true;
+  if (!send_top_languages.empty() &&
+      send_top_languages != kSendTopLanguagesDisabled) {
+    LOG(WARNING) << "Invalid value \"" << send_top_languages
+                 << "\" for variation parameter " << kSendTopLanguagesName;
+  }
+  return false;
+}
+
 bool UsesChromeContentSuggestionsAPI(const GURL& endpoint) {
   if (endpoint == GURL(kChromeReaderServer))
     return false;
 
   if (endpoint != GURL(kContentSuggestionsServer) &&
       endpoint != GURL(kContentSuggestionsDevServer) &&
       endpoint != GURL(kContentSuggestionsAlphaServer)) {
     LOG(WARNING) << "Unknown value for " << kContentSuggestionsBackend << ": "
                  << "assuming chromecontentsuggestions-style API";
   }
@@ skipping 26 matching lines @@
   return true;
 }
 
 // Translate the BCP 47 |language_code| into a posix locale string.
 std::string PosixLocaleFromBCP47Language(const std::string& language_code) {
   char locale[ULOC_FULLNAME_CAPACITY];
   UErrorCode error = U_ZERO_ERROR;
   // Translate the input to a posix locale.
   uloc_forLanguageTag(language_code.c_str(), locale, ULOC_FULLNAME_CAPACITY,
                       nullptr, &error);
-  DLOG_IF(WARNING, U_ZERO_ERROR != error)
-      << "Error in translating language code to a locale string: " << error;
+  if (error != U_ZERO_ERROR) {
+    DLOG(WARNING) << "Error in translating language code to a locale string: "
+                  << error;
+    return std::string();
+  }
   return locale;
 }
 
+std::string ISO639FromPosixLocale(const std::string& locale) {
+  char language[ULOC_LANG_CAPACITY];
+  UErrorCode error = U_ZERO_ERROR;
+  uloc_getLanguage(locale.c_str(), language, ULOC_LANG_CAPACITY, &error);
+  if (error != U_ZERO_ERROR) {
+    DLOG(WARNING)
+        << "Error in translating locale string to a ISO639 language code: "
+        << error;
+    return std::string();
+  }
+  return language;
+}
+
+void AppendLanguageInfoToList(base::ListValue* list,
+                              const LanguageModel::LanguageInfo& info) {
+  auto lang = base::MakeUnique<base::DictionaryValue>();
+  lang->SetString("language", info.language_code);
+  lang->SetDouble("frequency", info.frequency);
+  list->Append(std::move(lang));
+}
+
 }  // namespace
 
 NTPSnippetsFetcher::FetchedCategory::FetchedCategory(Category c)
     : category(c) {}
 
 NTPSnippetsFetcher::FetchedCategory::FetchedCategory(FetchedCategory&&) =
     default;
 NTPSnippetsFetcher::FetchedCategory::~FetchedCategory() = default;
 NTPSnippetsFetcher::FetchedCategory& NTPSnippetsFetcher::FetchedCategory::
 operator=(FetchedCategory&&) = default;
 
 NTPSnippetsFetcher::NTPSnippetsFetcher(
     SigninManagerBase* signin_manager,
     OAuth2TokenService* token_service,
     scoped_refptr<URLRequestContextGetter> url_request_context_getter,
     PrefService* pref_service,
     CategoryFactory* category_factory,
+    LanguageModel* language_model,
     const ParseJSONCallback& parse_json_callback,
     const std::string& api_key)
     : OAuth2TokenService::Consumer("ntp_snippets"),
       signin_manager_(signin_manager),
       token_service_(token_service),
       waiting_for_refresh_token_(false),
       url_request_context_getter_(std::move(url_request_context_getter)),
       category_factory_(category_factory),
+      language_model_(language_model),
       parse_json_callback_(parse_json_callback),
       count_to_fetch_(0),
       fetch_url_(GetFetchEndpoint()),
       fetch_api_(UsesChromeContentSuggestionsAPI(fetch_url_)
                      ? CHROME_CONTENT_SUGGESTIONS_API
                      : CHROME_READER_API),
       api_key_(api_key),
       interactive_request_(false),
       tick_clock_(new base::DefaultTickClock()),
       request_throttler_(
@@ skipping 146 matching lines @@
                                          : "BACKGROUND_PREFETCH");
 
       auto excluded = base::MakeUnique<base::ListValue>();
       for (const auto& id : excluded_ids) {
         excluded->AppendString(id);
         if (excluded->GetSize() >= kMaxExcludedIds)
           break;
       }
       request->Set("excludedSuggestionIds", std::move(excluded));
 
+      if (ui_language.frequency == 0 && other_top_language.frequency == 0)
+        break;
+
+      auto language_list = base::MakeUnique<base::ListValue>();
+      if (ui_language.frequency > 0)
+        AppendLanguageInfoToList(language_list.get(), ui_language);
+      if (other_top_language.frequency > 0)
+        AppendLanguageInfoToList(language_list.get(), other_top_language);
+      request->Set("top_languages", std::move(language_list));
+
       // TODO(sfiera): support authentication and personalization
       // TODO(sfiera): support count_to_fetch
       break;
     }
   }
 
   std::string request_json;
   bool success = base::JSONWriter::WriteWithOptions(
       *request, base::JSONWriter::OPTIONS_PRETTY_PRINT, &request_json);
   DCHECK(success);
@@ skipping 32 matching lines @@
   // Try to make fetching the files bit more robust even with poor connection.
   url_fetcher_->SetMaxRetriesOn5xx(3);
   url_fetcher_->Start();
 }
 
 bool NTPSnippetsFetcher::UsesAuthentication() const {
   return (personalization_ == Personalization::kPersonal ||
           personalization_ == Personalization::kBoth);
 }
 
+void NTPSnippetsFetcher::SetUpCommonFetchingParameters(
+    RequestParams* params) const {
+  params->fetch_api = fetch_api_;
+  params->host_restricts = hosts_;
+  params->user_locale = locale_;
+  params->excluded_ids = excluded_ids_;
+  params->count_to_fetch = count_to_fetch_;
+  params->interactive_request = interactive_request_;
+  // TODO(jkrcal): add the initializers into the struct and remove it from here
+  // and from the unit-tests (building the request).
+  params->ui_language.frequency = 0;
+  params->other_top_language.frequency = 0;
+
+  // TODO(jkrcal): Add language model factory for iOS and add fakes to tests so
+  // that |language_model_| is never nullptr. Remove this check and add a DCHECK
+  // into the constructor.
+  if (!language_model_ || !IsSendingTopLanguagesEnabled())
+    return;
+
+  params->ui_language.language_code = ISO639FromPosixLocale(locale_);
+  params->ui_language.frequency =
+      language_model_->GetLanguageFrequency(params->ui_language.language_code);
+
+  std::vector<LanguageModel::LanguageInfo> top_languages =
+      language_model_->GetTopLanguages();
+  for (const LanguageModel::LanguageInfo& info : top_languages) {
+    if (info.language_code != params->ui_language.language_code) {
+      params->other_top_language = info;
+      break;
+    }
+  }
+}
+
 void NTPSnippetsFetcher::FetchSnippetsNonAuthenticated() {
   // When not providing OAuth token, we need to pass the Google API key.
   GURL url(base::StringPrintf(kSnippetsServerNonAuthorizedFormat,
                               fetch_url_.spec().c_str(), api_key_.c_str()));
 
   RequestParams params;
-  params.fetch_api = fetch_api_;
-  params.host_restricts = hosts_;
-  params.excluded_ids = excluded_ids_;
-  params.count_to_fetch = count_to_fetch_;
-  params.interactive_request = interactive_request_;
-  params.user_locale = locale_;
+  SetUpCommonFetchingParameters(&params);
   FetchSnippetsImpl(url, std::string(), params.BuildRequest());
 }
 
 void NTPSnippetsFetcher::FetchSnippetsAuthenticated(
     const std::string& account_id,
     const std::string& oauth_access_token) {
   RequestParams params;
-  params.fetch_api = fetch_api_;
+  SetUpCommonFetchingParameters(&params);
   params.obfuscated_gaia_id = account_id;
   params.only_return_personalized_results =
       personalization_ == Personalization::kPersonal;
-  params.user_locale = locale_;
-  params.host_restricts = hosts_;
-  params.excluded_ids = excluded_ids_;
-  params.count_to_fetch = count_to_fetch_;
-  params.interactive_request = interactive_request_;
   // TODO(jkrcal, treib): Add unit-tests for authenticated fetches.
   FetchSnippetsImpl(fetch_url_,
                     base::StringPrintf(kAuthorizationRequestHeaderFormat,
                                        oauth_access_token.c_str()),
                     params.BuildRequest());
 }
 
 void NTPSnippetsFetcher::StartTokenRequest() {
   OAuth2TokenService::ScopeSet scopes;
   scopes.insert(fetch_api_ == CHROME_CONTENT_SUGGESTIONS_API
@@ skipping 184 matching lines @@
   UMA_HISTOGRAM_ENUMERATION("NewTabPage.Snippets.FetchResult",
                             static_cast<int>(result),
                             static_cast<int>(FetchResult::RESULT_MAX));
 
   DVLOG(1) << "Fetch finished: " << last_status_;
   if (!snippets_available_callback_.is_null())
     snippets_available_callback_.Run(std::move(fetched_categories));
 }
 
 }  // namespace ntp_snippets