Sending LanguageModel info to suggestions server

components/ntp_snippets/remote/ntp_snippets_fetcher.cc

 // Copyright 2016 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "components/ntp_snippets/remote/ntp_snippets_fetcher.h"
 
 #include <cstdlib>
 
 #include "base/command_line.h"
 #include "base/files/file_path.h"
 #include "base/files/file_util.h"
 #include "base/json/json_writer.h"
 #include "base/memory/ptr_util.h"
 #include "base/metrics/histogram_macros.h"
 #include "base/metrics/sparse_histogram.h"
 #include "base/path_service.h"
 #include "base/strings/string_number_conversions.h"
 #include "base/strings/string_util.h"
 #include "base/strings/stringprintf.h"
 #include "base/strings/utf_string_conversions.h"
 #include "base/time/default_tick_clock.h"
 #include "base/values.h"
 #include "components/data_use_measurement/core/data_use_user_data.h"
 #include "components/ntp_snippets/category_factory.h"
 #include "components/ntp_snippets/ntp_snippets_constants.h"
 #include "components/signin/core/browser/profile_oauth2_token_service.h"
 #include "components/signin/core/browser/signin_manager.h"
 #include "components/signin/core/browser/signin_manager_base.h"
+#include "components/translate/core/browser/language_model.h"
 #include "components/variations/net/variations_http_headers.h"
 #include "components/variations/variations_associated_data.h"
 #include "net/base/load_flags.h"
 #include "net/http/http_request_headers.h"
 #include "net/http/http_response_headers.h"
 #include "net/http/http_status_code.h"
 #include "net/url_request/url_fetcher.h"
 #include "third_party/icu/source/common/unicode/uloc.h"
 #include "third_party/icu/source/common/unicode/utypes.h"
 
 using net::URLFetcher;
 using net::URLRequestContextGetter;
 using net::HttpRequestHeaders;
 using net::URLRequestStatus;
+using translate::LanguageModel;
 
 namespace ntp_snippets {
 
 namespace {
 
 const char kChromeReaderApiScope[] =
     "https://www.googleapis.com/auth/webhistory";
 const char kContentSuggestionsApiScope[] =
     "https://www.googleapis.com/auth/chrome-content-suggestions";
 const char kSnippetsServerNonAuthorizedFormat[] = "%s?key=%s";
@@ skipping 106 matching lines @@
   char locale[ULOC_FULLNAME_CAPACITY];
   UErrorCode error = U_ZERO_ERROR;
   // Translate the input to a posix locale.
   uloc_forLanguageTag(language_code.c_str(), locale, ULOC_FULLNAME_CAPACITY,
                       nullptr, &error);
   DLOG_IF(WARNING, U_ZERO_ERROR != error)
       << "Error in translating language code to a locale string: " << error;
   return locale;
 }
 
+std::string ISO639FromPosixLocale(const std::string& locale) {
+  char language[ULOC_LANG_CAPACITY];

[Bernhard Bauer, 2016/10/07 12:26:13]

This is not going to be zero-initialized, right? So if for some reason we can't
convert the language, are we going to construct a string from uninitialized
data?

[jkrcal, 2016/10/07 12:58:43]

Good point! This applies to the function above as well. I prefer to be more
explicit and use a regular if for error. Is it okay for you?

+  UErrorCode error = U_ZERO_ERROR;
+  uloc_getLanguage(locale.c_str(), language, ULOC_LANG_CAPACITY, &error);

[Bernhard Bauer, 2016/10/07 12:26:13]

#include "third_party/icu/source/common/unicode/uloc.h"?

[jkrcal, 2016/10/07 12:58:43]

It is already there, isn't it? ;)

+  DLOG_IF(WARNING, U_ZERO_ERROR != error)

[Bernhard Bauer, 2016/10/07 12:26:13]

Write the condition the natural way around? A modern compiler would complain
anyway if you accidentally used = instead of ==.

[jkrcal, 2016/10/07 12:58:43]

Done.

+      << "Error in translating locale string to a ISO639 language code: "
+      << error;
+  return language;
+}
+
 }  // namespace
 
 NTPSnippetsFetcher::FetchedCategory::FetchedCategory(Category c)
     : category(c) {}
 
 NTPSnippetsFetcher::FetchedCategory::FetchedCategory(FetchedCategory&&) =
     default;
 NTPSnippetsFetcher::FetchedCategory::~FetchedCategory() = default;
 NTPSnippetsFetcher::FetchedCategory& NTPSnippetsFetcher::FetchedCategory::
 operator=(FetchedCategory&&) = default;
 
 NTPSnippetsFetcher::NTPSnippetsFetcher(
     SigninManagerBase* signin_manager,
     OAuth2TokenService* token_service,
     scoped_refptr<URLRequestContextGetter> url_request_context_getter,
     PrefService* pref_service,
     CategoryFactory* category_factory,
+    LanguageModel* language_model,
     const ParseJSONCallback& parse_json_callback,
     const std::string& api_key)
     : OAuth2TokenService::Consumer("ntp_snippets"),
       signin_manager_(signin_manager),
       token_service_(token_service),
       waiting_for_refresh_token_(false),
       url_request_context_getter_(std::move(url_request_context_getter)),
       category_factory_(category_factory),
+      language_model_(language_model),
       parse_json_callback_(parse_json_callback),
       count_to_fetch_(0),
       fetch_url_(GetFetchEndpoint()),
       fetch_api_(UsesChromeContentSuggestionsAPI(fetch_url_)
                      ? CHROME_CONTENT_SUGGESTIONS_API
                      : CHROME_READER_API),
       api_key_(api_key),
       interactive_request_(false),
       tick_clock_(new base::DefaultTickClock()),
       request_throttler_(
@@ skipping 63 matching lines @@
       // Wait until we get a refresh token.
       waiting_for_refresh_token_ = true;
       token_service_->AddObserver(this);
     }
   } else {
     // Not signed in: fetch snippets (without authentication).
     FetchSnippetsNonAuthenticated();
   }
 }
 
+NTPSnippetsFetcher::ContentLanguageInfo::ContentLanguageInfo()
+    : language_code(), frequency(0) {}
+
+void NTPSnippetsFetcher::ContentLanguageInfo::AppendToList(
+    base::ListValue* list) {
+  auto lang = base::MakeUnique<base::DictionaryValue>();
+  lang->SetString("language", language_code);
+  lang->SetDouble("frequency", frequency);
+  list->Append(std::move(lang));
+}
+
 NTPSnippetsFetcher::RequestParams::RequestParams()
     : fetch_api(),
       obfuscated_gaia_id(),
       only_return_personalized_results(),
       user_locale(),
       host_restricts(),
       count_to_fetch(),
       interactive_request() {}
 
 NTPSnippetsFetcher::RequestParams::~RequestParams() = default;
@@ skipping 63 matching lines @@
                                          : "BACKGROUND_PREFETCH");
 
       auto excluded = base::MakeUnique<base::ListValue>();
       for (const auto& id : excluded_ids) {
         excluded->AppendString(id);
         if (excluded->GetSize() >= kMaxExcludedIds)
           break;
       }
       request->Set("excludedSuggestionIds", std::move(excluded));
 
+      if (ui_language.frequency == 0 && other_top_language.frequency == 0)
+        break;
+
+      auto language_list = base::MakeUnique<base::ListValue>();
+      if (ui_language.frequency > 0)
+        ui_language.AppendToList(language_list.get());
+      if (other_top_language.frequency > 0)
+        other_top_language.AppendToList(language_list.get());
+      request->Set("top_languages", std::move(language_list));
+
       // TODO(sfiera): support authentication and personalization
       // TODO(sfiera): support count_to_fetch
       break;
     }
   }
 
   std::string request_json;
   bool success = base::JSONWriter::WriteWithOptions(
       *request, base::JSONWriter::OPTIONS_PRETTY_PRINT, &request_json);
   DCHECK(success);
@@ skipping 32 matching lines @@
   // Try to make fetching the files bit more robust even with poor connection.
   url_fetcher_->SetMaxRetriesOn5xx(3);
   url_fetcher_->Start();
 }
 
 bool NTPSnippetsFetcher::UsesAuthentication() const {
   return (personalization_ == Personalization::kPersonal ||
           personalization_ == Personalization::kBoth);
 }
 
+void NTPSnippetsFetcher::SetUpCommonFetchingParameters(
+    RequestParams* params) const {
+  params->fetch_api = fetch_api_;
+  params->host_restricts = hosts_;
+  params->user_locale = locale_;
+  params->excluded_ids = excluded_ids_;
+  params->count_to_fetch = count_to_fetch_;
+  params->interactive_request = interactive_request_;
+
+  // TODO(jkrcal): Add language model factory for iOS and add fakes to tests so
+  // that |language_model_| is never nullptr. Remove this check and add a DCHECK
+  // into the constructor.
+  if (!language_model_)
+    return;
+
+  params->ui_language.language_code = ISO639FromPosixLocale(locale_);
+  params->ui_language.frequency =
+      language_model_->GetLanguageFrequency(params->ui_language.language_code);
+
+  std::vector<LanguageModel::LanguageInfo> top_languages =
+      language_model_->GetTopLanguages();
+  for (const LanguageModel::LanguageInfo& info : top_languages) {
+    if (info.language_code != params->ui_language.language_code) {
+      params->other_top_language.language_code = info.language_code;
+      params->other_top_language.frequency = info.frequency;
+      break;
+    }
+  }
+}
+
 void NTPSnippetsFetcher::FetchSnippetsNonAuthenticated() {
   // When not providing OAuth token, we need to pass the Google API key.
   GURL url(base::StringPrintf(kSnippetsServerNonAuthorizedFormat,
                               fetch_url_.spec().c_str(), api_key_.c_str()));
 
   RequestParams params;
-  params.fetch_api = fetch_api_;
-  params.host_restricts = hosts_;
-  params.excluded_ids = excluded_ids_;
-  params.count_to_fetch = count_to_fetch_;
-  params.interactive_request = interactive_request_;
-  params.user_locale = locale_;
+  SetUpCommonFetchingParameters(&params);
   FetchSnippetsImpl(url, std::string(), params.BuildRequest());
 }
 
 void NTPSnippetsFetcher::FetchSnippetsAuthenticated(
     const std::string& account_id,
     const std::string& oauth_access_token) {
   RequestParams params;
-  params.fetch_api = fetch_api_;
+  SetUpCommonFetchingParameters(&params);
   params.obfuscated_gaia_id = account_id;
   params.only_return_personalized_results =
       personalization_ == Personalization::kPersonal;
-  params.user_locale = locale_;
-  params.host_restricts = hosts_;
-  params.excluded_ids = excluded_ids_;
-  params.count_to_fetch = count_to_fetch_;
-  params.interactive_request = interactive_request_;
   // TODO(jkrcal, treib): Add unit-tests for authenticated fetches.
   FetchSnippetsImpl(fetch_url_,
                     base::StringPrintf(kAuthorizationRequestHeaderFormat,
                                        oauth_access_token.c_str()),
                     params.BuildRequest());
 }
 
 void NTPSnippetsFetcher::StartTokenRequest() {
   OAuth2TokenService::ScopeSet scopes;
   scopes.insert(fetch_api_ == CHROME_CONTENT_SUGGESTIONS_API
@@ skipping 181 matching lines @@
   UMA_HISTOGRAM_ENUMERATION("NewTabPage.Snippets.FetchResult",
                             static_cast<int>(result),
                             static_cast<int>(FetchResult::RESULT_MAX));
 
   DVLOG(1) << "Fetch finished: " << last_status_;
   if (!snippets_available_callback_.is_null())
     snippets_available_callback_.Run(std::move(snippets));
 }
 
 }  // namespace ntp_snippets