Translate: split supporting language list handling to TranslateLanguageList

chrome/browser/translate/translate_manager.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "chrome/browser/translate/translate_manager.h"
 
 #include "base/bind.h"
 #include "base/command_line.h"
 #include "base/compiler_specific.h"
-#include "base/json/json_reader.h"
 #include "base/memory/singleton.h"
 #include "base/message_loop.h"
 #include "base/metrics/histogram.h"
 #include "base/prefs/pref_service.h"
 #include "base/string_util.h"
 #include "base/stringprintf.h"
 #include "base/strings/string_split.h"
 #include "base/time.h"
 #include "base/values.h"
 #include "chrome/browser/browser_process.h"
 #include "chrome/browser/infobars/infobar_service.h"
 #include "chrome/browser/profiles/profile.h"
 #include "chrome/browser/tab_contents/language_state.h"
 #include "chrome/browser/tab_contents/tab_util.h"
 #include "chrome/browser/translate/page_translated_details.h"
 #include "chrome/browser/translate/translate_infobar_delegate.h"
+#include "chrome/browser/translate/translate_language_list.h"
 #include "chrome/browser/translate/translate_manager_metrics.h"
 #include "chrome/browser/translate/translate_prefs.h"
 #include "chrome/browser/translate/translate_tab_helper.h"
+#include "chrome/browser/translate/translate_url_util.h"
 #include "chrome/browser/ui/browser.h"
 #include "chrome/browser/ui/browser_finder.h"
 #include "chrome/browser/ui/browser_tabstrip.h"
 #include "chrome/browser/ui/tabs/tab_strip_model.h"
 #include "chrome/common/chrome_constants.h"
 #include "chrome/common/chrome_notification_types.h"
 #include "chrome/common/chrome_switches.h"
 #include "chrome/common/language_detection_details.h"
 #include "chrome/common/pref_names.h"
 #include "chrome/common/render_messages.h"
@@ skipping 23 matching lines @@
 #include "chrome/browser/chromeos/extensions/file_manager/file_manager_util.h"
 #include "extensions/common/constants.h"
 #endif
 
 using content::NavigationController;
 using content::NavigationEntry;
 using content::WebContents;
 
 namespace {
 
-// The default list of languages the Google translation server supports.
-// We use this list until we receive the list that the server exposes.
-// For information, here is the list of languages that Chrome can be run in
-// but that the translation server does not support:
-// am Amharic
-// bn Bengali
-// gu Gujarati
-// kn Kannada
-// ml Malayalam
-// mr Marathi
-// ta Tamil
-// te Telugu
-const char* const kDefaultSupportedLanguages[] = {
-    "af",     // Afrikaans
-    "sq",     // Albanian
-    "ar",     // Arabic
-    "be",     // Belarusian
-    "bg",     // Bulgarian
-    "ca",     // Catalan
-    "zh-CN",  // Chinese (Simplified)
-    "zh-TW",  // Chinese (Traditional)
-    "hr",     // Croatian
-    "cs",     // Czech
-    "da",     // Danish
-    "nl",     // Dutch
-    "en",     // English
-    "eo",     // Esperanto
-    "et",     // Estonian
-    "tl",     // Filipino
-    "fi",     // Finnish
-    "fr",     // French
-    "gl",     // Galician
-    "de",     // German
-    "el",     // Greek
-    "ht",     // Haitian Creole
-    "iw",     // Hebrew
-    "hi",     // Hindi
-    "hu",     // Hungarian
-    "is",     // Icelandic
-    "id",     // Indonesian
-    "ga",     // Irish
-    "it",     // Italian
-    "ja",     // Japanese
-    "ko",     // Korean
-    "lv",     // Latvian
-    "lt",     // Lithuanian
-    "mk",     // Macedonian
-    "ms",     // Malay
-    "mt",     // Maltese
-    "no",     // Norwegian
-    "fa",     // Persian
-    "pl",     // Polish
-    "pt",     // Portuguese
-    "ro",     // Romanian
-    "ru",     // Russian
-    "sr",     // Serbian
-    "sk",     // Slovak
-    "sl",     // Slovenian
-    "es",     // Spanish
-    "sw",     // Swahili
-    "sv",     // Swedish
-    "th",     // Thai
-    "tr",     // Turkish
-    "uk",     // Ukrainian
-    "vi",     // Vietnamese
-    "cy",     // Welsh
-    "yi",     // Yiddish
-};
-
 const char kTranslateScriptURL[] =
     "https://translate.google.com/translate_a/element.js";
 const char kTranslateScriptHeader[] = "Google-Translate-Element-Mode: library";
 const char kReportLanguageDetectionErrorURL[] =
     "https://translate.google.com/translate_error?client=cr&action=langidc";
-const char kLanguageListFetchURL[] =
-    "https://translate.googleapis.com/translate_a/l?client=chrome&cb=sl";
-
-// Used in kTranslateScriptURL to request supporting languages list including
-// "alpha languages".
-const char kAlphaLanguageQueryName[] = "alpha";
-const char kAlphaLanguageQueryValue[] = "1";
-
-// Used in all translate URLs to specify API Key.
-const char kApiKeyName[] = "key";
 
 // Used in kTranslateScriptURL to specify a callback function name.
 const char kCallbackQueryName[] = "cb";
 const char kCallbackQueryValue[] =
     "cr.googleTranslate.onTranslateElementLoad";
 
-// Used in kTranslateScriptURL and kLanguageListFetchURL to specify the
-// application locale.
-const char kHostLocaleQueryName[] = "hl";
-
 // Used in kReportLanguageDetectionErrorURL to specify the original page
 // language.
 const char kSourceLanguageQueryName[] = "sl";
 
 // Used in kReportLanguageDetectionErrorURL to specify the page URL.
 const char kUrlQueryName[] = "u";
 
 // The delay in ms that we'll wait to check if a page has finished loading
 // before attempting a translation.
 const int kTranslateLoadCheckDelayMs = 150;
 
 // The maximum number of attempts we'll do to see if the page has finshed
 // loading before giving up the translation
 const int kMaxTranslateLoadCheckAttempts = 20;
 
-const int kMaxRetryLanguageListFetch = 5;
 const int kTranslateScriptExpirationDelayDays = 1;
 
-GURL AddApiKeyToUrl(const GURL& url) {
-  return net::AppendQueryParameter(url, kApiKeyName, google_apis::GetAPIKey());
-}
-
-GURL AddHostLocaleToUrl(const GURL& url) {
-  return net::AppendQueryParameter(
-      url,
-      kHostLocaleQueryName,
-      TranslateManager::GetLanguageCode(
-          g_browser_process->GetApplicationLocale()));
-}
-
 }  // namespace
 
-// This must be kept in sync with the &cb= value in the kLanguageListFetchURL.
-const char TranslateManager::kLanguageListCallbackName[] = "sl(";
-const char TranslateManager::kTargetLanguagesKey[] = "tl";
-
-// static
-base::LazyInstance<std::set<std::string> >
-    TranslateManager::supported_languages_ = LAZY_INSTANCE_INITIALIZER;
-
 TranslateManager::~TranslateManager() {
   weak_method_factory_.InvalidateWeakPtrs();
 }
 
 // static
 TranslateManager* TranslateManager::GetInstance() {
   return Singleton<TranslateManager>::get();
 }
 
 // static
 bool TranslateManager::IsTranslatableURL(const GURL& url) {
   // A URLs is translatable unless it is one of the following:
   // - empty (can happen for popups created with window.open(""))
   // - an internal URL (chrome:// and others)
   // - the devtools (which is considered UI)
   // - Chrome OS file manager extension
   // - an FTP page (as FTP pages tend to have long lists of filenames that may
   //   confuse the CLD)
   return !url.is_empty() &&
          !url.SchemeIs(chrome::kChromeUIScheme) &&
          !url.SchemeIs(chrome::kChromeDevToolsScheme) &&
 #ifdef FILE_MANAGER_EXTENSION
          !(url.SchemeIs(extensions::kExtensionScheme) &&
            url.DomainIs(kFileBrowserDomain)) &&
 #endif
          !url.SchemeIs(chrome::kFtpScheme);
 }
 
 // static
-void TranslateManager::SetSupportedLanguages(const std::string& language_list) {
-  // The format is:
-  // sl({"sl": {"XX": "LanguageName", ...}, "tl": {"XX": "LanguageName", ...}})
-  // Where "sl(" is set in kLanguageListCallbackName
-  // and "tl" is kTargetLanguagesKey
-  if (!StartsWithASCII(language_list, kLanguageListCallbackName, false) ||
-      !EndsWith(language_list, ")", false)) {
-    // We don't have a NOTREACHED here since this can happen in ui_tests, even
-    // though the the BrowserMain function won't call us with parameters.ui_task
-    // is NULL some tests don't set it, so we must bail here.
-    return;
-  }
-  static const size_t kLanguageListCallbackNameLength =
-      strlen(kLanguageListCallbackName);
-  std::string languages_json = language_list.substr(
-      kLanguageListCallbackNameLength,
-      language_list.size() - kLanguageListCallbackNameLength - 1);
-  scoped_ptr<Value> json_value(
-      base::JSONReader::Read(languages_json, base::JSON_ALLOW_TRAILING_COMMAS));
-  if (json_value == NULL || !json_value->IsType(Value::TYPE_DICTIONARY)) {
-    NOTREACHED();
-    return;
-  }
-  // The first level dictionary contains two sub-dict, one for source languages
-  // and the other for target languages, we want to use the target languages.
-  DictionaryValue* language_dict =
-      static_cast<DictionaryValue*>(json_value.get());
-  DictionaryValue* target_languages = NULL;
-  if (!language_dict->GetDictionary(kTargetLanguagesKey, &target_languages) ||
-      target_languages == NULL) {
-    NOTREACHED();
-    return;
-  }
-  // Now we can clear our current state...
-  std::set<std::string>* supported_languages = supported_languages_.Pointer();
-  supported_languages->clear();
-  // ... and replace it with the values we just fetched from the server.
-  for (DictionaryValue::Iterator iter(*target_languages); !iter.IsAtEnd();
-       iter.Advance()) {
-    supported_languages_.Pointer()->insert(iter.key());
-  }
-}
-
-// static
-void TranslateManager::InitSupportedLanguages() {
-  // If our list of supported languages have not been set yet, we default
-  // to our hard coded list of languages in kDefaultSupportedLanguages.
-  if (supported_languages_.Pointer()->empty()) {
-    for (size_t i = 0; i < arraysize(kDefaultSupportedLanguages); ++i)
-      supported_languages_.Pointer()->insert(kDefaultSupportedLanguages[i]);
-  }
-}
-
-// static
 void TranslateManager::GetSupportedLanguages(
     std::vector<std::string>* languages) {
-  DCHECK(languages && languages->empty());
-  InitSupportedLanguages();
-  std::set<std::string>* supported_languages = supported_languages_.Pointer();
-  std::set<std::string>::const_iterator iter = supported_languages->begin();
-  for (; iter != supported_languages->end(); ++iter)
-    languages->push_back(*iter);
+  if (GetInstance()->language_list_.get()) {
+    GetInstance()->language_list_->GetSupportedLanguages(languages);
+    return;
+  }
+  NOTREACHED();
 }
 
 // static
 std::string TranslateManager::GetLanguageCode(
     const std::string& chrome_locale) {
-  // Only remove the country code for country specific languages we don't
-  // support specifically yet.
-  if (IsSupportedLanguage(chrome_locale))
-    return chrome_locale;
-
-  size_t hypen_index = chrome_locale.find('-');
-  if (hypen_index == std::string::npos)
-    return chrome_locale;
-  return chrome_locale.substr(0, hypen_index);
+  if (GetInstance()->language_list_.get())
+    return GetInstance()->language_list_->GetLanguageCode(chrome_locale);
+  NOTREACHED();
+  return chrome_locale;
 }
 
 // static
-bool TranslateManager::IsSupportedLanguage(const std::string& page_language) {
-  InitSupportedLanguages();
-  return supported_languages_.Pointer()->count(page_language) != 0;
+bool TranslateManager::IsSupportedLanguage(const std::string& language) {
+  if (GetInstance()->language_list_.get())
+    return GetInstance()->language_list_->IsSupportedLanguage(language);
+  NOTREACHED();
+  return false;
 }
 
 void TranslateManager::Observe(int type,
                                const content::NotificationSource& source,
                                const content::NotificationDetails& details) {
   switch (type) {
     case content::NOTIFICATION_NAV_ENTRY_COMMITTED: {
       NavigationController* controller =
           content::Source<NavigationController>(source).ptr();
       content::LoadCommittedDetails* load_details =
@@ skipping 95 matching lines @@
       DCHECK(count == 1u);
       delete pref_change_registrar;
       break;
     }
     default:
       NOTREACHED();
   }
 }
 
 void TranslateManager::OnURLFetchComplete(const net::URLFetcher* source) {
-  if (translate_script_request_pending_.get() != source &&
-      language_list_request_pending_.get() != source) {
+  if (translate_script_request_pending_.get() != source) {
     // Looks like crash on Mac is possibly caused with callback entering here
     // with unknown fetcher when network is refreshed.
     scoped_ptr<const net::URLFetcher> delete_ptr(source);
     return;
   }
 
   bool error =
-      (source->GetStatus().status() != net::URLRequestStatus::SUCCESS ||
-      source->GetResponseCode() != net::HTTP_OK);
+      source->GetStatus().status() != net::URLRequestStatus::SUCCESS ||
+      source->GetResponseCode() != net::HTTP_OK;
   if (translate_script_request_pending_.get() == source) {
     scoped_ptr<const net::URLFetcher> delete_ptr(
         translate_script_request_pending_.release());
     if (!error) {
       base::StringPiece str = ResourceBundle::GetSharedInstance().
           GetRawDataResource(IDR_TRANSLATE_JS);
       DCHECK(translate_script_.empty());
       str.CopyToString(&translate_script_);
       std::string argument = "('";
       std::string api_key = google_apis::GetAPIKey();
@@ skipping 41 matching lines @@
             ShortcutConfig(),
             request.source_lang,
             request.target_lang);
       } else {
         // Translate the page.
         DoTranslatePage(web_contents, translate_script_,
                         request.source_lang, request.target_lang);
       }
     }
     pending_requests_.clear();
-  } else {  // if (translate_script_request_pending_.get() == source)
-    scoped_ptr<const net::URLFetcher> delete_ptr(
-        language_list_request_pending_.release());
-    if (!error) {
-      std::string data;
-      source->GetResponseAsString(&data);
-      SetSupportedLanguages(data);
-    } else {
-      VLOG(9) << "Failed to Fetch languages from: " << kLanguageListFetchURL;
-    }
   }
 }
 
 void TranslateManager::AddObserver(Observer* obs) {
   observer_list_.AddObserver(obs);
 }
 
 void TranslateManager::RemoveObserver(Observer* obs) {
   observer_list_.RemoveObserver(obs);
 }
 
 void TranslateManager::NotifyLanguageDetection(
     const LanguageDetectionDetails& details) {
   FOR_EACH_OBSERVER(Observer, observer_list_, OnLanguageDetection(details));
 }
 
 
 TranslateManager::TranslateManager()
   : weak_method_factory_(this),
     translate_script_expiration_delay_(base::TimeDelta::FromDays(
         kTranslateScriptExpirationDelayDays)),
     max_reload_check_attempts_(kMaxTranslateLoadCheckAttempts) {
   notification_registrar_.Add(this, content::NOTIFICATION_NAV_ENTRY_COMMITTED,
                               content::NotificationService::AllSources());
   notification_registrar_.Add(this,
                               chrome::NOTIFICATION_TAB_LANGUAGE_DETERMINED,
                               content::NotificationService::AllSources());
   notification_registrar_.Add(this, chrome::NOTIFICATION_PAGE_TRANSLATED,
                               content::NotificationService::AllSources());
+  language_list_.reset(new TranslateLanguageList);
 }
 
 void TranslateManager::InitiateTranslation(WebContents* web_contents,
                                            const std::string& page_lang) {
   Profile* profile =
       Profile::FromBrowserContext(web_contents->GetBrowserContext());
   PrefService* prefs = profile->GetOriginalProfile()->GetPrefs();
   if (!prefs->GetBoolean(prefs::kEnableTranslate)) {
     TranslateManagerMetrics::ReportInitiationStatus(
         TranslateManagerMetrics::INITIATION_STATUS_DISABLED_BY_PREFS);
@@ skipping 200 matching lines @@
       kUrlQueryName,
       page_url.spec());
 
   TranslateTabHelper* translate_tab_helper =
       TranslateTabHelper::FromWebContents(web_contents);
   report_error_url = net::AppendQueryParameter(
       report_error_url,
       kSourceLanguageQueryName,
       translate_tab_helper->language_state().original_language());
 
-  report_error_url = AddHostLocaleToUrl(report_error_url);
-  report_error_url = AddApiKeyToUrl(report_error_url);
+  report_error_url = TranslateURLUtil::AddHostLocaleToUrl(report_error_url);
+  report_error_url = TranslateURLUtil::AddApiKeyToUrl(report_error_url);
 
   chrome::AddSelectedTabWithURL(browser, report_error_url,
                                 content::PAGE_TRANSITION_AUTO_BOOKMARK);
 }
 
 void TranslateManager::DoTranslatePage(WebContents* web_contents,
                                        const std::string& translate_script,
                                        const std::string& source_lang,
                                        const std::string& target_lang) {
   NavigationEntry* entry = web_contents->GetController().GetActiveEntry();
@@ skipping 95 matching lines @@
     // the default Accept-Language values for most locales, remove this
     // special-casing.
     if (accept_lang != "en" || is_ui_english)
       accept_langs_set.insert(accept_lang);
   }
   accept_languages_[prefs] = accept_langs_set;
 }
 
 void TranslateManager::FetchLanguageListFromTranslateServer(
     PrefService* prefs) {
-  if (language_list_request_pending_.get() != NULL)
-    return;
+  DCHECK(language_list_.get());
 
   // We don't want to do this when translate is disabled.
   DCHECK(prefs != NULL);
   if (CommandLine::ForCurrentProcess()->HasSwitch(
       switches::kDisableTranslate) ||
       (prefs != NULL && !prefs->GetBoolean(prefs::kEnableTranslate))) {
     return;
   }
 
-  GURL language_list_fetch_url = GURL(kLanguageListFetchURL);
-  language_list_fetch_url = AddHostLocaleToUrl(language_list_fetch_url);
-  language_list_fetch_url = AddApiKeyToUrl(language_list_fetch_url);
-
-  const CommandLine& command_line = *CommandLine::ForCurrentProcess();
-  if (command_line.HasSwitch(switches::kEnableTranslateAlphaLanguages)) {
-    language_list_fetch_url = net::AppendQueryParameter(
-        language_list_fetch_url,
-        kAlphaLanguageQueryName,
-        kAlphaLanguageQueryValue);
-  }
-
-  VLOG(9) << "Fetch supporting language list from: "
-          << language_list_fetch_url.spec().c_str();
-
-  language_list_request_pending_.reset(net::URLFetcher::Create(
-      1, language_list_fetch_url, net::URLFetcher::GET, this));
-  language_list_request_pending_->SetLoadFlags(net::LOAD_DO_NOT_SEND_COOKIES |
-                                               net::LOAD_DO_NOT_SAVE_COOKIES);
-  language_list_request_pending_->SetRequestContext(
-      g_browser_process->system_request_context());
-  language_list_request_pending_->SetMaxRetriesOn5xx(
-      kMaxRetryLanguageListFetch);
-  language_list_request_pending_->Start();
+  if (language_list_.get())
+    language_list_->RequestLanguageList();
 }
 
 void TranslateManager::CleanupPendingUlrFetcher() {
-  language_list_request_pending_.reset();
+  language_list_.reset();
   translate_script_request_pending_.reset();
 }
 
 void TranslateManager::RequestTranslateScript() {
   if (translate_script_request_pending_.get() != NULL)
     return;
 
   GURL translate_script_url;
   // Check if command-line contains an alternative URL for translate service.
   const CommandLine& command_line = *CommandLine::ForCurrentProcess();
@@ skipping 11 matching lines @@
   // Use default URL when command-line argument is not specified, or specified
   // URL is invalid.
   if (translate_script_url.is_empty())
     translate_script_url = GURL(kTranslateScriptURL);
 
   translate_script_url = net::AppendQueryParameter(
       translate_script_url,
       kCallbackQueryName,
       kCallbackQueryValue);
 
-  translate_script_url = AddHostLocaleToUrl(translate_script_url);
-  translate_script_url = AddApiKeyToUrl(translate_script_url);
+  translate_script_url =
+      TranslateURLUtil::AddHostLocaleToUrl(translate_script_url);
+  translate_script_url =
+      TranslateURLUtil::AddApiKeyToUrl(translate_script_url);
 
   translate_script_request_pending_.reset(net::URLFetcher::Create(
       0, translate_script_url, net::URLFetcher::GET, this));
   translate_script_request_pending_->SetLoadFlags(
       net::LOAD_DO_NOT_SEND_COOKIES | net::LOAD_DO_NOT_SAVE_COOKIES);
   translate_script_request_pending_->SetRequestContext(
       g_browser_process->system_request_context());
   translate_script_request_pending_->SetExtraRequestHeaders(
       kTranslateScriptHeader);
   translate_script_request_pending_->Start();
 }
 
 // static
 std::string TranslateManager::GetTargetLanguage(PrefService* prefs) {
   std::string ui_lang =
-    GetLanguageCode(g_browser_process->GetApplicationLocale());
+      GetLanguageCode(g_browser_process->GetApplicationLocale());
   if (IsSupportedLanguage(ui_lang))
     return ui_lang;
 
   // Getting the accepted languages list
   std::string accept_langs_str = prefs->GetString(prefs::kAcceptLanguages);
 
   std::vector<std::string> accept_langs_list;
   base::SplitString(accept_langs_str, ',', &accept_langs_list);
 
   // Will translate to the first supported language on the Accepted Language
@@ skipping 16 matching lines @@
   // reason so we are more aggressive showing the shortcuts for never translate.
   #if defined(OS_ANDROID)
   config.never_translate_min_count = 1;
   #else
   config.never_translate_min_count = 3;
   #endif  // defined(OS_ANDROID)
 
   config.always_translate_min_count = 3;
   return config;
 }