Switch to using the new Link Doctor API.

chrome/renderer/net/net_error_helper_core.cc

 // Copyright 2013 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "chrome/renderer/net/net_error_helper_core.h"
 
 #include <string>
 
+#include "base/i18n/rtl.h"
+#include "base/json/json_reader.h"
+#include "base/json/json_writer.h"
 #include "base/metrics/histogram.h"
+#include "base/strings/string16.h"
+#include "base/values.h"
 #include "chrome/common/localized_error.h"
+#include "grit/generated_resources.h"
 #include "net/base/escape.h"
 #include "net/base/net_errors.h"
+#include "net/base/net_util.h"
 #include "third_party/WebKit/public/platform/WebString.h"
 #include "third_party/WebKit/public/platform/WebURLError.h"
+#include "ui/base/l10n/l10n_util.h"
 #include "url/gurl.h"
 
 namespace {
 
+struct CorrectionTypeToResourceTable {
+  int resource_id;
+  const char* correction_type;
+};
+
+const CorrectionTypeToResourceTable kCorrectionResourceTable[] = {
+  {IDS_ERRORPAGES_SUGGESTION_VISIT_GOOGLE_CACHE, "cachedPage"},
+  // "reloadPage" is has special handling.
+  {IDS_ERRORPAGES_SUGGESTION_CORRECTED_URL, "urlCorrection"},
+  {IDS_ERRORPAGES_SUGGESTION_ALTERNATE_URL, "siteDomain"},
+  {IDS_ERRORPAGES_SUGGESTION_ALTERNATE_URL, "host"},
+  {IDS_ERRORPAGES_SUGGESTION_ALTERNATE_URL, "sitemap"},
+  {IDS_ERRORPAGES_SUGGESTION_ALTERNATE_URL, "pathParentFolder"},
+  // "siteSearchQuery" is not yet supported.
+  // TODO(mmenke):  Figure out what format "siteSearchQuery" uses for its
+  // suggestions.
+  // "webSearchQuery" has special handling.
+  {IDS_ERRORPAGES_SUGGESTION_ALTERNATE_URL, "contentOverlap"},
+  {IDS_ERRORPAGES_SUGGESTION_CORRECTED_URL, "emphasizedUrlCorrection"},
+};
+
 // Returns whether |net_error| is a DNS-related error (and therefore whether
 // the tab helper should start a DNS probe after receiving it.)
 bool IsDnsError(const blink::WebURLError& error) {
   return error.domain.utf8() == net::kErrorDomain &&
          (error.reason == net::ERR_NAME_NOT_RESOLVED ||
           error.reason == net::ERR_NAME_RESOLUTION_FAILED);
 }
 
-// If an alternate error page should be retrieved remotely for a main frame load
-// that failed with |error|, returns true and sets |error_page_url| to the URL
-// of the remote error page.
-bool GetErrorPageURL(const blink::WebURLError& error,
-                     const GURL& alt_error_page_url,
-                     GURL* error_page_url) {
-  if (!alt_error_page_url.is_valid())
-    return false;
+GURL SanitizeURL(const GURL& url) {
+  GURL::Replacements remove_params;
+  remove_params.ClearUsername();
+  remove_params.ClearPassword();
+  remove_params.ClearQuery();
+  remove_params.ClearRef();
+  return url.ReplaceComponents(remove_params);
+}
 
+// If URL correction information should be retrieved remotely for a main frame
+// load that failed with |error|, returns true and sets
+// |fix_url_request_body| to be the body for the correcion request.
+bool GetFixUrlRequestBody(const blink::WebURLError& error,
+                          const std::string& language,
+                          const std::string& country_code,
+                          const std::string& api_key,
+                          std::string* fix_url_request_body) {
   // Parameter to send to the error page indicating the error type.
   std::string error_param;
 
   std::string domain = error.domain.utf8();
   if (domain == "http" && error.reason == 404) {
     error_param = "http404";
   } else if (IsDnsError(error)) {
     error_param = "dnserror";
   } else if (domain == net::kErrorDomain &&
              (error.reason == net::ERR_CONNECTION_FAILED ||
               error.reason == net::ERR_CONNECTION_REFUSED ||
               error.reason == net::ERR_ADDRESS_UNREACHABLE ||
               error.reason == net::ERR_CONNECTION_TIMED_OUT)) {
-    error_param = "connectionfailure";
+    error_param = "connectionFailure";
   } else {
     return false;
   }
 
   // Don't use the Link Doctor for HTTPS (for privacy reasons).
   GURL unreachable_url(error.unreachableURL);
   if (unreachable_url.SchemeIsSecure())
     return false;
 
-  // Sanitize the unreachable URL.
-  GURL::Replacements remove_params;
-  remove_params.ClearUsername();
-  remove_params.ClearPassword();
-  remove_params.ClearQuery();
-  remove_params.ClearRef();
-  // TODO(yuusuke): change to net::FormatUrl when Link Doctor becomes
+  // TODO(yuusuke): Change to net::FormatUrl when Link Doctor becomes
   // unicode-capable.
-  std::string spec_to_send =
-      unreachable_url.ReplaceComponents(remove_params).spec();
+  std::string spec_to_send = SanitizeURL(unreachable_url).spec();
 
   // Notify Link Doctor of the url truncation by sending of "?" at the end.
   if (unreachable_url.has_query())
     spec_to_send.append("?");
 
-  std::string params(alt_error_page_url.query());
-  params.append("&url=");
-  params.append(net::EscapeQueryParamValue(spec_to_send, true));
-  params.append("&sourceid=chrome");
-  params.append("&error=");
-  params.append(error_param);
+  // Assemble request body, which is a JSON string.
+  // TODO(mmenke):  Investigate open sourcing the relevant protocol buffers and
+  //                using those directly instead.
 
-  // Build the final url to request.
-  GURL::Replacements link_doctor_params;
-  link_doctor_params.SetQueryStr(params);
-  *error_page_url = alt_error_page_url.ReplaceComponents(link_doctor_params);
+  base::DictionaryValue request_dict;
+  request_dict.SetString("method", "linkdoctor.fixurl.fixurl");
+  request_dict.SetString("apiVersion", "v1");
+
+  base::DictionaryValue* params_dict = new base::DictionaryValue();
+  request_dict.Set("params", params_dict);
+
+  params_dict->SetString("key", api_key);
+  params_dict->SetString("urlQuery", spec_to_send);
+  params_dict->SetString("clientName", "chrome");
+  params_dict->SetString("error", error_param);
+
+  if (!language.empty())
+    params_dict->SetString("language", language);
+
+  // TODO(mmenke):  Figure out if the originCountry can be set correctly.
+  //                locale.getCountry() just seems to give the default country
+  //                for the language resources currently in use.
+
+  if (!country_code.empty())
+    params_dict->SetString("originCountry", country_code);
+
+  base::JSONWriter::Write(&request_dict, fix_url_request_body);
   return true;
 }
 
+base::string16 FormatURLForDisplay(const GURL& url, bool is_rtl,
+                                   const std::string accept_languages) {
+  // Translate punycode into UTF8, unescape UTF8 URLs.
+  base::string16 url_for_display(net::FormatUrl(
+      url, accept_languages, net::kFormatUrlOmitNothing,
+      net::UnescapeRule::NORMAL, NULL, NULL, NULL));
+  // URLs are always LTR.
+  if (is_rtl)
+    base::i18n::WrapStringWithLTRFormatting(&url_for_display);
+  return url_for_display;
+}
+
+LocalizedError::ErrorPageParams* ParseAdditionalSuggestions(
+    const std::string& data,
+    const GURL& original_url,
+    const GURL& search_url,
+    const std::string& accept_languages,
+    bool is_rtl) {
+  scoped_ptr<base::Value> parsed(base::JSONReader::Read(data));
+  if (!parsed)
+    return NULL;
+  // TODO(mmenke):  Open source related protocol buffers and use them directly.
+  base::DictionaryValue* parsed_dict;
+  base::ListValue* corrections;
+  if (!parsed->GetAsDictionary(&parsed_dict))
+    return NULL;
+  if (!parsed_dict->GetList("result.UrlCorrections", &corrections))
+    return NULL;
+
+  // Version of URL for display in suggestions.  It has to be sanitized first
+  // because any received suggestions will be relative to the sanitized URL.
+  base::string16 original_url_for_display =
+      FormatURLForDisplay(SanitizeURL(original_url), is_rtl, accept_languages);
+
+  scoped_ptr<LocalizedError::ErrorPageParams> params(
+      new LocalizedError::ErrorPageParams());
+  params->override_suggestions.reset(new base::ListValue());
+  scoped_ptr<base::ListValue> parsed_corrections(new base::ListValue());
+  for (base::ListValue::iterator it = corrections->begin();
+       it != corrections->end(); ++it) {
+    base::DictionaryValue* correction;
+    if (!(*it)->GetAsDictionary(&correction))
+      continue;
+
+    // Doesn't seem like a good idea to show these.

[Deprecated (see juliatuttle), 2014/02/04 17:44:21]

What if that's what the user was looking for?  (Is the FixURL server clever
enough to return (soft) porn results only when the original URL indicated that
the user was looking for that?)

[mmenke, 2014/02/04 19:43:56]

In my experience, FixURL isn't really that clever.  Since Chrome has no safe
search settings, I think we should just exclude this for now.  If we really show
porn links, should talk to Chris and the UI guys about it.

[Deprecated (see juliatuttle), 2014/02/04 20:20:20]

Alright.  I'll tinker with it and see how clever it is.  I'd like it if we could
end up typo-correcting all sites if possible.

[mmenke, 2014/02/04 20:31:09]

May also want to see if we're already excluding porn links.  If we're currently
displaying them, then I'd have less concern about removing the checks, since
we'd be matching current behavior.

+    bool is_porn;
+    if (correction->GetBoolean("isPorn", &is_porn) && is_porn)
+      continue;
+    if (correction->GetBoolean("isSoftPorn", &is_porn) && is_porn)
+      continue;
+
+    std::string correction_type;
+    std::string url_correction;
+    if (!correction->GetString("correctionType", &correction_type) ||
+        !correction->GetString("urlCorrection", &url_correction)) {
+      continue;
+    }
+
+    std::string click_tracking_url;
+    correction->GetString("clickTrackingUrl", &click_tracking_url);
+
+    if (correction_type == "reloadPage") {
+      params->suggest_reload = true;
+      continue;
+    }
+
+    if (correction_type == "webSearchQuery") {
+      // If there are mutliple searches suggested, use the first suggestion.
+      if (params->search_terms.empty()) {
+        params->search_url = search_url;
+        params->search_terms = url_correction;
+      }
+      continue;
+    }
+
+    size_t correction_index;
+    for (correction_index = 0;
+         correction_index < arraysize(kCorrectionResourceTable);
+         ++correction_index) {
+      if (correction_type !=
+              kCorrectionResourceTable[correction_index].correction_type) {
+        continue;
+      }
+      base::DictionaryValue* suggest = new base::DictionaryValue();
+      suggest->SetString("header",
+          l10n_util::GetStringUTF16(
+              kCorrectionResourceTable[correction_index].resource_id));
+      suggest->SetString("urlCorrection",
+          !click_tracking_url.empty() ? click_tracking_url :
+                                        url_correction);
+      suggest->SetString(
+          "urlCorrectionForDisplay",
+          FormatURLForDisplay(GURL(url_correction), is_rtl, accept_languages));
+      suggest->SetString("originalUrlForDisplay", original_url_for_display);
+      params->override_suggestions->Append(suggest);
+      break;
+    }
+  }
+
+  if (params->override_suggestions->empty() &&
+      !params->search_url.is_valid()) {
+    return NULL;
+  }
+  return params.release();
+}
+
 }  // namespace
 
 struct NetErrorHelperCore::ErrorPageInfo {
   ErrorPageInfo(blink::WebURLError error, bool was_failed_post)
       : error(error),
         was_failed_post(was_failed_post),
         needs_dns_updates(false),
         is_finished_loading(false) {
   }
 
   // Information about the failed page load.
   blink::WebURLError error;
   bool was_failed_post;
 
   // Information about the status of the error page.
 
   // True if a page is a DNS error page and has not yet received a final DNS
   // probe status.
   bool needs_dns_updates;
 
-  // URL of an alternate error page to repace this error page with, if it's a
-  // valid URL.  Request will be issued when the error page finishes loading.
-  // This is done on load complete to ensure that there are two complete loads
-  // for tests to wait for.
-  GURL alternate_error_page_url;
+  // URL of the FixURL service, which will be used to request suggestions on
+  // certain types of network errors.  This is also stored by the
+  // NetErrorHelperCore itself, but it stored here as well in case its modified
+  // in the middle of an error page load.  Empty when no error page should be
+  // fetched, or if there's already a fetch in progress.
+  GURL link_doctor_url;
+
+  // Request body to use when requesting suggestions from the FixURL service.
+  // TODO(mmenke):  Investigate loading the error page at the same time as
+  //                the blank page is loading, to get rid of these.
+  std::string link_doctor_request_body;
 
   // True if a page has completed loading, at which point it can receive
   // updates.
   bool is_finished_loading;
 };
 
 NetErrorHelperCore::NetErrorHelperCore(Delegate* delegate)
     : delegate_(delegate),
       last_probe_status_(chrome_common_net::DNS_PROBE_POSSIBLE) {
 }
 
 NetErrorHelperCore::~NetErrorHelperCore() {
 }
 
 void NetErrorHelperCore::OnStop() {
   // On stop, cancel loading the alternate error page, and prevent any pending
   // error page load from starting a new error page load.  Swapping in the error
   // page when it's finished loading could abort the navigation, otherwise.
-  if (committed_error_page_info_)
-    committed_error_page_info_->alternate_error_page_url = GURL();
-  if (pending_error_page_info_)
-    pending_error_page_info_->alternate_error_page_url = GURL();
+  if (committed_error_page_info_) {
+    committed_error_page_info_->link_doctor_url = GURL();
+    committed_error_page_info_->link_doctor_request_body.clear();
+  }
+  if (pending_error_page_info_) {
+    pending_error_page_info_->link_doctor_url = GURL();
+    pending_error_page_info_->link_doctor_request_body.clear();
+  }
   delegate_->CancelFetchErrorPage();
 }
 
 void NetErrorHelperCore::OnStartLoad(FrameType frame_type, PageType page_type) {
   if (frame_type != MAIN_FRAME)
     return;
 
   // If there's no pending error page information associated with the page load,
   // or the new page is not an error page, then reset pending error page state.
   if (!pending_error_page_info_ || page_type != ERROR_PAGE) {
     OnStop();
   }
 }
 
 void NetErrorHelperCore::OnCommitLoad(FrameType frame_type) {
   if (frame_type != MAIN_FRAME)
     return;
 
   committed_error_page_info_.reset(pending_error_page_info_.release());
 }
 
 void NetErrorHelperCore::OnFinishLoad(FrameType frame_type) {
   if (frame_type != MAIN_FRAME || !committed_error_page_info_)
     return;
 
   committed_error_page_info_->is_finished_loading = true;
 
-  if (committed_error_page_info_->alternate_error_page_url.is_valid()) {
-    // If there is another pending error page load,
-    // |replace_with_alternate_error_page| should have been set to false.
+  if (committed_error_page_info_->link_doctor_url.is_valid()) {
+    // If there is another pending error page load, |fix_url| should have been
+    // cleared.
     DCHECK(!pending_error_page_info_);
     DCHECK(!committed_error_page_info_->needs_dns_updates);
-    GURL error_page_url;
     delegate_->FetchErrorPage(
-        committed_error_page_info_->alternate_error_page_url);
+        committed_error_page_info_->link_doctor_url,
+        committed_error_page_info_->link_doctor_request_body);
   }
 
   if (!committed_error_page_info_->needs_dns_updates ||
       last_probe_status_ == chrome_common_net::DNS_PROBE_POSSIBLE) {
     return;
   }
   DVLOG(1) << "Error page finished loading; sending saved status.";
   UpdateErrorPage();
 }
 
 void NetErrorHelperCore::GetErrorHTML(
     FrameType frame_type,
     const blink::WebURLError& error,
     bool is_failed_post,
     std::string* error_html) {
   if (frame_type == MAIN_FRAME) {
     // If an alternate error page was going to be fetched, that should have been
     // cancelled by loading a new page load (Which has now failed to load).
     DCHECK(!committed_error_page_info_ ||
-           !committed_error_page_info_->alternate_error_page_url.is_valid());
+           !committed_error_page_info_->link_doctor_url.is_valid());
+
+    std::string link_doctor_request_body;
+
+    if (link_doctor_url_.is_valid() &&
+        GetFixUrlRequestBody(error, language_, country_code_, api_key_,
+                             &link_doctor_request_body)) {
+      pending_error_page_info_.reset(new ErrorPageInfo(error, is_failed_post));
+      pending_error_page_info_->link_doctor_url = link_doctor_url_;
+      pending_error_page_info_->link_doctor_request_body =
+          link_doctor_request_body;
+      return;
+    }
 
     // The last probe status needs to be reset if this is a DNS error.  This
     // means that if a DNS error page is committed but has not yet finished
     // loading, a DNS probe status scheduled to be sent to it may be thrown
     // out, but since the new error page should trigger a new DNS probe, it
     // will just get the results for the next page load.
     if (IsDnsError(error))
       last_probe_status_ = chrome_common_net::DNS_PROBE_POSSIBLE;
-
-    GURL error_page_url;
-    if (GetErrorPageURL(error, alt_error_page_url_, &error_page_url)) {
-      pending_error_page_info_.reset(new ErrorPageInfo(error, is_failed_post));
-      pending_error_page_info_->alternate_error_page_url = error_page_url;
-      return;
-    }
   }
 
-  GenerateLocalErrorPage(frame_type, error, is_failed_post, error_html);
+  GenerateLocalErrorPage(frame_type, error, is_failed_post,
+                         scoped_ptr<LocalizedError::ErrorPageParams>(),
+                         error_html);
 }
 
 void NetErrorHelperCore::GenerateLocalErrorPage(
     FrameType frame_type,
     const blink::WebURLError& error,
     bool is_failed_post,
+    scoped_ptr<LocalizedError::ErrorPageParams> params,
     std::string* error_html) {
   if (frame_type == MAIN_FRAME) {
     pending_error_page_info_.reset(new ErrorPageInfo(error, is_failed_post));
-    if (IsDnsError(error)) {
+    // Skip DNS logic if suggestions were received from a remote server.
+    // TODO(mmenke):  Consider integrating probe results with Link Doctor
+    //                suggestions.
+    if (IsDnsError(error) && !params) {
       // This is not strictly necessary, but waiting for a new status to be
       // sent as a result of the DidFinishLoading call keeps the histograms
       // consistent with older versions of the code, at no real cost.
       last_probe_status_ = chrome_common_net::DNS_PROBE_POSSIBLE;
 
       delegate_->GenerateLocalizedErrorPage(
-          GetUpdatedError(error), is_failed_post, error_html);
+          GetUpdatedError(error), is_failed_post, params.Pass(),
+          error_html);
       pending_error_page_info_->needs_dns_updates = true;
       return;
     }
   }
-  delegate_->GenerateLocalizedErrorPage(error, is_failed_post, error_html);
+
+  delegate_->GenerateLocalizedErrorPage(error, is_failed_post,
+                                        params.Pass(), error_html);
 }
 
 void NetErrorHelperCore::OnNetErrorInfo(
     chrome_common_net::DnsProbeStatus status) {
   DCHECK_NE(chrome_common_net::DNS_PROBE_POSSIBLE, status);
 
   last_probe_status_ = status;
 
   if (!committed_error_page_info_ ||
       !committed_error_page_info_->needs_dns_updates ||
       !committed_error_page_info_->is_finished_loading) {
     return;
   }
 
   UpdateErrorPage();
 }
 
+void NetErrorHelperCore::OnSetLinkDoctorInfo(const GURL& link_doctor_url,
+                                             const std::string& language,
+                                             const std::string& country_code,
+                                             const std::string& api_key,
+                                             const GURL& search_url) {
+  link_doctor_url_ = link_doctor_url;
+  language_ = language;
+  country_code_ = country_code;
+  api_key_ = api_key;
+  search_url_ = search_url;
+}
+
 void NetErrorHelperCore::UpdateErrorPage() {
   DCHECK(committed_error_page_info_->needs_dns_updates);
   DCHECK(committed_error_page_info_->is_finished_loading);
   DCHECK_NE(chrome_common_net::DNS_PROBE_POSSIBLE, last_probe_status_);
 
   UMA_HISTOGRAM_ENUMERATION("DnsProbe.ErrorPageUpdateStatus",
                             last_probe_status_,
                             chrome_common_net::DNS_PROBE_MAX);
   // Every status other than DNS_PROBE_POSSIBLE and DNS_PROBE_STARTED is a
   // final status code.  Once one is reached, the page does not need further
   // updates.
   if (last_probe_status_ != chrome_common_net::DNS_PROBE_STARTED)
     committed_error_page_info_->needs_dns_updates = false;
 
   delegate_->UpdateErrorPage(
       GetUpdatedError(committed_error_page_info_->error),
       committed_error_page_info_->was_failed_post);
 }
 
-void NetErrorHelperCore::OnAlternateErrorPageFetched(const std::string& data) {
+void NetErrorHelperCore::OnAlternateErrorPageFetched(
+    const std::string& data, const std::string& accept_languages, bool is_rtl) {
   // Alternate error page load only starts when an error page finishes loading,
-  // and is cancelled with a new load
+  // and is cancelled with a new load.
   DCHECK(!pending_error_page_info_);
   DCHECK(committed_error_page_info_->is_finished_loading);
 
-  const std::string* error_html = NULL;
-  std::string generated_html;
-  if (!data.empty()) {
-    // If the request succeeded, use the response in place of a generated error
-    // page.
-    pending_error_page_info_.reset(
-        new ErrorPageInfo(committed_error_page_info_->error,
-                          committed_error_page_info_->was_failed_post));
-    error_html = &data;
-  } else {
-    // Otherwise, generate a local error page.  |pending_error_page_info_| will
-    // be set by GenerateLocalErrorPage.
-    GenerateLocalErrorPage(MAIN_FRAME,
-                           committed_error_page_info_->error,
-                           committed_error_page_info_->was_failed_post,
-                           &generated_html);
-    error_html = &generated_html;
-  }
+  scoped_ptr<LocalizedError::ErrorPageParams> params(
+    ParseAdditionalSuggestions(
+        data, GURL(committed_error_page_info_->error.unreachableURL),
+        search_url_, accept_languages, is_rtl));
+  std::string error_html;
+  GenerateLocalErrorPage(MAIN_FRAME,
+                         committed_error_page_info_->error,
+                         committed_error_page_info_->was_failed_post,
+                         params.Pass(),
+                         &error_html);
 
   // |error_page_info| may have been destroyed by this point, since
   // |pending_error_page_info_| was set to a new ErrorPageInfo.
 
   // TODO(mmenke):  Once the new API is in place, look into replacing this
   //                double page load by just updating the error page, like DNS
   //                probes do.
   delegate_->LoadErrorPageInMainFrame(
-      *error_html,
+      error_html,
       pending_error_page_info_->error.unreachableURL);
 }
 
 blink::WebURLError NetErrorHelperCore::GetUpdatedError(
     const blink::WebURLError& error) const {
   // If a probe didn't run or wasn't conclusive, restore the original error.
   if (last_probe_status_ == chrome_common_net::DNS_PROBE_NOT_RUN ||
       last_probe_status_ ==
           chrome_common_net::DNS_PROBE_FINISHED_INCONCLUSIVE) {
     return error;
   }
 
   blink::WebURLError updated_error;
   updated_error.domain = blink::WebString::fromUTF8(
       chrome_common_net::kDnsProbeErrorDomain);
   updated_error.reason = last_probe_status_;
   updated_error.unreachableURL = error.unreachableURL;
 
   return updated_error;
 }