Make it possible to read CLD data from a file

chrome/renderer/translate/translate_helper.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "chrome/renderer/translate/translate_helper.h"
 
 #include "base/bind.h"
 #include "base/compiler_specific.h"
+#include "base/files/memory_mapped_file.h"
 #include "base/logging.h"
 #include "base/message_loop/message_loop.h"
 #include "base/strings/string16.h"
 #include "base/strings/string_util.h"
 #include "base/strings/utf_string_conversions.h"
 #include "chrome/common/render_messages.h"
 #include "chrome/renderer/extensions/extension_groups.h"
 #include "chrome/renderer/isolated_world_ids.h"
 #include "components/translate/core/common/translate_constants.h"
 #include "components/translate/core/common/translate_metrics.h"
 #include "components/translate/core/common/translate_util.h"
 #include "components/translate/language_detection/language_detection_util.h"
 #include "content/public/renderer/render_view.h"
+#include "extensions/common/constants.h"
+#include "ipc/ipc_platform_file.h"
+#if defined(CLD2_DYNAMIC_MODE)
+#include "content/public/common/url_constants.h"
+#include "third_party/cld_2/src/public/compact_lang_det.h"
+#endif
 #include "third_party/WebKit/public/web/WebDocument.h"
 #include "third_party/WebKit/public/web/WebElement.h"
 #include "third_party/WebKit/public/web/WebFrame.h"
 #include "third_party/WebKit/public/web/WebNode.h"
 #include "third_party/WebKit/public/web/WebNodeList.h"
 #include "third_party/WebKit/public/web/WebScriptSource.h"
 #include "third_party/WebKit/public/web/WebView.h"
 #include "third_party/WebKit/public/web/WebWidget.h"
 #include "url/gurl.h"
 #include "v8/include/v8.h"
@@ skipping 32 matching lines @@
 
 }  // namespace
 
 ////////////////////////////////////////////////////////////////////////////////
 // TranslateHelper, public:
 //
 TranslateHelper::TranslateHelper(content::RenderView* render_view)
     : content::RenderViewObserver(render_view),
       page_id_(-1),
       translation_pending_(false),
-      weak_method_factory_(this) {
+      weak_method_factory_(this)
+#if defined(CLD2_DYNAMIC_MODE)
+  ,cld2_data_file_polling_started(false),
+  cld2_data_file_polling_canceled(false),
+  deferred_page_capture_(false),
+  deferred_page_id_(-1),
+  deferred_contents_(ASCIIToUTF16(""))

[bulach, 2014/03/23 20:57:02]

nit: I think string16 default initializes to empty.
also, should be indented to match 78-81

[Andrew Hayden (chromium.org), 2014/03/24 15:18:24]

Done.

[bulach, 2014/03/25 09:29:26]

sorry, how about the string16 initialization, is it needed?

+#endif
+  {
 }
 
 TranslateHelper::~TranslateHelper() {
   CancelPendingTranslation();
+#if defined(CLD2_DYNAMIC_MODE)
+  CancelCLD2DataFilePolling();
+#endif
 }
 
+void TranslateHelper::PrepareForUrl(const GURL& url) {
+#if defined(CLD2_DYNAMIC_MODE)
+  deferred_page_capture_ = false;
+  deferred_contents_.clear();
+  if (cld2_data_file_polling_started)
+    return;
+
+  // TODO(andrewhayden): Refactor translate_manager.cc's IsTranslatableURL to
+  // components/translate/core/common/translate_util.cc, and ignore any URL
+  // that fails that check. This will require moving unit tests and rewiring
+  // other function calls as wlel, so for now we replicate the logic here.

[bulach, 2014/03/23 20:57:02]

nit: s/wlel/well/

[Andrew Hayden (chromium.org), 2014/03/24 15:18:24]

Done.

+  if (url.is_empty())
+      return;

[bulach, 2014/03/23 20:57:02]

nit: here and all "return" clauses below, unindent

[Andrew Hayden (chromium.org), 2014/03/24 15:18:24]

Done.

+  if (url.SchemeIs(content::kChromeUIScheme))
+      return;
+  if (url.SchemeIs(content::kChromeDevToolsScheme))
+      return;
+  if (url.SchemeIs(content::kFtpScheme))
+      return;
+#if defined(OS_CHROMEOS)
+  if (url.SchemeIs(extensions::kExtensionScheme) &&
+      url.DomainIs(file_manager::kFileManagerAppId))
+    return;
+#endif
+
+  // If the URL isn't an extension URL, we might conceivably want translation
+  // capabilities. Start polling for CLD data.
+  cld2_data_file_polling_started = true;
+  TranslateHelper::SendCLD2DataFileRequest(0, 1000);
+#endif
+}
+
+#if defined(CLD2_DYNAMIC_MODE)
+void TranslateHelper::DeferPageCaptured(const int page_id,
+                                        const base::string16& contents) {
+  deferred_page_capture_ = true;
+  deferred_page_id_ = page_id;
+  deferred_contents_ = contents;
+}
+#endif
+
 void TranslateHelper::PageCaptured(int page_id,
                                    const base::string16& contents) {
   // Get the document language as set by WebKit from the http-equiv
   // meta tag for "content-language".  This may or may not also
   // have a value derived from the actual Content-Language HTTP
   // header.  The two actually have different meanings (despite the
   // original intent of http-equiv to be an equivalent) with the former
   // being the language of the document and the latter being the
   // language of the intended audience (a distinction really only
   // relevant for things like langauge textbooks).  This distinction
   // shouldn't affect translation.
   WebFrame* main_frame = GetMainFrame();
   if (!main_frame || render_view()->GetPageId() != page_id)
     return;
+
+  // TODO(andrewhayden): UMA insertion point here: Track if data is available.
+  // TODO(andrewhayden): Retry insertion point here, retry till data available.
+#if defined(CLD2_DYNAMIC_MODE)
+  if (!CLD2::isDataLoaded()) {
+    // We're in dynamic mode and CLD data isn't loaded. Retry when CLD data
+    // is loaded, if ever.
+    TranslateHelper::DeferPageCaptured(page_id, contents);
+    return;
+  }
+#endif
   page_id_ = page_id;
   WebDocument document = main_frame->document();
   std::string content_language = document.contentLanguage().utf8();
   WebElement html_element = document.documentElement();
   std::string html_lang;
   // |html_element| can be null element, e.g. in
   // BrowserTest.WindowOpenClose.
   if (!html_element.isNull())
     html_lang = html_element.getAttribute("lang").utf8();
   std::string cld_language;
@@ skipping 24 matching lines @@
       routing_id(),
       details,
       IsTranslationAllowed(&document) && !language.empty()));
 }
 
 void TranslateHelper::CancelPendingTranslation() {
   weak_method_factory_.InvalidateWeakPtrs();
   translation_pending_ = false;
   source_lang_.clear();
   target_lang_.clear();
+#if defined(CLD2_DYNAMIC_MODE)
+  CancelCLD2DataFilePolling();
+#endif
 }
 
 ////////////////////////////////////////////////////////////////////////////////
 // TranslateHelper, protected:
 //
 bool TranslateHelper::IsTranslateLibAvailable() {
   return ExecuteScriptAndGetBoolResult(
       "typeof cr != 'undefined' && typeof cr.googleTranslate != 'undefined' && "
       "typeof cr.googleTranslate.translate == 'function'", false);
 }
@@ skipping 154 matching lines @@
       return false;
   }
   return true;
 }
 
 bool TranslateHelper::OnMessageReceived(const IPC::Message& message) {
   bool handled = true;
   IPC_BEGIN_MESSAGE_MAP(TranslateHelper, message)
     IPC_MESSAGE_HANDLER(ChromeViewMsg_TranslatePage, OnTranslatePage)
     IPC_MESSAGE_HANDLER(ChromeViewMsg_RevertTranslation, OnRevertTranslation)
+#if defined(CLD2_DYNAMIC_MODE)
+    IPC_MESSAGE_HANDLER(ChromeViewMsg_CLDDataAvailable, OnCLDDataAvailable);
+#endif
     IPC_MESSAGE_UNHANDLED(handled = false)
   IPC_END_MESSAGE_MAP()
   return handled;
 }
 
 void TranslateHelper::OnTranslatePage(int page_id,
                                       const std::string& translate_script,
                                       const std::string& source_lang,
                                       const std::string& target_lang) {
   WebFrame* main_frame = GetMainFrame();
@@ skipping 169 matching lines @@
 
 WebFrame* TranslateHelper::GetMainFrame() {
   WebView* web_view = render_view()->GetWebView();
 
   // When the tab is going to be closed, the web_view can be NULL.
   if (!web_view)
     return NULL;
 
   return web_view->mainFrame();
 }
+
+#if defined(CLD2_DYNAMIC_MODE)
+void TranslateHelper::CancelCLD2DataFilePolling() {
+  cld2_data_file_polling_canceled = true;
+}
+
+void TranslateHelper::SendCLD2DataFileRequest(int delay_millis,
+                                              int next_delay_millis) {
+  // We terminate immediately if we've been told to stop polling
+  if (cld2_data_file_polling_canceled)
+    return;
+
+  // We terminate immediately if we've already loaded the data.
+  if (CLD2::isDataLoaded())
+    return;
+
+  // Else, send the IPC message to the browser process requesting the data...
+  Send(new ChromeViewHostMsg_NeedCLDData(routing_id()));
+
+  // ... and enqueue another delayed task to call again. This will start a
+  // chain of polling that will last until the pointer stops being null,
+  // which is the right thing to do.
+  // NB: In the great majority of cases, the data file will be available and
+  // the very first delayed task will be a no-op that terminates the chain.
+  // It's only while downloading the file that we expect this to chain for a
+  // nontrivial amount of time.
+  // Use a weak pointer so that we don't keep this helper object around forever.
+  base::MessageLoop::current()->PostDelayedTask(
+      FROM_HERE,
+      base::Bind(&TranslateHelper::SendCLD2DataFileRequest,
+                 weak_method_factory_.GetWeakPtr(),
+                 next_delay_millis, next_delay_millis),
+      base::TimeDelta::FromMilliseconds(delay_millis));
+}
+
+void TranslateHelper::OnCLDDataAvailable(
+    IPC::PlatformFileForTransit ipc_file_handle) {
+  LoadCLDDData(ipc_file_handle);
+  if (deferred_page_capture_) {

[bulach, 2014/03/23 20:57:02]

nit: remove one level:

if (deferred_page_capture_ && CLD2::isDataLoaded()) {
  ...
}

[Andrew Hayden (chromium.org), 2014/03/24 15:18:24]

Done.

+    if (CLD2::isDataLoaded()) {
+      // We have data available now, and we previously deferred a request to
+      // start translation. Finish that request up and clear our state.
+      // The real situation this translates to is as follows:
+      // The user loaded a page before the CLD data was available - either
+      // because the browser process took too long to respond to our CLD data
+      // request, or because the data truly hasn't been available;
+      // The data is now available, so we should get that language check
+      // done and post the message back to the browser ASAP.
+      deferred_page_capture_ = false; // Don't do this a second time.
+      PageCaptured(deferred_page_id_, deferred_contents_);
+      deferred_page_id_ = -1; // clean up for sanity
+      deferred_contents_.clear(); // clean up for sanity
+    }
+  }
+}
+
+void TranslateHelper::LoadCLDDData(
+    IPC::PlatformFileForTransit ipc_file_handle) {
+  // We terminate immediately if we've been told to stop polling
+  if (cld2_data_file_polling_canceled)
+    return;
+
+  // We terminate immediately if we've already loaded the data.
+  if (CLD2::isDataLoaded())
+    return;
+
+  // The mmap must outlive this function and must not be destroyed because
+  // the destructor for mmap will unmap the memory segment and close the file
+  // handle. Thus, declare it static.
+  static base::MemoryMappedFile* mmap = NULL;

[bulach, 2014/03/23 20:57:02]

no function-scoped statics... what's the lifetime of this class? could it be a
regular scoped_ptr member of this class itself?

[Andrew Hayden (chromium.org), 2014/03/24 15:18:24]

Yes. I'll make that change.

+
+  // Grab the file handle
+  base::PlatformFile platformFile =
+      IPC::PlatformFileForTransitToPlatformFile(ipc_file_handle);
+  DCHECK(platformFile > 0) << "Can't find the CLD data file!";
+  base::File basicFile(platformFile);

[bulach, 2014/03/23 20:57:02]

nit: s/basicFile/basic_file/

[Andrew Hayden (chromium.org), 2014/03/24 15:18:24]

Done.

+
+  // mmap the file
+  mmap = new base::MemoryMappedFile();
+  bool loadedOk = mmap->Initialize(basicFile.Pass());

[bulach, 2014/03/23 20:57:02]

nit: s/loadedOk/loaded_ok/
perhaps "initialized" would be clearer?

[Andrew Hayden (chromium.org), 2014/03/24 15:18:24]

Done.

+  DCHECK(loadedOk) << "Failed to initialize CLD data mmap";
+  if (!loadedOk) {
+    delete mmap;
+    mmap = NULL;
+    return;
+  }
+
+  // Initialize the CLD subsystem... we're done!
+  CLD2::loadDataFromRawAddress(mmap->data(), mmap->length());
+  DCHECK(CLD2::isDataLoaded()) << "Failed to load CLD data from mmap";
+}
+#endif