Make it possible to read CLD data from a file

chrome/renderer/translate/translate_helper.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "chrome/renderer/translate/translate_helper.h"
 
 #include "base/bind.h"
 #include "base/compiler_specific.h"
+#include "base/files/memory_mapped_file.h"
 #include "base/logging.h"
 #include "base/message_loop/message_loop.h"
 #include "base/strings/string16.h"
 #include "base/strings/string_util.h"
 #include "base/strings/utf_string_conversions.h"
 #include "chrome/common/render_messages.h"
 #include "chrome/renderer/extensions/extension_groups.h"
 #include "chrome/renderer/isolated_world_ids.h"
 #include "components/translate/core/common/translate_constants.h"
 #include "components/translate/core/common/translate_metrics.h"
 #include "components/translate/core/common/translate_util.h"
 #include "components/translate/language_detection/language_detection_util.h"
 #include "content/public/renderer/render_view.h"
+#include "extensions/common/constants.h"
+#include "ipc/ipc_platform_file.h"
+#if defined(CLD2_DYNAMIC_MODE)
+#include "third_party/cld_2/src/public/compact_lang_det.h"
+#endif
 #include "third_party/WebKit/public/web/WebDocument.h"
 #include "third_party/WebKit/public/web/WebElement.h"
 #include "third_party/WebKit/public/web/WebFrame.h"
 #include "third_party/WebKit/public/web/WebNode.h"
 #include "third_party/WebKit/public/web/WebNodeList.h"
 #include "third_party/WebKit/public/web/WebScriptSource.h"
 #include "third_party/WebKit/public/web/WebView.h"
 #include "third_party/WebKit/public/web/WebWidget.h"
 #include "url/gurl.h"
 #include "v8/include/v8.h"
@@ skipping 14 matching lines @@
 
 // The delay in milliseconds that we'll wait before checking to see if the
 // translate library injected in the page is ready.
 const int kTranslateInitCheckDelayMs = 150;
 
 // The maximum number of times we'll check to see if the translate library
 // injected in the page is ready.
 const int kMaxTranslateInitCheckAttempts = 5;
 
 // The delay we wait in milliseconds before checking whether the translation has
-// finished.
+// finished.cld2_data_file = NULL;

[Takashi Toyoshima, 2014/03/13 18:37:37]

paste it mistakenly?

[Andrew Hayden (chromium.org), 2014/03/14 10:10:52]

Done.

 const int kTranslateStatusCheckDelayMs = 400;
 
 // Language name passed to the Translate element for it to detect the language.
 const char kAutoDetectionLanguage[] = "auto";
 
 // Isolated world sets following content-security-policy.
 const char kContentSecurityPolicy[] = "script-src 'self' 'unsafe-eval'";
 
 }  // namespace
 
 ////////////////////////////////////////////////////////////////////////////////
 // TranslateHelper, public:
 //
 TranslateHelper::TranslateHelper(content::RenderView* render_view)
     : content::RenderViewObserver(render_view),
       page_id_(-1),
       translation_pending_(false),
       weak_method_factory_(this) {
+#if defined(CLD2_DYNAMIC_MODE)
+  cld2_data_file_polling_started = false;
+  cld2_data_file_polling_canceled = false;
+  deferred_page_capture_ = false;
+  deferred_page_id_ = -1;

[palmer, 2014/03/13 17:59:50]

Put these up above in the : initialization list.

[Andrew Hayden (chromium.org), 2014/03/13 23:20:21]

Done.

+  deferred_contents_ = ASCIIToUTF16("");
+#endif
 }
 
 TranslateHelper::~TranslateHelper() {
   CancelPendingTranslation();
+#if defined(CLD2_DYNAMIC_MODE)
+  CancelCLD2DataFilePolling();
+#endif
 }
 
+void TranslateHelper::PrepareForUrl(const GURL& url) {
+#if defined(CLD2_DYNAMIC_MODE)

[Takashi Toyoshima, 2014/03/13 18:37:37]

you may want to reset deferred_* here?

[Andrew Hayden (chromium.org), 2014/03/14 10:10:52]

Hmm, good thought. It's SAFE to leave it alone, because we will check the page
id before we resurrect the language detection request, but IIUC we could end up
showing the translate bar after the user has begun navigation to another page
but before that page has actually loaded. So yeah... we should do the reset for
maximum sanity. Good catch.

+  if (cld2_data_file_polling_started) return;

[palmer, 2014/03/13 17:59:50]

Style nit.

[Andrew Hayden (chromium.org), 2014/03/13 23:20:21]

Done.

+
+  // There is no reason to ever bother loading CLD data for an extension URL.
+  // There's also no render_view_host_observer to answer us, so don't ask for
+  // CLD data!
+  if (url.SchemeIs(extensions::kExtensionScheme)) return;

[Takashi Toyoshima, 2014/03/13 18:37:37]

TranslateManager::IsTranslatableURL() is the function exactly you may want to
use here. But the function is for browser process. Can we move the function to
components/translate/core/common/translate_util.h|cc, and use it from both
TranslateManager and TranslateHelper?

WDYT? I guess this check do not have to be so strict. So, this current check may
work for you.

[Andrew Hayden (chromium.org), 2014/03/14 10:10:52]

For clarity I'll submit a separate patchset in this review with just this
change, so holding off for a few minutes.

+
+  // If the URL isn't an extension URL, we might conceivably want translation
+  // capabilities. Start polling for CLD data.
+  cld2_data_file_polling_started = true;
+  TranslateHelper::SendCLD2DataFileRequest(0, 1000);
+#endif
+}
+
+#if defined(CLD2_DYNAMIC_MODE)
+void TranslateHelper::DeferPageCaptured(const int page_id,
+                                           const base::string16& contents) {

[palmer, 2014/03/13 17:59:50]

Nit: indentation.

[Takashi Toyoshima, 2014/03/13 18:37:37]

wrong indent

[Andrew Hayden (chromium.org), 2014/03/13 23:20:21]

Done.

+  deferred_page_capture_ = true;
+  deferred_page_id_ = page_id;
+  deferred_contents_.clear();
+  deferred_contents_.append(contents);

[Takashi Toyoshima, 2014/03/13 18:37:37]

Just a question.
Why don't you simply assign it, but call clean() and append()?

[Andrew Hayden (chromium.org), 2014/03/14 10:10:52]

I'll assign it instead. I was concerned about the difference between operator=
and assign, but after reading more about std::string (the underlying construct
for string16) it seems this should do the right thing. I wanted to make sure I
wasn't going to be left with a string pointing to the memory in the original
contents reference, but it looks like operator= will definitely copy. Sorry, my
C++ is a bit rusty :)

+}
+#endif
+
 void TranslateHelper::PageCaptured(int page_id,
                                    const base::string16& contents) {
   // Get the document language as set by WebKit from the http-equiv
   // meta tag for "content-language".  This may or may not also
   // have a value derived from the actual Content-Language HTTP
   // header.  The two actually have different meanings (despite the
   // original intent of http-equiv to be an equivalent) with the former
   // being the language of the document and the latter being the
   // language of the intended audience (a distinction really only
   // relevant for things like langauge textbooks).  This distinction
   // shouldn't affect translation.
   WebFrame* main_frame = GetMainFrame();
   if (!main_frame || render_view()->GetPageId() != page_id)
     return;
+
+  // TODO(andrewhayden): UMA insertion point here: Track if data is available.

[Takashi Toyoshima, 2014/03/13 18:37:37]

Just a question:
How long time does it take from calling SendCLD2DataFileRequest() to file mmap
being completed. Do you have any data? IIUC, it affects only against the first
page loading, so the latency does not matter. But just I'm interested in.
If you do not have any data, please ignore this comment.

[Andrew Hayden (chromium.org), 2014/03/14 10:10:52]

I don't have data but I'd be happy to measure it :)

+  // TODO(andrewhayden): Retry insertion point here, retry till data available.
+#if defined(CLD2_DYNAMIC_MODE)
+  if (!CLD2::isDataLoaded()) {
+    // We're in dynamic mode and CLD data isn't loaded. Retry when CLD data
+    // is loaded, if ever.
+    TranslateHelper::DeferPageCaptured(page_id, contents);
+    return;
+  }
+#endif
   page_id_ = page_id;
   WebDocument document = main_frame->document();
   std::string content_language = document.contentLanguage().utf8();
   WebElement html_element = document.documentElement();
   std::string html_lang;
   // |html_element| can be null element, e.g. in
   // BrowserTest.WindowOpenClose.
   if (!html_element.isNull())
     html_lang = html_element.getAttribute("lang").utf8();
   std::string cld_language;
@@ skipping 24 matching lines @@
       routing_id(),
       details,
       IsTranslationAllowed(&document) && !language.empty()));
 }
 
 void TranslateHelper::CancelPendingTranslation() {
   weak_method_factory_.InvalidateWeakPtrs();
   translation_pending_ = false;
   source_lang_.clear();
   target_lang_.clear();
+#if defined(CLD2_DYNAMIC_MODE)
+  CancelCLD2DataFilePolling();
+#endif
 }
 
 ////////////////////////////////////////////////////////////////////////////////
 // TranslateHelper, protected:
 //
 bool TranslateHelper::IsTranslateLibAvailable() {
   return ExecuteScriptAndGetBoolResult(
       "typeof cr != 'undefined' && typeof cr.googleTranslate != 'undefined' && "
       "typeof cr.googleTranslate.translate == 'function'", false);
 }
@@ skipping 154 matching lines @@
       return false;
   }
   return true;
 }
 
 bool TranslateHelper::OnMessageReceived(const IPC::Message& message) {
   bool handled = true;
   IPC_BEGIN_MESSAGE_MAP(TranslateHelper, message)
     IPC_MESSAGE_HANDLER(ChromeViewMsg_TranslatePage, OnTranslatePage)
     IPC_MESSAGE_HANDLER(ChromeViewMsg_RevertTranslation, OnRevertTranslation)
+#if defined(CLD2_DYNAMIC_MODE)
+    IPC_MESSAGE_HANDLER(ChromeViewMsg_CLDDataAvailable, OnCLDDataAvailable);
+#endif
     IPC_MESSAGE_UNHANDLED(handled = false)
   IPC_END_MESSAGE_MAP()
   return handled;
 }
 
 void TranslateHelper::OnTranslatePage(int page_id,
                                       const std::string& translate_script,
                                       const std::string& source_lang,
                                       const std::string& target_lang) {
   WebFrame* main_frame = GetMainFrame();
@@ skipping 169 matching lines @@
 
 WebFrame* TranslateHelper::GetMainFrame() {
   WebView* web_view = render_view()->GetWebView();
 
   // When the tab is going to be closed, the web_view can be NULL.
   if (!web_view)
     return NULL;
 
   return web_view->mainFrame();
 }
+
+#if defined(CLD2_DYNAMIC_MODE)
+void TranslateHelper::CancelCLD2DataFilePolling() {
+  cld2_data_file_polling_canceled = true;
+}
+
+void TranslateHelper::SendCLD2DataFileRequest(int delay_millis,
+                                              int next_delay_millis) {
+  // We terminate immediately if we've been told to stop polling
+  if (cld2_data_file_polling_canceled) return;

[palmer, 2014/03/13 17:59:50]

Style (everywhere this occurs).

[Andrew Hayden (chromium.org), 2014/03/13 23:20:21]

Done.

+
+  // We terminate immediately if we've already loaded the data.
+  if (CLD2::isDataLoaded()) return;
+
+  // Else, send the IPC message to the browser process requesting the data...
+  Send(new ChromeViewHostMsg_NeedCLDData(routing_id()));
+
+  // ... and enqueue another delayed task to call again. This will start a
+  // chain of polling that will last until the pointer stops being null,
+  // which is the right thing to do.
+  // NB: In the great majority of cases, the data file will be available and
+  // the very first delayed task will be a no-op that terminates the chain.
+  // It's only while downloading the file that we expect this to chain for a
+  // nontrivial amount of time.
+  // Use a weak pointer so that we don't keep this helper object around forever.
+  base::MessageLoop::current()->PostDelayedTask(
+      FROM_HERE,
+      base::Bind(&TranslateHelper::SendCLD2DataFileRequest,
+                 weak_method_factory_.GetWeakPtr(),
+                 next_delay_millis, next_delay_millis),
+      base::TimeDelta::FromMilliseconds(delay_millis));
+}
+
+void TranslateHelper::OnCLDDataAvailable(
+    IPC::PlatformFileForTransit ipc_file_handle) {
+  LoadCLDDData(ipc_file_handle);
+  if (deferred_page_capture_) {
+    if (CLD2::isDataLoaded()) {
+      // We have data available now, and we previously deferred a request to
+      // start translation. Finish that request up and clear our state.
+      // The real situation this translates to is as follows:
+      // The user loaded a page before the CLD data was available - either
+      // because the browser process took too long to respond to our CLD data
+      // request, or because the data truly hasn't been available;
+      // The data is now available, so we should get that language check
+      // done and post the message back to the browser ASAP.
+      deferred_page_capture_ = false; // Don't do this a second time.
+      PageCaptured(deferred_page_id_, deferred_contents_);
+      deferred_page_id_ = -1; // clean up for sanity
+      deferred_contents_.clear(); // clean up for sanity
+    }
+  }
+}
+
+void TranslateHelper::LoadCLDDData(
+    IPC::PlatformFileForTransit ipc_file_handle) {
+  // We terminate immediately if we've been told to stop polling
+  if (cld2_data_file_polling_canceled) return;
+
+  // We terminate immediately if we've already loaded the data.
+  if (CLD2::isDataLoaded()) return;
+
+  // The mmap must outlive this function and must not be destroyed because
+  // the destructor for mmap will unmap the memory segment and close the file
+  // handle. Thus, declare it static.
+  static base::MemoryMappedFile* mmap = NULL;
+  DCHECK(mmap != NULL) << "Lost CLD mmap!"; // Should be impossible!
+
+  // Grab the file handle
+  base::PlatformFile platformFile =
+      IPC::PlatformFileForTransitToPlatformFile(ipc_file_handle);
+  DCHECK(platformFile > 0) << "Can't find the CLD data file!";
+  base::File basicFile(platformFile);
+
+  // mmap the file
+  mmap = new base::MemoryMappedFile();
+  bool loadedOk = mmap->Initialize(basicFile.Pass());
+  DCHECK(loadedOk) << "Failed to initialize CLD data mmap";
+  if (!loadedOk) {
+    delete mmap;
+    mmap = NULL;
+    return;
+  }
+
+  // Initialize the CLD subsystem... we're done!
+  CLD2::loadDataFromRawAddress(mmap->data(), mmap->length());
+  DCHECK(CLD2::isDataLoaded()) << "Failed to load CLD data from mmap";
+}
+#endif