[translate] Add translate ranker model loader.

components/translate/core/browser/ranker_model_loader.cc

+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "components/translate/core/browser/ranker_model_loader.h"
+
+#include "base/bind.h"
+#include "base/bind_helpers.h"
+#include "base/command_line.h"
+#include "base/files/file_path.h"
+#include "base/files/file_util.h"
+#include "base/files/important_file_writer.h"
+#include "base/memory/ptr_util.h"
+#include "base/memory/ref_counted.h"
+#include "base/metrics/histogram_macros.h"
+#include "base/profiler/scoped_tracker.h"
+#include "base/strings/string_number_conversions.h"
+#include "base/strings/string_util.h"
+#include "base/task_runner.h"
+#include "base/threading/platform_thread.h"
+#include "components/translate/core/browser/translate_url_fetcher.h"
+#include "components/translate/core/common/translate_switches.h"
+#include "components/variations/variations_associated_data.h"
+
+namespace {
+

[groby-ooo-7-16, 2017/01/23 21:41:56]

Any tests for the loader?

[Roger McFarlane (Chromium), 2017/02/08 23:08:09]

Coming.

+class MyScopedHistogramTimer {

[groby-ooo-7-16, 2017/01/23 21:41:56]

Why not SCOPED_UMA_HISTOGRAM_TIMER?

[Roger McFarlane (Chromium), 2017/02/08 23:08:09]

The UMA macros require static const data for the metric names.

+ public:
+  MyScopedHistogramTimer(const base::StringPiece& name)
+      : name_(name.begin(), name.end()), start_(base::TimeTicks::Now()) {}
+
+  ~MyScopedHistogramTimer() {
+    base::TimeDelta duration = base::TimeTicks::Now() - start_;
+    base::HistogramBase* counter = base::Histogram::FactoryTimeGet(
+        name_, base::TimeDelta::FromMilliseconds(10),
+        base::TimeDelta::FromMilliseconds(200000), 100,
+        base::HistogramBase::kUmaTargetedHistogramFlag);
+    if (counter)
+      counter->AddTime(duration);
+  }
+
+ private:
+  const std::string name_;
+  const base::TimeTicks start_;
+};
+
+bool IsExpired(const chrome_intelligence::RankerModel& model) {

[groby-ooo-7-16, 2017/01/23 21:41:56]

Why is "IsExpired" not a member on the model? It pretty much exclusively depends
on model data?

[Roger McFarlane (Chromium), 2017/02/08 23:08:09]

RankerModel is a proto.

I could rename the proto to RankerModelProto and add a RankerModel wrapper
class, to which I could attach helper methods.

+  DCHECK(model.has_metadata());
+  const auto& metadata = model.metadata();
+
+  // If the age of the model cannot be determined, presume it to be expired.
+  if (!metadata.has_last_modified_sec())
+    return true;
+
+  // If the model has no set cache duration, then it never expires.
+  if (!metadata.has_cache_duration_sec() || metadata.cache_duration_sec() == 0)
+    return false;
+
+  // Otherwise, a model is expired if its age exceeds the cache duration.
+  base::Time last_modified =
+      base::Time() + base::TimeDelta::FromSeconds(metadata.last_modified_sec());
+  base::TimeDelta age = base::Time::Now() - last_modified;
+  return age > base::TimeDelta::FromSeconds(metadata.cache_duration_sec());
+}
+
+}  // namespace
+
+namespace translate {
+
+const int kUrlFetcherId = 2;
+const char kWriteTimerHistogram[] = ".Timer.WriteModel";
+const char kReadTimerHistogram[] = ".Timer.ReadModel";
+const char kDownloadTimerHistogram[] = ".Timer.DownloadModel";
+const char kParsetimerHistogram[] = ".Timer.ParseModel";
+const char kModelStatusHistogram[] = ".Model.Status";
+
+RankerModelLoader::RankerModelLoader(RankerModelObserver* observer,
+                                     const base::FilePath& model_path,
+                                     const GURL& model_url,
+                                     const std::string& uma_prefix)
+    : thread_(uma_prefix + ".LoaderThread"),
+      observer_(observer),
+      model_path_(model_path),
+      model_url_(model_url),
+      uma_prefix_(uma_prefix) {
+  DCHECK(observer_ != nullptr);
+  sequence_checker_.DetachFromSequence();
+
+  base::Thread::Options options;
+  options.message_loop_type = base::MessageLoop::TYPE_IO;

[groby-ooo-7-16, 2017/01/23 21:41:56]

Does this need a separate thread? Can we use a WorkerPool instead?

[Roger McFarlane (Chromium), 2017/02/08 23:08:09]

Ideally, I'd use the task scheduler api and just get a sequenced runner. I'm
working with gab@ and fdoray@ to sort out how to set up the proper unittest
scaffolding for this.

I.e., the only reason it's a thread is because we're trying to figure out how to
set up the unittests for the task scheduler properly (otherwise crashes ensue)

+  options.priority = base::ThreadPriority::BACKGROUND;
+  thread_.StartWithOptions(options);
+}
+
+RankerModelLoader::~RankerModelLoader() {}
+
+void RankerModelLoader::Start() {
+  thread_.task_runner()->PostTask(
+      FROM_HERE,
+      base::Bind(&RankerModelLoader::LoadFromFile, base::Unretained(this)));
+}
+
+void RankerModelLoader::NotifyOfNetworkAvailability() {
+  if (url_fetcher_) {
+    thread_.task_runner()->PostTask(
+        FROM_HERE,
+        base::Bind(&RankerModelLoader::LoadFromURL, base::Unretained(this)));
+  }
+}
+
+void RankerModelLoader::ReportModelStatus(RankerModelStatus model_status) {
+  base::HistogramBase* histogram = base::LinearHistogram::FactoryGet(

[groby-ooo-7-16, 2017/01/23 21:41:56]

Why do we need to do this manually, instead of via a macro?

[Roger McFarlane (Chromium), 2017/02/08 23:08:09]

The macros require static const histogram names.

+      uma_prefix_ + kModelStatusHistogram, 1,
+      static_cast<int>(RankerModelStatus::MAX),

[groby-ooo-7-16, 2017/01/23 21:41:56]

Why the cast? IIRC the standard says underlying type for enum class should be
int?

[Roger McFarlane (Chromium), 2017/02/08 23:08:09]

clang++ fails to resolve the conversion from the enum to Sample (aka int). This
happens even with enum class RankerModelStatus : int { ... };

:(

+      static_cast<int>(RankerModelStatus::MAX) + 1,
+      base::HistogramBase::kUmaTargetedHistogramFlag);
+  if (histogram)
+    histogram->Add(static_cast<int>(model_status));
+}
+
+std::unique_ptr<chrome_intelligence::RankerModel> RankerModelLoader::Parse(
+    const std::string& data) {
+  DCHECK(sequence_checker_.CalledOnValidSequence());
+  MyScopedHistogramTimer timer(uma_prefix_ + kParsetimerHistogram);
+
+  auto model = base::MakeUnique<chrome_intelligence::RankerModel>();
+
+  if (!model->ParseFromString(data)) {
+    ReportModelStatus(RankerModelStatus::PARSE_FAILED);
+    return nullptr;
+  }
+
+  RankerModelStatus model_status = observer_->Validate(*model);
+  ReportModelStatus(model_status);
+  if (model_status != RankerModelStatus::OK)
+    return nullptr;
+
+  return model;
+}
+
+void RankerModelLoader::LoadFromFile() {
+  DCHECK(sequence_checker_.CalledOnValidSequence());
+  // Attempt to read the model data from the cache file.
+  std::string data;
+  if (!model_path_.empty()) {
+    DVLOG(2) << "Attempting to load model from: " << model_path_.value();
+    MyScopedHistogramTimer timer(uma_prefix_ + kReadTimerHistogram);
+    if (!base::ReadFileToString(model_path_, &data)) {
+      DVLOG(2) << "Failed to read model from: " << model_path_.value();
+      data.clear();
+    }
+  }
+
+  // If the model was successfully was read and is compatible, then notify
+  // the "owner" of this model loader of the models availability (transferring
+  // ownership of the model). If the model originated at the model_url_ then
+  // the model is also up to date, no need to download a new one.
+  if (!data.empty()) {
+    std::unique_ptr<chrome_intelligence::RankerModel> model = Parse(data);
+    if (model) {
+      DVLOG(2) << "Model in '" << model_path_.value()
+               << "'' was originally downloaded from '" << model_url_ << "'.";
+
+      std::string source = model->metadata().source();

[groby-ooo-7-16, 2017/01/23 21:41:56]

Technically, we only need src inside the !IsExpired expression

[Roger McFarlane (Chromium), 2017/02/08 23:08:09]

Oops! the DVLOG line above had the wrong source.

Good catch!

+      if (!IsExpired(*model)) {
+        TransferModelToObserver(std::move(model));
+        if (source == model_url_.spec())
+          return;
+      }
+    }
+  }
+
+  // Reaching this point means that a model download is required. If there is
+  // no download URL configured, then there is nothing further to do.
+  if (!model_url_.is_valid())
+    return;
+
+  // Otherwise, initialize the model fetcher to be non-null and trigger an
+  // initial download attempt.
+  url_fetcher_ = base::MakeUnique<TranslateURLFetcher>(kUrlFetcherId);
+  url_fetcher_->set_max_retry_on_5xx(kMaxRetryOn5xx);
+  LoadFromURL();
+}
+
+void RankerModelLoader::LoadFromURL() {
+  DCHECK(sequence_checker_.CalledOnValidSequence());
+  // Immediately exit if there is no download pending.
+  if (!url_fetcher_)
+    return;
+
+  // If a request is already in flight, do not issue a new one.
+  if (url_fetcher_->state() == TranslateURLFetcher::REQUESTING) {
+    DVLOG(2) << "RankerModelLoader: Download is in progress.";
+    return;
+  }
+  // Do nothing if the download attempts should be throttled.
+  if (base::TimeTicks::Now() < next_earliest_download_time_) {
+    DVLOG(2) << "ModelLoadser: Last download attempt was too recent.";
+    return;
+  }
+
+  DVLOG(2) << "Downloading model from: " << model_url_;
+
+  // Reset the time of the next earliest allowable download attempt.
+  next_earliest_download_time_ =
+      base::TimeTicks::Now() +
+      base::TimeDelta::FromMinutes(kDownloadRefractoryPeriodMin);
+
+  // Kick off the next download attempt.
+  download_start_time_ = base::TimeTicks::Now();
+  bool result = url_fetcher_->Request(
+      model_url_, base::Bind(&RankerModelLoader::OnDownloadComplete,
+                             base::Unretained(this)));
+
+  // The maximum number of download attempts has been surpassed. Don't make
+  // any further attempts.
+  if (!result) {
+    DVLOG(2) << "Model download abandoned.";
+    ReportModelStatus(RankerModelStatus::DOWNLOAD_FAILED);
+    url_fetcher_.reset();
+  }
+}
+
+void RankerModelLoader::OnDownloadComplete(int /* id */,
+                                           bool success,
+                                           const std::string& data) {
+  DCHECK(sequence_checker_.CalledOnValidSequence());
+
+  // Record the duration of the download.
+  base::TimeDelta duration = base::TimeTicks::Now() - download_start_time_;
+  base::HistogramBase* counter = base::Histogram::FactoryTimeGet(
+      uma_prefix_ + kDownloadTimerHistogram,
+      base::TimeDelta::FromMilliseconds(10),
+      base::TimeDelta::FromMilliseconds(200000), 100,
+      base::HistogramBase::kUmaTargetedHistogramFlag);
+  if (counter)
+    counter->AddTime(duration);
+
+  // On failure, we just abort. The TranslateRanker will retry on a subsequent
+  // translation opportunity. The TranslateURLFetcher enforces a limit for
+  // retried requests.
+  if (!success) {
+    DVLOG(2) << "Download from '" << model_url_ << "'' failed.";
+    return;
+  }
+
+  auto model = Parse(data);
+  if (!model) {
+    DVLOG(2) << "Model from '" << model_url_ << "'' failed to parse.";
+    return;
+  }
+  url_fetcher_.reset();
+
+  auto* metadata = model->mutable_metadata();
+  metadata->set_source(model_url_.spec());
+  metadata->set_last_modified_sec(
+      (base::Time::Now() - base::Time()).InSeconds());
+
+  if (!model_path_.empty()) {
+    DVLOG(2) << "Saving model from '" << model_url_ << "'' to '"
+             << model_path_.value() << "'.";
+    MyScopedHistogramTimer timer(uma_prefix_ + kWriteTimerHistogram);
+    base::ImportantFileWriter::WriteFileAtomically(model_path_,
+                                                   model->SerializeAsString());
+  }
+
+  // Notify the owner that a compatible model is available.
+  TransferModelToObserver(std::move(model));
+}
+
+void RankerModelLoader::TransferModelToObserver(
+    std::unique_ptr<chrome_intelligence::RankerModel> model) {
+  DCHECK(sequence_checker_.CalledOnValidSequence());
+  observer_->OnModelAvailable(std::move(model));
+}
+
+}  // namespace translate