OLD | NEW |
(Empty) | |
| 1 // Copyright 2016 The Chromium Authors. All rights reserved. |
| 2 // Use of this source code is governed by a BSD-style license that can be |
| 3 // found in the LICENSE file. |
| 4 |
| 5 #include "components/translate/core/browser/ranker_model_loader.h" |
| 6 |
| 7 #include "base/bind.h" |
| 8 #include "base/bind_helpers.h" |
| 9 #include "base/command_line.h" |
| 10 #include "base/files/file_path.h" |
| 11 #include "base/files/file_util.h" |
| 12 #include "base/files/important_file_writer.h" |
| 13 #include "base/macros.h" |
| 14 #include "base/memory/ptr_util.h" |
| 15 #include "base/metrics/histogram_macros.h" |
| 16 #include "base/profiler/scoped_tracker.h" |
| 17 #include "base/sequenced_task_runner.h" |
| 18 #include "base/strings/string_util.h" |
| 19 #include "base/task_scheduler/post_task.h" |
| 20 #include "base/threading/sequenced_task_runner_handle.h" |
| 21 #include "components/translate/core/browser/proto/ranker_model.pb.h" |
| 22 #include "components/translate/core/browser/ranker_model.h" |
| 23 #include "components/translate/core/browser/translate_url_fetcher.h" |
| 24 #include "url/gurl.h" |
| 25 |
| 26 namespace translate { |
| 27 namespace { |
| 28 |
| 29 using chrome_intelligence::RankerModel; |
| 30 using chrome_intelligence::RankerModelProto; |
| 31 |
| 32 constexpr int kUrlFetcherId = 2; |
| 33 |
| 34 // The maximum number of model download attempts to make. Download may fail |
| 35 // due to server error or network availability issues. |
| 36 constexpr int kMaxRetryOn5xx = 8; |
| 37 |
| 38 // The minimum duration, in minutes, between download attempts. |
| 39 constexpr int kMinRetryDelayMins = 3; |
| 40 |
| 41 // Suffixes for the various histograms produced by the backend. |
| 42 const char kWriteTimerHistogram[] = ".Timer.WriteModel"; |
| 43 const char kReadTimerHistogram[] = ".Timer.ReadModel"; |
| 44 const char kDownloadTimerHistogram[] = ".Timer.DownloadModel"; |
| 45 const char kParsetimerHistogram[] = ".Timer.ParseModel"; |
| 46 const char kModelStatusHistogram[] = ".Model.Status"; |
| 47 |
| 48 // A helper class to produce a scoped timer histogram that supports using a |
| 49 // non-static-const name. |
| 50 class MyScopedHistogramTimer { |
| 51 public: |
| 52 MyScopedHistogramTimer(const base::StringPiece& name) |
| 53 : name_(name.begin(), name.end()), start_(base::TimeTicks::Now()) {} |
| 54 |
| 55 ~MyScopedHistogramTimer() { |
| 56 base::TimeDelta duration = base::TimeTicks::Now() - start_; |
| 57 base::HistogramBase* counter = base::Histogram::FactoryTimeGet( |
| 58 name_, base::TimeDelta::FromMilliseconds(10), |
| 59 base::TimeDelta::FromMilliseconds(200000), 100, |
| 60 base::HistogramBase::kUmaTargetedHistogramFlag); |
| 61 if (counter) |
| 62 counter->AddTime(duration); |
| 63 } |
| 64 |
| 65 private: |
| 66 const std::string name_; |
| 67 const base::TimeTicks start_; |
| 68 |
| 69 DISALLOW_COPY_AND_ASSIGN(MyScopedHistogramTimer); |
| 70 }; |
| 71 |
| 72 } // namespace |
| 73 |
| 74 // ============================================================================= |
| 75 // RankerModelLoader::Backend |
| 76 |
| 77 class RankerModelLoader::Backend { |
| 78 public: |
| 79 // An internal version of RankerModelLoader::OnModelAvailableCallback that |
| 80 // bundles calling the real callback with a notification of whether or not |
| 81 // tha backend is finished. |
| 82 using InternalOnModelAvailableCallback = |
| 83 base::Callback<void(std::unique_ptr<RankerModel>, bool)>; |
| 84 |
| 85 Backend(const ValidateModelCallback& validate_model_cb, |
| 86 const InternalOnModelAvailableCallback& on_model_available_cb, |
| 87 const base::FilePath& model_path, |
| 88 const GURL& model_url, |
| 89 const std::string& uma_prefix); |
| 90 ~Backend(); |
| 91 |
| 92 // Reads the model from |model_path_|. |
| 93 void LoadFromFile(); |
| 94 |
| 95 // Reads the model from |model_url_|. |
| 96 void AsyncLoadFromURL(); |
| 97 |
| 98 private: |
| 99 // Log and return the result of loading a model to UMA. |
| 100 RankerModelStatus ReportModelStatus(RankerModelStatus model_status); |
| 101 |
| 102 // Constructs a model from the given |data|. |
| 103 std::unique_ptr<chrome_intelligence::RankerModel> CreateModel( |
| 104 const std::string& data); |
| 105 |
| 106 // Accepts downloaded model data. This signature is mandated by the callback |
| 107 // defined by TransalteURLFetcher. |
| 108 // |
| 109 // id - the id given to the TranslateURLFetcher on creation |
| 110 // success - true of the download was successful |
| 111 // data - the body of the downloads response |
| 112 void OnDownloadComplete(int id, bool success, const std::string& data); |
| 113 |
| 114 // Transfers ownership of |model| to the client using the |
| 115 // |internal_on_model_available_cb_|. |is_finished| denotes whether the |
| 116 // backend is finished (or has given up on) loading the model. |
| 117 void TransferModelToClient( |
| 118 std::unique_ptr<chrome_intelligence::RankerModel> model, |
| 119 bool is_finished); |
| 120 |
| 121 // Validates that ranker model loader backend tasks are all performed on the |
| 122 // same sequence. |
| 123 base::SequenceChecker sequence_checker_; |
| 124 |
| 125 // The TaskRunner on which |this| was constructed. |
| 126 const scoped_refptr<base::SequencedTaskRunner> origin_task_runner_; |
| 127 |
| 128 // Validates a ranker model on behalf of the model loader client. This may |
| 129 // be called on any sequence and must, therefore, be thread-safe. |
| 130 const ValidateModelCallback validate_model_cb_; |
| 131 |
| 132 // Transfers ownership of a loaded model back to the model loader client. |
| 133 // This will be called on the sequence on which the model loader was |
| 134 // constructed. |
| 135 const InternalOnModelAvailableCallback internal_on_model_available_cb_; |
| 136 |
| 137 // The path at which the model is (or should be) cached. |
| 138 const base::FilePath model_path_; |
| 139 |
| 140 // The URL from which to download the model if the model is not in the cache |
| 141 // or the cached model is invalid/expired. |
| 142 const GURL model_url_; |
| 143 |
| 144 // This will prefix all UMA metrics generated by the model loader. |
| 145 const std::string uma_prefix_; |
| 146 |
| 147 // Used to download model data from |model_url_|. |
| 148 // TODO(rogerm): Use net::URLFetcher directly? |
| 149 std::unique_ptr<TranslateURLFetcher> url_fetcher_; |
| 150 |
| 151 // The next time before which no new attempts to download the model should be |
| 152 // attempted. |
| 153 base::TimeTicks next_earliest_download_time_; |
| 154 |
| 155 // Tracks the last time of the last attempt to download a model. Used for UMA |
| 156 // reporting of download duration. |
| 157 base::TimeTicks download_start_time_; |
| 158 |
| 159 DISALLOW_COPY_AND_ASSIGN(Backend); |
| 160 }; |
| 161 |
| 162 RankerModelLoader::Backend::Backend( |
| 163 const ValidateModelCallback& validate_model_cb, |
| 164 const InternalOnModelAvailableCallback& internal_on_model_available_cb, |
| 165 const base::FilePath& model_path, |
| 166 const GURL& model_url, |
| 167 const std::string& uma_prefix) |
| 168 : origin_task_runner_(base::SequencedTaskRunnerHandle::Get()), |
| 169 validate_model_cb_(validate_model_cb), |
| 170 internal_on_model_available_cb_(internal_on_model_available_cb), |
| 171 model_path_(model_path), |
| 172 model_url_(model_url), |
| 173 uma_prefix_(uma_prefix) { |
| 174 sequence_checker_.DetachFromSequence(); |
| 175 } |
| 176 |
| 177 RankerModelLoader::Backend::~Backend() {} |
| 178 |
| 179 RankerModelStatus RankerModelLoader::Backend::ReportModelStatus( |
| 180 RankerModelStatus model_status) { |
| 181 base::HistogramBase* histogram = base::LinearHistogram::FactoryGet( |
| 182 uma_prefix_ + kModelStatusHistogram, 1, |
| 183 static_cast<int>(RankerModelStatus::MAX), |
| 184 static_cast<int>(RankerModelStatus::MAX) + 1, |
| 185 base::HistogramBase::kUmaTargetedHistogramFlag); |
| 186 if (histogram) |
| 187 histogram->Add(static_cast<int>(model_status)); |
| 188 return model_status; |
| 189 } |
| 190 |
| 191 std::unique_ptr<chrome_intelligence::RankerModel> |
| 192 RankerModelLoader::Backend::CreateModel(const std::string& data) { |
| 193 DCHECK(sequence_checker_.CalledOnValidSequence()); |
| 194 MyScopedHistogramTimer timer(uma_prefix_ + kParsetimerHistogram); |
| 195 auto model = RankerModel::FromString(data); |
| 196 if (ReportModelStatus(model ? validate_model_cb_.Run(*model) |
| 197 : RankerModelStatus::PARSE_FAILED) != |
| 198 RankerModelStatus::OK) { |
| 199 return nullptr; |
| 200 } |
| 201 return model; |
| 202 } |
| 203 |
| 204 void RankerModelLoader::Backend::LoadFromFile() { |
| 205 DCHECK(sequence_checker_.CalledOnValidSequence()); |
| 206 |
| 207 // If there is not cache path set, move on to loading the model by URL. |
| 208 if (model_path_.empty()) { |
| 209 AsyncLoadFromURL(); |
| 210 return; |
| 211 } |
| 212 |
| 213 DVLOG(2) << "Attempting to load model from: " << model_path_.value(); |
| 214 |
| 215 std::string data; |
| 216 |
| 217 { |
| 218 MyScopedHistogramTimer timer(uma_prefix_ + kReadTimerHistogram); |
| 219 if (!base::ReadFileToString(model_path_, &data) || data.empty()) { |
| 220 DVLOG(2) << "Failed to read model from: " << model_path_.value(); |
| 221 data.clear(); |
| 222 } |
| 223 } |
| 224 |
| 225 // If model data was loaded, check if it can be parsed to a valid model. |
| 226 if (!data.empty()) { |
| 227 auto model = CreateModel(data); |
| 228 if (model) { |
| 229 // The model is valid. The client is willing/able to use it. Keep track |
| 230 // of where it originated and whether or not is has expired. |
| 231 std::string url_spec = model->GetSourceURL(); |
| 232 bool is_expired = model->IsExpired(); |
| 233 bool is_finished = url_spec == model_url_.spec() && !is_expired; |
| 234 |
| 235 DVLOG(2) << (is_expired ? "Expired m" : "M") << "odel in '" |
| 236 << model_path_.value() << "' was originally downloaded from '" |
| 237 << url_spec << "'."; |
| 238 |
| 239 // Transfer the model to the client. Beyond this line, |model| is invalid. |
| 240 TransferModelToClient(std::move(model), is_finished); |
| 241 |
| 242 // If the cached model came from currently configured |model_url_| and has |
| 243 // not expired, there is no need schedule a model download. |
| 244 if (is_finished) |
| 245 return; |
| 246 |
| 247 // Otherwise, fall out of this block to schedule a download. The client |
| 248 // can continue to use the valid but expired model until the download |
| 249 // completes. |
| 250 } |
| 251 } |
| 252 |
| 253 // Reaching this point means that a model download is required. If there is |
| 254 // no download URL configured, then there is nothing further to do. |
| 255 AsyncLoadFromURL(); |
| 256 } |
| 257 |
| 258 void RankerModelLoader::Backend::AsyncLoadFromURL() { |
| 259 DCHECK(sequence_checker_.CalledOnValidSequence()); |
| 260 |
| 261 if (!model_url_.is_valid()) |
| 262 return; |
| 263 |
| 264 // Do nothing if the download attempts should be throttled. |
| 265 if (base::TimeTicks::Now() < next_earliest_download_time_) { |
| 266 DVLOG(2) << "Last download attempt was too recent."; |
| 267 return; |
| 268 } |
| 269 |
| 270 // Otherwise, initialize the model fetcher to be non-null and trigger an |
| 271 // initial download attempt. |
| 272 if (!url_fetcher_) { |
| 273 url_fetcher_ = base::MakeUnique<TranslateURLFetcher>(kUrlFetcherId); |
| 274 url_fetcher_->set_max_retry_on_5xx(kMaxRetryOn5xx); |
| 275 } |
| 276 |
| 277 // If a request is already in flight, do not issue a new one. |
| 278 if (url_fetcher_->state() == TranslateURLFetcher::REQUESTING) { |
| 279 DVLOG(2) << "Download is in progress."; |
| 280 return; |
| 281 } |
| 282 |
| 283 DVLOG(2) << "Downloading model from: " << model_url_; |
| 284 |
| 285 // Reset the time of the next earliest allowable download attempt. |
| 286 next_earliest_download_time_ = |
| 287 base::TimeTicks::Now() + base::TimeDelta::FromMinutes(kMinRetryDelayMins); |
| 288 |
| 289 // Kick off the next download attempt. |
| 290 download_start_time_ = base::TimeTicks::Now(); |
| 291 bool result = url_fetcher_->Request( |
| 292 model_url_, base::Bind(&RankerModelLoader::Backend::OnDownloadComplete, |
| 293 base::Unretained(this))); |
| 294 |
| 295 // The maximum number of download attempts has been surpassed. Don't make |
| 296 // any further attempts. |
| 297 if (!result) { |
| 298 DVLOG(2) << "Model download abandoned."; |
| 299 ReportModelStatus(RankerModelStatus::DOWNLOAD_FAILED); |
| 300 url_fetcher_.reset(); |
| 301 |
| 302 // Notify the loader that model loading has been abandoned. |
| 303 TransferModelToClient(nullptr, true); |
| 304 } |
| 305 } |
| 306 |
| 307 void RankerModelLoader::Backend::OnDownloadComplete(int /* id */, |
| 308 bool success, |
| 309 const std::string& data) { |
| 310 DCHECK(sequence_checker_.CalledOnValidSequence()); |
| 311 |
| 312 // Record the duration of the download. |
| 313 base::TimeDelta duration = base::TimeTicks::Now() - download_start_time_; |
| 314 base::HistogramBase* counter = base::Histogram::FactoryTimeGet( |
| 315 uma_prefix_ + kDownloadTimerHistogram, |
| 316 base::TimeDelta::FromMilliseconds(10), |
| 317 base::TimeDelta::FromMilliseconds(200000), 100, |
| 318 base::HistogramBase::kUmaTargetedHistogramFlag); |
| 319 if (counter) |
| 320 counter->AddTime(duration); |
| 321 |
| 322 // On failure, we just abort. The TranslateRanker will retry on a subsequent |
| 323 // translation opportunity. The TranslateURLFetcher enforces a limit for |
| 324 // retried requests. |
| 325 if (!success || data.empty()) { |
| 326 DVLOG(2) << "Download from '" << model_url_ << "'' failed."; |
| 327 return; |
| 328 } |
| 329 |
| 330 auto model = CreateModel(data); |
| 331 if (!model) { |
| 332 DVLOG(2) << "Model from '" << model_url_ << "'' not valid."; |
| 333 return; |
| 334 } |
| 335 |
| 336 url_fetcher_.reset(); |
| 337 |
| 338 auto* metadata = model->mutable_proto()->mutable_metadata(); |
| 339 metadata->set_source(model_url_.spec()); |
| 340 metadata->set_last_modified_sec( |
| 341 (base::Time::Now() - base::Time()).InSeconds()); |
| 342 |
| 343 if (!model_path_.empty()) { |
| 344 DVLOG(2) << "Saving model from '" << model_url_ << "'' to '" |
| 345 << model_path_.value() << "'."; |
| 346 MyScopedHistogramTimer timer(uma_prefix_ + kWriteTimerHistogram); |
| 347 base::ImportantFileWriter::WriteFileAtomically(model_path_, |
| 348 model->SerializeAsString()); |
| 349 } |
| 350 |
| 351 // Notify the owner that a compatible model is available. |
| 352 TransferModelToClient(std::move(model), true); |
| 353 } |
| 354 |
| 355 void RankerModelLoader::Backend::TransferModelToClient( |
| 356 std::unique_ptr<chrome_intelligence::RankerModel> model, |
| 357 bool is_finished) { |
| 358 DCHECK(sequence_checker_.CalledOnValidSequence()); |
| 359 origin_task_runner_->PostTask( |
| 360 FROM_HERE, base::Bind(internal_on_model_available_cb_, |
| 361 base::Passed(std::move(model)), is_finished)); |
| 362 } |
| 363 |
| 364 // ============================================================================= |
| 365 // RankerModelLoader |
| 366 |
| 367 RankerModelLoader::RankerModelLoader( |
| 368 const ValidateModelCallback& validate_model_cb, |
| 369 const OnModelAvailableCallback& on_model_available_cb, |
| 370 const base::FilePath& model_path, |
| 371 const GURL& model_url, |
| 372 const std::string& uma_prefix) |
| 373 : backend_task_runner_(base::CreateSequencedTaskRunnerWithTraits( |
| 374 base::TaskTraits().MayBlock().WithShutdownBehavior( |
| 375 base::TaskShutdownBehavior::SKIP_ON_SHUTDOWN))), |
| 376 weak_ptr_factory_(this) { |
| 377 auto internal_on_model_available_cb = |
| 378 base::Bind(&RankerModelLoader::InternalOnModelAvailable, |
| 379 weak_ptr_factory_.GetWeakPtr(), on_model_available_cb); |
| 380 backend_ = base::MakeUnique<Backend>(validate_model_cb, |
| 381 internal_on_model_available_cb, |
| 382 model_path, model_url, uma_prefix); |
| 383 } |
| 384 |
| 385 RankerModelLoader::~RankerModelLoader() { |
| 386 DCHECK(sequence_checker_.CalledOnValidSequence()); |
| 387 // This is guaranteed to be sequenced after any pending backend operation. |
| 388 backend_task_runner_->DeleteSoon(FROM_HERE, backend_.release()); |
| 389 } |
| 390 |
| 391 void RankerModelLoader::Start() { |
| 392 DCHECK(sequence_checker_.CalledOnValidSequence()); |
| 393 DCHECK_EQ(state_, LoaderState::NOT_STARTED); |
| 394 state_ = LoaderState::RUNNING; |
| 395 backend_task_runner_->PostTask( |
| 396 FROM_HERE, base::Bind(&RankerModelLoader::Backend::LoadFromFile, |
| 397 base::Unretained(backend_.get()))); |
| 398 } |
| 399 |
| 400 void RankerModelLoader::NotifyOfRankerActivity() { |
| 401 DCHECK(sequence_checker_.CalledOnValidSequence()); |
| 402 if (state_ == LoaderState::RUNNING) { |
| 403 backend_task_runner_->PostTask( |
| 404 FROM_HERE, base::Bind(&RankerModelLoader::Backend::AsyncLoadFromURL, |
| 405 base::Unretained(backend_.get()))); |
| 406 } |
| 407 } |
| 408 |
| 409 void RankerModelLoader::InternalOnModelAvailable( |
| 410 const OnModelAvailableCallback& callback, |
| 411 std::unique_ptr<RankerModel> model, |
| 412 bool finished) { |
| 413 DCHECK(sequence_checker_.CalledOnValidSequence()); |
| 414 if (finished) |
| 415 state_ = LoaderState::FINISHED; |
| 416 if (model) |
| 417 callback.Run(std::move(model)); |
| 418 } |
| 419 |
| 420 } // namespace translate |
OLD | NEW |