OLD | NEW |
---|---|
(Empty) | |
1 // Copyright 2016 The Chromium Authors. All rights reserved. | |
2 // Use of this source code is governed by a BSD-style license that can be | |
3 // found in the LICENSE file. | |
4 | |
5 #include "components/translate/core/browser/ranker_model_loader.h" | |
6 | |
7 #include "base/bind.h" | |
8 #include "base/bind_helpers.h" | |
9 #include "base/command_line.h" | |
10 #include "base/files/file_path.h" | |
fdoray
2017/02/23 16:41:25
Already included in the .h file.
Roger McFarlane (Chromium)
2017/02/23 21:17:56
Done.
| |
11 #include "base/files/file_util.h" | |
12 #include "base/files/important_file_writer.h" | |
13 #include "base/macros.h" | |
14 #include "base/memory/ptr_util.h" | |
15 #include "base/memory/ref_counted.h" | |
fdoray
2017/02/23 16:41:25
Already included in the .h file.
Roger McFarlane (Chromium)
2017/02/23 21:17:56
Done.
| |
16 #include "base/metrics/histogram_macros.h" | |
17 #include "base/profiler/scoped_tracker.h" | |
18 #include "base/strings/string_util.h" | |
19 #include "base/task_scheduler/post_task.h" | |
20 #include "base/threading/sequenced_task_runner_handle.h" | |
21 #include "components/translate/core/browser/proto/ranker_model.pb.h" | |
22 #include "components/translate/core/browser/ranker_model.h" | |
23 #include "components/translate/core/browser/translate_url_fetcher.h" | |
24 #include "components/translate/core/common/translate_switches.h" | |
25 #include "components/variations/variations_associated_data.h" | |
26 | |
27 namespace translate { | |
28 namespace { | |
29 | |
30 using chrome_intelligence::RankerModel; | |
31 using chrome_intelligence::RankerModelProto; | |
32 | |
33 constexpr int kUrlFetcherId = 2; | |
34 | |
35 // The maximum number of model download attempts to make. Download may fail | |
36 // due to server error or network availability issues. | |
37 constexpr int kMaxRetryOn5xx = 8; | |
38 | |
39 // The minimum duration, in minutes, between download attempts. | |
40 constexpr int kMinRetryDelayMins = 3; | |
41 | |
42 // Suffixes for the various histograms produced by the backend. | |
43 const char kWriteTimerHistogram[] = ".Timer.WriteModel"; | |
44 const char kReadTimerHistogram[] = ".Timer.ReadModel"; | |
45 const char kDownloadTimerHistogram[] = ".Timer.DownloadModel"; | |
46 const char kParsetimerHistogram[] = ".Timer.ParseModel"; | |
47 const char kModelStatusHistogram[] = ".Model.Status"; | |
48 | |
49 // A helper class to produce a scoped timer histogram that supports using a | |
50 // non-static-const name. | |
51 class MyScopedHistogramTimer { | |
52 public: | |
53 MyScopedHistogramTimer(const base::StringPiece& name) | |
54 : name_(name.begin(), name.end()), start_(base::TimeTicks::Now()) {} | |
55 | |
56 ~MyScopedHistogramTimer() { | |
57 base::TimeDelta duration = base::TimeTicks::Now() - start_; | |
58 base::HistogramBase* counter = base::Histogram::FactoryTimeGet( | |
59 name_, base::TimeDelta::FromMilliseconds(10), | |
60 base::TimeDelta::FromMilliseconds(200000), 100, | |
61 base::HistogramBase::kUmaTargetedHistogramFlag); | |
62 if (counter) | |
63 counter->AddTime(duration); | |
64 } | |
65 | |
66 private: | |
67 const std::string name_; | |
68 const base::TimeTicks start_; | |
69 | |
70 DISALLOW_COPY_AND_ASSIGN(MyScopedHistogramTimer); | |
71 }; | |
72 | |
73 } // namespace | |
74 | |
75 // ============================================================================= | |
76 // RankerModelLoader::Backend | |
77 | |
78 class RankerModelLoader::Backend { | |
79 public: | |
80 // An internal version of RankerModelLoader::OnModelAvailableCallback that | |
81 // bundles calling the real callback with a notification of whether or not | |
82 // tha backend is finished. | |
83 using InternalOnModelAvailableCallback = | |
84 base::Callback<void(std::unique_ptr<RankerModel>, bool)>; | |
85 | |
86 Backend(const ValidateModelCallback& validate_model_cb, | |
87 const InternalOnModelAvailableCallback& on_model_available_cb, | |
88 const base::FilePath& model_path, | |
89 const GURL& model_url, | |
90 const std::string& uma_prefix); | |
91 ~Backend(); | |
92 | |
93 // Reads the model from |model_path_|. | |
94 void LoadFromFile(); | |
95 | |
96 // Reads the model from |model_url_|. | |
97 void AsyncLoadFromURL(); | |
98 | |
99 private: | |
100 // Log and return the result of loading a model to UMA. | |
101 RankerModelStatus ReportModelStatus(RankerModelStatus model_status); | |
102 | |
103 // Constructs a model from the given |data|. | |
104 std::unique_ptr<chrome_intelligence::RankerModel> CreateModel( | |
105 const std::string& data); | |
106 | |
107 // Accepts downloaded model data. This signature is mandated by the callback | |
108 // defined by TransalteURLFetcher. | |
109 // | |
110 // id - the id given to the TranslateURLFetcher on creation | |
111 // success - true of the download was successful | |
112 // data - the body of the downloads response | |
113 void OnDownloadComplete(int id, bool success, const std::string& data); | |
114 | |
115 // Transfers ownership of |model| to the client using the | |
116 // |internaL_on_model_available_cb_|. |is_finished| denotes whether the | |
fdoray
2017/02/23 16:41:25
L -> l
Roger McFarlane (Chromium)
2017/02/23 21:17:56
Done.
| |
117 // backend is finished (or has given up on) loading the model. | |
118 void TransferModelToClient( | |
119 std::unique_ptr<chrome_intelligence::RankerModel> model, | |
120 bool is_finished); | |
121 | |
122 // Validates that ranker model loader backend tasks are all performed on the | |
123 // correct sequence. | |
fdoray
2017/02/23 16:41:25
*same* sequence?
Roger McFarlane (Chromium)
2017/02/23 21:17:56
Done.
| |
124 base::SequenceChecker sequence_checker_; | |
125 | |
126 // The TaskRunner on which |this| was constructed. | |
127 const scoped_refptr<base::SequencedTaskRunner> origin_task_runner_; | |
128 | |
129 // Validates a ranker model on behalf of the model loader client. This may | |
130 // be called on any sequence and must, therefore, be thread-safe. | |
131 const ValidateModelCallback validate_model_cb_; | |
132 | |
133 // Transfers ownership of a loaded model back to the model loader client. | |
134 // This will be called on the sequence on which the model loader was | |
135 // constructed. | |
136 const InternalOnModelAvailableCallback internal_on_model_available_cb_; | |
137 | |
138 // The path at which the model is (or should be) cached. | |
139 const base::FilePath model_path_; | |
140 | |
141 // The URL from which to download the model if the model is not in the cache | |
142 // or the cached model is invalid/expired. | |
143 const GURL model_url_; | |
144 | |
145 // This will prefix all UMA metrics generated by the model loader. | |
146 const std::string uma_prefix_; | |
147 | |
148 // Used to download model data from |model_url_|. | |
149 // TODO(rogerm): Use net::URLFetcher directly? | |
150 std::unique_ptr<TranslateURLFetcher> url_fetcher_; | |
151 | |
152 // The next time before which no new attempts to download the model should be | |
153 // attempted. | |
154 base::TimeTicks next_earliest_download_time_; | |
155 | |
156 // Tracks the last time of the last attempt to download a model. Used for UMA | |
157 // reporting of download duration. | |
158 base::TimeTicks download_start_time_; | |
159 | |
160 DISALLOW_COPY_AND_ASSIGN(Backend); | |
161 }; | |
162 | |
163 RankerModelLoader::Backend::Backend( | |
164 const ValidateModelCallback& validate_model_cb, | |
165 const InternalOnModelAvailableCallback& internal_on_model_available_cb, | |
166 const base::FilePath& model_path, | |
167 const GURL& model_url, | |
168 const std::string& uma_prefix) | |
169 : origin_task_runner_(base::SequencedTaskRunnerHandle::Get()), | |
170 validate_model_cb_(validate_model_cb), | |
171 internal_on_model_available_cb_(internal_on_model_available_cb), | |
172 model_path_(model_path), | |
173 model_url_(model_url), | |
174 uma_prefix_(uma_prefix) { | |
175 sequence_checker_.DetachFromSequence(); | |
176 } | |
177 | |
178 RankerModelLoader::Backend::~Backend() {} | |
179 | |
180 RankerModelStatus RankerModelLoader::Backend::ReportModelStatus( | |
181 RankerModelStatus model_status) { | |
182 base::HistogramBase* histogram = base::LinearHistogram::FactoryGet( | |
183 uma_prefix_ + kModelStatusHistogram, 1, | |
184 static_cast<int>(RankerModelStatus::MAX), | |
185 static_cast<int>(RankerModelStatus::MAX) + 1, | |
186 base::HistogramBase::kUmaTargetedHistogramFlag); | |
187 if (histogram) | |
188 histogram->Add(static_cast<int>(model_status)); | |
189 return model_status; | |
190 } | |
191 | |
192 std::unique_ptr<chrome_intelligence::RankerModel> | |
193 RankerModelLoader::Backend::CreateModel(const std::string& data) { | |
194 DCHECK(sequence_checker_.CalledOnValidSequence()); | |
195 MyScopedHistogramTimer timer(uma_prefix_ + kParsetimerHistogram); | |
196 auto model = RankerModel::FromString(data); | |
197 if (ReportModelStatus(model ? validate_model_cb_.Run(*model) | |
198 : RankerModelStatus::PARSE_FAILED) != | |
199 RankerModelStatus::OK) { | |
200 return nullptr; | |
201 } | |
202 return model; | |
203 } | |
204 | |
205 void RankerModelLoader::Backend::LoadFromFile() { | |
206 DCHECK(sequence_checker_.CalledOnValidSequence()); | |
207 | |
208 // If there is not cache path set, move on to loading the model by URL. | |
209 if (model_path_.empty()) { | |
210 AsyncLoadFromURL(); | |
211 return; | |
212 } | |
213 | |
214 DVLOG(2) << "Attempting to load model from: " << model_path_.value(); | |
215 | |
216 std::string data; | |
217 | |
218 { | |
219 MyScopedHistogramTimer timer(uma_prefix_ + kReadTimerHistogram); | |
220 if (!base::ReadFileToString(model_path_, &data) || data.empty()) { | |
221 DVLOG(2) << "Failed to read model from: " << model_path_.value(); | |
222 return; | |
223 } | |
224 } | |
225 | |
226 // If model data was loaded, check if it can be parsed to a valid model. | |
227 auto model = CreateModel(data); | |
228 if (model) { | |
229 // The model is valid. The client is willing/able to use it. Keep track | |
230 // of where it originated and whether or not is has expired. | |
231 std::string url_spec = model->GetSourceURL(); | |
232 bool is_expired = model->IsExpired(); | |
233 bool is_finished = url_spec == model_url_.spec() && !is_expired; | |
234 | |
235 DVLOG(2) << (is_expired ? "Expired m" : "M") << "odel in '" | |
236 << model_path_.value() << "'' was originally downloaded from '" | |
237 << url_spec << "'."; | |
238 | |
239 // Transfer the model to the client. Beyond this line, |model| is invalid. | |
240 TransferModelToClient(std::move(model), is_finished); | |
241 | |
242 // If the cached model came from currently configured |model_url_| and has | |
243 // not expired, there is no need schedule a model download. | |
244 if (is_finished) | |
245 return; | |
246 | |
247 // Otherwise, fall out of this block to schedule a download. The client | |
248 // can continue to use the valid but expired model until the download | |
249 // completes. | |
250 } | |
251 | |
252 // Reaching this point means that a model download is required. If there is | |
253 // no download URL configured, then there is nothing further to do. | |
254 AsyncLoadFromURL(); | |
255 } | |
256 | |
257 void RankerModelLoader::Backend::AsyncLoadFromURL() { | |
258 DCHECK(sequence_checker_.CalledOnValidSequence()); | |
259 | |
260 if (!model_url_.is_valid()) | |
261 return; | |
262 | |
263 // Do nothing if the download attempts should be throttled. | |
264 if (base::TimeTicks::Now() < next_earliest_download_time_) { | |
265 DVLOG(2) << "Last download attempt was too recent."; | |
266 return; | |
267 } | |
268 | |
269 // Otherwise, initialize the model fetcher to be non-null and trigger an | |
270 // initial download attempt. | |
271 if (!url_fetcher_) { | |
272 url_fetcher_ = base::MakeUnique<TranslateURLFetcher>(kUrlFetcherId); | |
273 url_fetcher_->set_max_retry_on_5xx(kMaxRetryOn5xx); | |
274 } | |
275 | |
276 // If a request is already in flight, do not issue a new one. | |
277 if (url_fetcher_->state() == TranslateURLFetcher::REQUESTING) { | |
278 DVLOG(2) << "Download is in progress."; | |
279 return; | |
280 } | |
281 | |
282 DVLOG(2) << "Downloading model from: " << model_url_; | |
283 | |
284 // Reset the time of the next earliest allowable download attempt. | |
285 next_earliest_download_time_ = | |
286 base::TimeTicks::Now() + base::TimeDelta::FromMinutes(kMinRetryDelayMins); | |
287 | |
288 // Kick off the next download attempt. | |
289 download_start_time_ = base::TimeTicks::Now(); | |
290 bool result = url_fetcher_->Request( | |
291 model_url_, | |
292 base::Bind(&RankerModelLoader::Backend::OnDownloadComplete, | |
293 base::Unretained(this))); | |
294 | |
295 // The maximum number of download attempts has been surpassed. Don't make | |
296 // any further attempts. | |
297 if (!result) { | |
298 DVLOG(2) << "Model download abandoned."; | |
299 ReportModelStatus(RankerModelStatus::DOWNLOAD_FAILED); | |
300 url_fetcher_.reset(); | |
301 | |
302 // Notify the loader that model loading has been abandoned. | |
303 TransferModelToClient(nullptr, true); | |
304 } | |
305 } | |
306 | |
307 void RankerModelLoader::Backend::OnDownloadComplete(int /* id */, | |
308 bool success, | |
309 const std::string& data) { | |
310 DCHECK(sequence_checker_.CalledOnValidSequence()); | |
311 | |
312 // Record the duration of the download. | |
313 base::TimeDelta duration = base::TimeTicks::Now() - download_start_time_; | |
314 base::HistogramBase* counter = base::Histogram::FactoryTimeGet( | |
315 uma_prefix_ + kDownloadTimerHistogram, | |
316 base::TimeDelta::FromMilliseconds(10), | |
317 base::TimeDelta::FromMilliseconds(200000), 100, | |
318 base::HistogramBase::kUmaTargetedHistogramFlag); | |
319 if (counter) | |
320 counter->AddTime(duration); | |
321 | |
322 // On failure, we just abort. The TranslateRanker will retry on a subsequent | |
323 // translation opportunity. The TranslateURLFetcher enforces a limit for | |
324 // retried requests. | |
325 if (!success || data.empty()) { | |
326 DVLOG(2) << "Download from '" << model_url_ << "'' failed."; | |
327 return; | |
328 } | |
329 | |
330 auto model = CreateModel(data); | |
331 if (!model) { | |
332 DVLOG(2) << "Model from '" << model_url_ << "'' not valid."; | |
333 return; | |
334 } | |
335 | |
336 url_fetcher_.reset(); | |
337 | |
338 auto* metadata = model->mutable_proto()->mutable_metadata(); | |
339 metadata->set_source(model_url_.spec()); | |
340 metadata->set_last_modified_sec( | |
341 (base::Time::Now() - base::Time()).InSeconds()); | |
342 | |
343 if (!model_path_.empty()) { | |
344 DVLOG(2) << "Saving model from '" << model_url_ << "'' to '" | |
345 << model_path_.value() << "'."; | |
346 MyScopedHistogramTimer timer(uma_prefix_ + kWriteTimerHistogram); | |
347 base::ImportantFileWriter::WriteFileAtomically(model_path_, | |
348 model->SerializeAsString()); | |
349 } | |
350 | |
351 // Notify the owner that a compatible model is available. | |
352 TransferModelToClient(std::move(model), true); | |
353 } | |
354 | |
355 void RankerModelLoader::Backend::TransferModelToClient( | |
356 std::unique_ptr<chrome_intelligence::RankerModel> model, | |
357 bool is_finished) { | |
358 DCHECK(sequence_checker_.CalledOnValidSequence()); | |
359 origin_task_runner_->PostTask( | |
360 FROM_HERE, | |
361 base::Bind(internal_on_model_available_cb_, | |
362 base::Passed(std::move(model)), is_finished)); | |
363 } | |
364 | |
365 // ============================================================================= | |
366 // RankerModelLoader | |
367 | |
368 RankerModelLoader::RankerModelLoader( | |
369 const ValidateModelCallback& validate_model_cb, | |
370 const OnModelAvailableCallback& on_model_available_cb, | |
371 const base::FilePath& model_path, | |
372 const GURL& model_url, | |
373 const std::string& uma_prefix) | |
374 : backend_task_runner_(base::CreateSequencedTaskRunnerWithTraits( | |
375 base::TaskTraits().MayBlock().WithShutdownBehavior( | |
376 base::TaskShutdownBehavior::SKIP_ON_SHUTDOWN))), | |
377 weak_ptr_factory_(this) { | |
378 auto internal_on_model_available_cb = | |
379 base::Bind(&RankerModelLoader::InternalOnModelAvailable, | |
380 weak_ptr_factory_.GetWeakPtr(), on_model_available_cb); | |
381 backend_ = base::MakeUnique<Backend>(validate_model_cb, | |
382 internal_on_model_available_cb, | |
383 model_path, model_url, uma_prefix); | |
384 } | |
385 | |
386 RankerModelLoader::~RankerModelLoader() { | |
387 DCHECK(sequence_checker_.CalledOnValidSequence()); | |
388 // This is guaranteed to be sequenced after any pending backend operation. | |
389 backend_task_runner_->DeleteSoon(FROM_HERE, backend_.release()); | |
390 } | |
391 | |
392 void RankerModelLoader::Start() { | |
393 DCHECK(sequence_checker_.CalledOnValidSequence()); | |
394 DCHECK_EQ(state_, LoaderState::NOT_STARTED); | |
395 state_ = LoaderState::RUNNING; | |
396 backend_task_runner_->PostTask( | |
397 FROM_HERE, | |
398 base::Bind(&RankerModelLoader::Backend::LoadFromFile, | |
399 base::Unretained(backend_.get()))); | |
400 } | |
401 | |
402 void RankerModelLoader::NotifyOfRankerActivity() { | |
403 DCHECK(sequence_checker_.CalledOnValidSequence()); | |
404 if (state_ == LoaderState::RUNNING) { | |
405 backend_task_runner_->PostTask( | |
406 FROM_HERE, | |
407 base::Bind(&RankerModelLoader::Backend::AsyncLoadFromURL, | |
408 base::Unretained(backend_.get()))); | |
409 } | |
410 } | |
411 | |
412 void RankerModelLoader::InternalOnModelAvailable( | |
413 const OnModelAvailableCallback& callback, | |
414 std::unique_ptr<RankerModel> model, | |
415 bool finished) { | |
416 DCHECK(sequence_checker_.CalledOnValidSequence()); | |
417 if (finished) | |
418 state_ = LoaderState::FINISHED; | |
419 if (model) | |
420 callback.Run(std::move(model)); | |
421 } | |
422 | |
423 } // namespace translate | |
OLD | NEW |