OLD | NEW |
---|---|
(Empty) | |
1 // Copyright 2016 The Chromium Authors. All rights reserved. | |
2 // Use of this source code is governed by a BSD-style license that can be | |
3 // found in the LICENSE file. | |
4 | |
5 #include "components/translate/core/browser/ranker_model_loader.h" | |
6 | |
7 #include "base/bind.h" | |
8 #include "base/bind_helpers.h" | |
9 #include "base/command_line.h" | |
10 #include "base/files/file_path.h" | |
11 #include "base/files/file_util.h" | |
12 #include "base/files/important_file_writer.h" | |
13 #include "base/memory/ptr_util.h" | |
14 #include "base/memory/ref_counted.h" | |
15 #include "base/metrics/histogram_macros.h" | |
16 #include "base/profiler/scoped_tracker.h" | |
17 #include "base/strings/string_number_conversions.h" | |
18 #include "base/strings/string_util.h" | |
19 #include "base/task_runner.h" | |
20 #include "base/threading/platform_thread.h" | |
21 #include "components/translate/core/browser/translate_url_fetcher.h" | |
22 #include "components/translate/core/common/translate_switches.h" | |
23 #include "components/variations/variations_associated_data.h" | |
24 | |
25 namespace { | |
26 | |
groby-ooo-7-16
2017/01/23 21:41:56
Any tests for the loader?
Roger McFarlane (Chromium)
2017/02/08 23:08:09
Coming.
| |
27 class MyScopedHistogramTimer { | |
groby-ooo-7-16
2017/01/23 21:41:56
Why not SCOPED_UMA_HISTOGRAM_TIMER?
Roger McFarlane (Chromium)
2017/02/08 23:08:09
The UMA macros require static const data for the m
| |
28 public: | |
29 MyScopedHistogramTimer(const base::StringPiece& name) | |
30 : name_(name.begin(), name.end()), start_(base::TimeTicks::Now()) {} | |
31 | |
32 ~MyScopedHistogramTimer() { | |
33 base::TimeDelta duration = base::TimeTicks::Now() - start_; | |
34 base::HistogramBase* counter = base::Histogram::FactoryTimeGet( | |
35 name_, base::TimeDelta::FromMilliseconds(10), | |
36 base::TimeDelta::FromMilliseconds(200000), 100, | |
37 base::HistogramBase::kUmaTargetedHistogramFlag); | |
38 if (counter) | |
39 counter->AddTime(duration); | |
40 } | |
41 | |
42 private: | |
43 const std::string name_; | |
44 const base::TimeTicks start_; | |
45 }; | |
46 | |
47 bool IsExpired(const chrome_intelligence::RankerModel& model) { | |
groby-ooo-7-16
2017/01/23 21:41:56
Why is "IsExpired" not a member on the model? It p
Roger McFarlane (Chromium)
2017/02/08 23:08:09
RankerModel is a proto.
I could rename the proto
| |
48 DCHECK(model.has_metadata()); | |
49 const auto& metadata = model.metadata(); | |
50 | |
51 // If the age of the model cannot be determined, presume it to be expired. | |
52 if (!metadata.has_last_modified_sec()) | |
53 return true; | |
54 | |
55 // If the model has no set cache duration, then it never expires. | |
56 if (!metadata.has_cache_duration_sec() || metadata.cache_duration_sec() == 0) | |
57 return false; | |
58 | |
59 // Otherwise, a model is expired if its age exceeds the cache duration. | |
60 base::Time last_modified = | |
61 base::Time() + base::TimeDelta::FromSeconds(metadata.last_modified_sec()); | |
62 base::TimeDelta age = base::Time::Now() - last_modified; | |
63 return age > base::TimeDelta::FromSeconds(metadata.cache_duration_sec()); | |
64 } | |
65 | |
66 } // namespace | |
67 | |
68 namespace translate { | |
69 | |
70 const int kUrlFetcherId = 2; | |
71 const char kWriteTimerHistogram[] = ".Timer.WriteModel"; | |
72 const char kReadTimerHistogram[] = ".Timer.ReadModel"; | |
73 const char kDownloadTimerHistogram[] = ".Timer.DownloadModel"; | |
74 const char kParsetimerHistogram[] = ".Timer.ParseModel"; | |
75 const char kModelStatusHistogram[] = ".Model.Status"; | |
76 | |
77 RankerModelLoader::RankerModelLoader(RankerModelObserver* observer, | |
78 const base::FilePath& model_path, | |
79 const GURL& model_url, | |
80 const std::string& uma_prefix) | |
81 : thread_(uma_prefix + ".LoaderThread"), | |
82 observer_(observer), | |
83 model_path_(model_path), | |
84 model_url_(model_url), | |
85 uma_prefix_(uma_prefix) { | |
86 DCHECK(observer_ != nullptr); | |
87 sequence_checker_.DetachFromSequence(); | |
88 | |
89 base::Thread::Options options; | |
90 options.message_loop_type = base::MessageLoop::TYPE_IO; | |
groby-ooo-7-16
2017/01/23 21:41:56
Does this need a separate thread? Can we use a Wor
Roger McFarlane (Chromium)
2017/02/08 23:08:09
Ideally, I'd use the task scheduler api and just g
| |
91 options.priority = base::ThreadPriority::BACKGROUND; | |
92 thread_.StartWithOptions(options); | |
93 } | |
94 | |
95 RankerModelLoader::~RankerModelLoader() {} | |
96 | |
97 void RankerModelLoader::Start() { | |
98 thread_.task_runner()->PostTask( | |
99 FROM_HERE, | |
100 base::Bind(&RankerModelLoader::LoadFromFile, base::Unretained(this))); | |
101 } | |
102 | |
103 void RankerModelLoader::NotifyOfNetworkAvailability() { | |
104 if (url_fetcher_) { | |
105 thread_.task_runner()->PostTask( | |
106 FROM_HERE, | |
107 base::Bind(&RankerModelLoader::LoadFromURL, base::Unretained(this))); | |
108 } | |
109 } | |
110 | |
111 void RankerModelLoader::ReportModelStatus(RankerModelStatus model_status) { | |
112 base::HistogramBase* histogram = base::LinearHistogram::FactoryGet( | |
groby-ooo-7-16
2017/01/23 21:41:56
Why do we need to do this manually, instead of via
Roger McFarlane (Chromium)
2017/02/08 23:08:09
The macros require static const histogram names.
| |
113 uma_prefix_ + kModelStatusHistogram, 1, | |
114 static_cast<int>(RankerModelStatus::MAX), | |
groby-ooo-7-16
2017/01/23 21:41:56
Why the cast? IIRC the standard says underlying ty
Roger McFarlane (Chromium)
2017/02/08 23:08:09
clang++ fails to resolve the conversion from the e
| |
115 static_cast<int>(RankerModelStatus::MAX) + 1, | |
116 base::HistogramBase::kUmaTargetedHistogramFlag); | |
117 if (histogram) | |
118 histogram->Add(static_cast<int>(model_status)); | |
119 } | |
120 | |
121 std::unique_ptr<chrome_intelligence::RankerModel> RankerModelLoader::Parse( | |
122 const std::string& data) { | |
123 DCHECK(sequence_checker_.CalledOnValidSequence()); | |
124 MyScopedHistogramTimer timer(uma_prefix_ + kParsetimerHistogram); | |
125 | |
126 auto model = base::MakeUnique<chrome_intelligence::RankerModel>(); | |
127 | |
128 if (!model->ParseFromString(data)) { | |
129 ReportModelStatus(RankerModelStatus::PARSE_FAILED); | |
130 return nullptr; | |
131 } | |
132 | |
133 RankerModelStatus model_status = observer_->Validate(*model); | |
134 ReportModelStatus(model_status); | |
135 if (model_status != RankerModelStatus::OK) | |
136 return nullptr; | |
137 | |
138 return model; | |
139 } | |
140 | |
141 void RankerModelLoader::LoadFromFile() { | |
142 DCHECK(sequence_checker_.CalledOnValidSequence()); | |
143 // Attempt to read the model data from the cache file. | |
144 std::string data; | |
145 if (!model_path_.empty()) { | |
146 DVLOG(2) << "Attempting to load model from: " << model_path_.value(); | |
147 MyScopedHistogramTimer timer(uma_prefix_ + kReadTimerHistogram); | |
148 if (!base::ReadFileToString(model_path_, &data)) { | |
149 DVLOG(2) << "Failed to read model from: " << model_path_.value(); | |
150 data.clear(); | |
151 } | |
152 } | |
153 | |
154 // If the model was successfully was read and is compatible, then notify | |
155 // the "owner" of this model loader of the models availability (transferring | |
156 // ownership of the model). If the model originated at the model_url_ then | |
157 // the model is also up to date, no need to download a new one. | |
158 if (!data.empty()) { | |
159 std::unique_ptr<chrome_intelligence::RankerModel> model = Parse(data); | |
160 if (model) { | |
161 DVLOG(2) << "Model in '" << model_path_.value() | |
162 << "'' was originally downloaded from '" << model_url_ << "'."; | |
163 | |
164 std::string source = model->metadata().source(); | |
groby-ooo-7-16
2017/01/23 21:41:56
Technically, we only need src inside the !IsExpire
Roger McFarlane (Chromium)
2017/02/08 23:08:09
Oops! the DVLOG line above had the wrong source.
| |
165 if (!IsExpired(*model)) { | |
166 TransferModelToObserver(std::move(model)); | |
167 if (source == model_url_.spec()) | |
168 return; | |
169 } | |
170 } | |
171 } | |
172 | |
173 // Reaching this point means that a model download is required. If there is | |
174 // no download URL configured, then there is nothing further to do. | |
175 if (!model_url_.is_valid()) | |
176 return; | |
177 | |
178 // Otherwise, initialize the model fetcher to be non-null and trigger an | |
179 // initial download attempt. | |
180 url_fetcher_ = base::MakeUnique<TranslateURLFetcher>(kUrlFetcherId); | |
181 url_fetcher_->set_max_retry_on_5xx(kMaxRetryOn5xx); | |
182 LoadFromURL(); | |
183 } | |
184 | |
185 void RankerModelLoader::LoadFromURL() { | |
186 DCHECK(sequence_checker_.CalledOnValidSequence()); | |
187 // Immediately exit if there is no download pending. | |
188 if (!url_fetcher_) | |
189 return; | |
190 | |
191 // If a request is already in flight, do not issue a new one. | |
192 if (url_fetcher_->state() == TranslateURLFetcher::REQUESTING) { | |
193 DVLOG(2) << "RankerModelLoader: Download is in progress."; | |
194 return; | |
195 } | |
196 // Do nothing if the download attempts should be throttled. | |
197 if (base::TimeTicks::Now() < next_earliest_download_time_) { | |
198 DVLOG(2) << "ModelLoadser: Last download attempt was too recent."; | |
199 return; | |
200 } | |
201 | |
202 DVLOG(2) << "Downloading model from: " << model_url_; | |
203 | |
204 // Reset the time of the next earliest allowable download attempt. | |
205 next_earliest_download_time_ = | |
206 base::TimeTicks::Now() + | |
207 base::TimeDelta::FromMinutes(kDownloadRefractoryPeriodMin); | |
208 | |
209 // Kick off the next download attempt. | |
210 download_start_time_ = base::TimeTicks::Now(); | |
211 bool result = url_fetcher_->Request( | |
212 model_url_, base::Bind(&RankerModelLoader::OnDownloadComplete, | |
213 base::Unretained(this))); | |
214 | |
215 // The maximum number of download attempts has been surpassed. Don't make | |
216 // any further attempts. | |
217 if (!result) { | |
218 DVLOG(2) << "Model download abandoned."; | |
219 ReportModelStatus(RankerModelStatus::DOWNLOAD_FAILED); | |
220 url_fetcher_.reset(); | |
221 } | |
222 } | |
223 | |
224 void RankerModelLoader::OnDownloadComplete(int /* id */, | |
225 bool success, | |
226 const std::string& data) { | |
227 DCHECK(sequence_checker_.CalledOnValidSequence()); | |
228 | |
229 // Record the duration of the download. | |
230 base::TimeDelta duration = base::TimeTicks::Now() - download_start_time_; | |
231 base::HistogramBase* counter = base::Histogram::FactoryTimeGet( | |
232 uma_prefix_ + kDownloadTimerHistogram, | |
233 base::TimeDelta::FromMilliseconds(10), | |
234 base::TimeDelta::FromMilliseconds(200000), 100, | |
235 base::HistogramBase::kUmaTargetedHistogramFlag); | |
236 if (counter) | |
237 counter->AddTime(duration); | |
238 | |
239 // On failure, we just abort. The TranslateRanker will retry on a subsequent | |
240 // translation opportunity. The TranslateURLFetcher enforces a limit for | |
241 // retried requests. | |
242 if (!success) { | |
243 DVLOG(2) << "Download from '" << model_url_ << "'' failed."; | |
244 return; | |
245 } | |
246 | |
247 auto model = Parse(data); | |
248 if (!model) { | |
249 DVLOG(2) << "Model from '" << model_url_ << "'' failed to parse."; | |
250 return; | |
251 } | |
252 url_fetcher_.reset(); | |
253 | |
254 auto* metadata = model->mutable_metadata(); | |
255 metadata->set_source(model_url_.spec()); | |
256 metadata->set_last_modified_sec( | |
257 (base::Time::Now() - base::Time()).InSeconds()); | |
258 | |
259 if (!model_path_.empty()) { | |
260 DVLOG(2) << "Saving model from '" << model_url_ << "'' to '" | |
261 << model_path_.value() << "'."; | |
262 MyScopedHistogramTimer timer(uma_prefix_ + kWriteTimerHistogram); | |
263 base::ImportantFileWriter::WriteFileAtomically(model_path_, | |
264 model->SerializeAsString()); | |
265 } | |
266 | |
267 // Notify the owner that a compatible model is available. | |
268 TransferModelToObserver(std::move(model)); | |
269 } | |
270 | |
271 void RankerModelLoader::TransferModelToObserver( | |
272 std::unique_ptr<chrome_intelligence::RankerModel> model) { | |
273 DCHECK(sequence_checker_.CalledOnValidSequence()); | |
274 observer_->OnModelAvailable(std::move(model)); | |
275 } | |
276 | |
277 } // namespace translate | |
OLD | NEW |