Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(329)

Side by Side Diff: storage/browser/blob/blob_memory_controller.cc

Issue 2055053003: [BlobAsync] Disk support for blob storage (Closed) Base URL: https://chromium.googlesource.com/chromium/src.git@master
Patch Set: Added back transport controller test, small cleanups Created 4 years, 5 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
(Empty)
1 // Copyright 2016 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "storage/browser/blob/blob_memory_controller.h"
6
7 #include <algorithm>
8
9 #include "base/callback.h"
10 #include "base/callback_helpers.h"
11 #include "base/files/file_util.h"
12 #include "base/location.h"
13 #include "base/memory/ptr_util.h"
14 #include "base/metrics/histogram_macros.h"
15 #include "base/numerics/safe_conversions.h"
16 #include "base/numerics/safe_math.h"
17 #include "base/single_thread_task_runner.h"
18 #include "base/strings/string_number_conversions.h"
19 #include "base/task_runner.h"
20 #include "base/task_runner_util.h"
21 #include "base/threading/thread_task_runner_handle.h"
22 #include "base/time/time.h"
23 #include "base/trace_event/trace_event.h"
24 #include "base/tuple.h"
25 #include "base/single_thread_task_runner.h"
26 #include "storage/browser/blob/blob_data_item.h"
27 #include "storage/browser/blob/shareable_blob_data_item.h"
28 #include "storage/browser/blob/shareable_file_reference.h"
29
30 using base::File;
31 using base::FilePath;
32 using FileCreationInfo = storage::BlobMemoryController::FileCreationInfo;
33
34 namespace storage {
35 namespace {
36
37 bool CalculateBlobMemorySize(const std::vector<DataElement>& elements,
38 size_t* shortcut_bytes,
39 uint64_t* total_bytes) {
40 DCHECK(shortcut_bytes);
41 DCHECK(total_bytes);
42
43 base::CheckedNumeric<uint64_t> total_size_checked = 0;
44 base::CheckedNumeric<size_t> shortcut_size_checked = 0;
45 for (const auto& e : elements) {
46 if (e.type() == DataElement::TYPE_BYTES) {
47 total_size_checked += e.length();
48 shortcut_size_checked += e.length();
49 } else if (e.type() == DataElement::TYPE_BYTES_DESCRIPTION) {
50 total_size_checked += e.length();
51 } else {
52 continue;
53 }
54 if (!total_size_checked.IsValid() || !shortcut_size_checked.IsValid()) {
55 return false;
56 }
57 }
58 *shortcut_bytes = shortcut_size_checked.ValueOrDie();
59 *total_bytes = total_size_checked.ValueOrDie();
60 return true;
61 }
62
63 // Creates a file in the given directory w/ the given filename and size.
64 BlobMemoryController::FileCreationInfo CreateFile(
65 scoped_refptr<ShareableFileReference> file_reference,
66 size_t size_bytes) {
67 LOG(ERROR) << "creating file for renderer";
68 DCHECK_NE(0u, size_bytes);
69 BlobMemoryController::FileCreationInfo creation_info;
70
71 // Try to open our file.
72 File file(file_reference->path(),
73 File::FLAG_CREATE_ALWAYS | File::FLAG_WRITE);
74 creation_info.file_reference = std::move(file_reference);
75 creation_info.error = file.error_details();
76 UMA_HISTOGRAM_ENUMERATION("Storage.Blob.TransportFileCreate",
77 -creation_info.error, -File::FILE_ERROR_MAX);
78 if (creation_info.error != File::FILE_OK)
79 return creation_info;
80
81 // Grab the file info to get the "last modified" time, and create the
82 // ShareableFileReference which will manage the lifetime of the file.
83 File::Info file_info;
84 bool success = file.GetInfo(&file_info);
85 UMA_HISTOGRAM_BOOLEAN("Storage.Blob.TransportFileInfoSuccess", success);
86 creation_info.error = success ? File::FILE_OK : File::FILE_ERROR_FAILED;
87 creation_info.last_modified = file_info.last_modified;
88 if (creation_info.error == File::FILE_OK) {
89 creation_info.file = std::move(file);
90 }
91 return creation_info;
92 }
93
94 BlobMemoryController::FileCreationInfo WriteItemsToFile(
95 std::vector<scoped_refptr<ShareableBlobDataItem>>* items,
96 size_t total_size_bytes,
97 scoped_refptr<ShareableFileReference> file_reference) {
98 DCHECK_NE(0u, total_size_bytes);
99 LOG(ERROR) << "writing to file!";
100 UMA_HISTOGRAM_MEMORY_KB("Storage.Blob.PageFileSize", total_size_bytes / 1024);
101
102 // Create our file.
103 BlobMemoryController::FileCreationInfo creation_info;
104 File file(file_reference->path(),
105 File::FLAG_CREATE_ALWAYS | File::FLAG_WRITE);
106 creation_info.file_reference = std::move(file_reference);
107 creation_info.error = file.error_details();
108 UMA_HISTOGRAM_ENUMERATION("Storage.Blob.PageFileCreate", -creation_info.error,
109 -File::FILE_ERROR_MAX);
110 if (creation_info.error != File::FILE_OK)
111 return creation_info;
112
113 // Write data.
114 file.SetLength(total_size_bytes);
115 int bytes_written = 0;
116 for (const auto& refptr : *items) {
117 const DataElement& element = refptr->item()->data_element();
118 DCHECK_EQ(DataElement::TYPE_BYTES, element.type());
119 int length = base::checked_cast<int, uint64_t>(element.length());
120 bytes_written = file.WriteAtCurrentPos(element.bytes(), length);
121 DCHECK_EQ(length, bytes_written);
122 if (bytes_written < 0) {
123 break;
124 }
125 }
126 UMA_HISTOGRAM_BOOLEAN("Storage.Blob.PageFileWriteSuccess", bytes_written > 0);
127
128 // Grab our modification time and create our SharedFileReference to manage the
129 // lifetime of the file.
130 File::Info info;
131 bool success = file.GetInfo(&info);
132 UMA_HISTOGRAM_BOOLEAN("Storage.Blob.PageFileInfoSuccess", success);
133 creation_info.error =
134 bytes_written < 0 || !success ? File::FILE_ERROR_FAILED : File::FILE_OK;
135 creation_info.last_modified = info.last_modified;
136 return creation_info;
137 }
138
139 } // namespace
140
141 BlobMemoryController::FileCreationInfo::FileCreationInfo() {}
142
143 BlobMemoryController::FileCreationInfo::~FileCreationInfo() {}
144
145 FileCreationInfo::FileCreationInfo(FileCreationInfo&&) = default;
146 FileCreationInfo& FileCreationInfo::operator=(FileCreationInfo&&) = default;
147
148 BlobMemoryController::BlobMemoryController()
149 : recent_item_cache_(RecentItemsCache::NO_AUTO_EVICT) {}
150
151 BlobMemoryController::~BlobMemoryController() {}
152
153 void BlobMemoryController::EnableDisk(
154 const base::FilePath storage_directory,
155 scoped_refptr<base::SingleThreadTaskRunner> file_runner) {
156 LOG(ERROR) << "enbling disk";
157 file_runner_ = std::move(file_runner);
158 blob_storage_dir_ = storage_directory;
159 enable_disk_ = true;
160 }
161
162 bool BlobMemoryController::DecideBlobTransportationMemoryStrategy(
163 const std::vector<DataElement>& descriptions,
164 uint64_t* total_bytes,
165 BlobMemoryController::MemoryStrategyResult* result) const {
166 DCHECK(total_bytes);
167 DCHECK(result);
168
169 // Step 1: Get the sizes.
170 size_t shortcut_memory_size_bytes;
171 uint64_t total_memory_size_bytes;
172 if (!CalculateBlobMemorySize(descriptions, &shortcut_memory_size_bytes,
173 &total_memory_size_bytes)) {
174 return false;
175 }
176 *total_bytes = total_memory_size_bytes;
177
178 // Step 2: Handle case where we have no memory to transport.
179 if (total_memory_size_bytes == 0) {
180 *result = MemoryStrategyResult::NONE_NEEDED;
181 return true;
182 }
183
184 // Step 3: Check if we have enough memory to store the blob.
185 if (total_memory_size_bytes > GetAvailableMemoryForBlobs() &&
186 total_memory_size_bytes > GetAvailableDiskSpaceForBlobs()) {
187 *result = MemoryStrategyResult::TOO_LARGE;
188 return true;
189 }
190
191 // From here on, we know we can fit the blob in memory or on disk.
192 // Step 4: Decide if we're using the shortcut method.
193 if (shortcut_memory_size_bytes == total_memory_size_bytes &&
194 blobs_waiting_for_paging_.empty() &&
195 shortcut_memory_size_bytes < GetAvailableMemoryForBlobs()) {
196 *result = MemoryStrategyResult::SHORTCUT;
197 return true;
198 }
199
200 // Step 5: Decide if we're going straight to disk.
201 if (enable_disk_ && (total_memory_size_bytes > max_blob_in_memory_size_)) {
202 *result = MemoryStrategyResult::FILE;
203 return true;
204 }
205 // From here on, we know the blob's size is less than:
206 // * max_blob_in_memory_size_ if enable_disk_ is true
207 // * max_blob_memory_space_ if enable_disk_ is false
208 // So we know we're < max(size_t).
209 // Step 6: Decide if we're using shared memory.
210 if (total_memory_size_bytes > max_ipc_memory_size_) {
211 *result = MemoryStrategyResult::SHARED_MEMORY;
212 return true;
213 }
214 // Step 7: We can fit in IPC.
215 *result = MemoryStrategyResult::IPC;
216 return true;
217 }
218
219 void BlobMemoryController::CreateTemporaryFileForRenderer(
220 uint64_t size_bytes,
221 const base::Callback<void(FileCreationInfo)>& done) {
222 if (!enable_disk_) {
223 BlobMemoryController::FileCreationInfo creation_info;
224 done.Run(std::move(creation_info));
225 return;
226 }
227
228 disk_used_ += size_bytes;
229 std::string file_name = base::Uint64ToString(current_file_num_++);
230 scoped_refptr<ShareableFileReference> file_ref =
231 ShareableFileReference::GetOrCreate(
232 blob_storage_dir_.Append(file_name),
233 ShareableFileReference::DELETE_ON_FINAL_RELEASE, file_runner_.get());
234 base::PostTaskAndReplyWithResult(
235 file_runner_.get(), FROM_HERE,
236 base::Bind(&CreateFile, std::move(file_ref), size_bytes),
237 base::Bind(&BlobMemoryController::OnCreateFile, this->AsWeakPtr(),
238 size_bytes, done));
239 }
240
241 void BlobMemoryController::FreeMemory(size_t memory_size_bytes) {
242 DCHECK_GE(blob_memory_used_, memory_size_bytes);
243 blob_memory_used_ -= memory_size_bytes;
244 MaybeScheduleWaitingBlobs();
245 }
246
247 bool BlobMemoryController::MaybeFitInMemoryNow(size_t memory_size) {
248 if (memory_size <= GetAvailableMemoryForBlobs()) {
249 blob_memory_used_ += memory_size;
250 LOG(ERROR) << "Yes, " << memory_size << " can fit in memory now.";
251 if (enable_disk_ && blob_memory_used_ > max_blob_in_memory_size_) {
252 LOG(ERROR) << memory_size << " out of " << blob_memory_used_;
253 ScheduleBlobPaging();
254 }
255 return true;
256 }
257 LOG(ERROR) << memory_size << "can't fit in memory yet.";
258 return false;
259 }
260
261 BlobMemoryController::PendingContructionEntry
262 BlobMemoryController::NotifyWhenMemoryCanPopulated(
263 size_t memory_size,
264 const base::Callback<void(bool)>& can_request) {
265 LOG(ERROR) << "waiting until " << memory_size << " can fit.";
266 if (memory_size > GetAvailableMemoryForBlobs() &&
267 memory_size > GetAvailableDiskSpaceForBlobs()) {
268 can_request.Run(false);
269 return blobs_waiting_for_paging_.end();
270 }
271
272 if (!enable_disk_) {
273 blob_memory_used_ += memory_size;
274 can_request.Run(true);
275 return blobs_waiting_for_paging_.end();
276 }
277
278 // If we're currently waiting for blobs to page already, then we add
279 // ourselves to the end of the queue. Once paging is complete, we'll schedule
280 // more paging for any more pending blobs.
281 if (!blobs_waiting_for_paging_.empty()) {
282 blobs_waiting_for_paging_.push_back(
283 std::make_pair(memory_size, can_request));
284 blobs_waiting_for_paging_size_ += memory_size;
285 return --blobs_waiting_for_paging_.end();
286 }
287
288 // Store right away if we can.
289 if (memory_size <= GetAvailableMemoryForBlobs()) {
290 // If we're past our blob memory limit, then schedule our paging.
291 if (blob_memory_used_ + memory_size > max_blob_in_memory_size_) {
292 ScheduleBlobPaging();
293 }
294 blob_memory_used_ += memory_size;
295 can_request.Run(true);
296 return blobs_waiting_for_paging_.end();
297 }
298
299 // This means we're too big for memory.
300 DCHECK(blobs_waiting_for_paging_.empty());
301 DCHECK_EQ(0u, blobs_waiting_for_paging_size_);
302 blobs_waiting_for_paging_.push_back(std::make_pair(memory_size, can_request));
303 blobs_waiting_for_paging_size_ = memory_size;
304 SchedulePagingUntilWeCanFit(memory_size);
305 return --blobs_waiting_for_paging_.end();
306 }
307
308 void BlobMemoryController::RemovePendingConstructionEntry(
309 const BlobMemoryController::PendingContructionEntry& entry) {
310 const std::pair<size_t, base::Callback<void(bool)>> pair = *entry;
311 blobs_waiting_for_paging_size_ -= pair.first;
312 }
313
314 void BlobMemoryController::UpdateBlobItemInRecents(
315 ShareableBlobDataItem* item) {
316 auto iterator = recent_item_cache_.Get(item->item_id());
317 if (iterator != recent_item_cache_.end())
318 recent_item_cache_.Erase(iterator);
319 recent_item_cache_.Put(item->item_id(), item);
320 }
321
322 void BlobMemoryController::RemoveBlobItemInRecents(uint64_t id) {
323 auto iterator = recent_item_cache_.Get(id);
324 if (iterator != recent_item_cache_.end())
325 recent_item_cache_.Erase(iterator);
326 }
327
328 void BlobMemoryController::OnCreateFile(
329 uint64_t file_size,
330 const base::Callback<void(FileCreationInfo)>& done,
331 FileCreationInfo result) {
332 if (result.error == File::FILE_OK) {
333 result.file_reference->AddFinalReleaseCallback(base::Bind(
334 &BlobMemoryController::OnBlobFileDelete, this->AsWeakPtr(), file_size));
335 } else {
336 disk_used_ -= file_size;
337 }
338 LOG(ERROR) << "Created file!";
339 done.Run(std::move(result));
340 }
341
342 void BlobMemoryController::MaybeScheduleWaitingBlobs() {
343 size_t space_available = max_blob_in_memory_size_ - blob_memory_used_;
344 while (!blobs_waiting_for_paging_.empty() &&
345 space_available >= blobs_waiting_for_paging_.front().first) {
346 auto size_callback_pair = blobs_waiting_for_paging_.front();
347 blobs_waiting_for_paging_.pop_front();
348 space_available -= size_callback_pair.first;
349 blobs_waiting_for_paging_size_ -= size_callback_pair.first;
350 blob_memory_used_ += size_callback_pair.first;
351 size_callback_pair.second.Run(true);
352 }
353 }
354
355 size_t BlobMemoryController::ScheduleBlobPaging() {
356 DCHECK(enable_disk_);
357 DCHECK_LT(min_page_file_size_, static_cast<uint64_t>(blob_memory_used_));
358 LOG(ERROR) << "scheduling paging";
359
360 size_t total_items_size = 0;
361 std::unique_ptr<std::vector<scoped_refptr<ShareableBlobDataItem>>>
362 items_for_disk(new std::vector<scoped_refptr<ShareableBlobDataItem>>());
363 // Collect our items.
364 while (total_items_size < min_page_file_size_ &&
365 !recent_item_cache_.empty()) {
366 auto iterator = --recent_item_cache_.end();
367 ShareableBlobDataItem* item = iterator->second;
368 DCHECK(item);
369 recent_item_cache_.Erase(iterator);
370 size_t size = base::checked_cast<size_t, uint64_t>(item->item()->length());
371 total_items_size += size;
372 items_for_disk->push_back(make_scoped_refptr(item));
373 }
374
375 // Update our bookkeeping.
376 disk_used_ += total_items_size;
377 DCHECK_GE(blob_memory_used_, total_items_size);
378 blob_memory_used_ -= total_items_size;
379 in_flight_memory_used_ += total_items_size;
380 std::string file_name = base::Uint64ToString(current_file_num_++);
381 scoped_refptr<ShareableFileReference> file_ref =
382 ShareableFileReference::GetOrCreate(
383 blob_storage_dir_.Append(file_name),
384 ShareableFileReference::DELETE_ON_FINAL_RELEASE, file_runner_.get());
385 base::PostTaskAndReplyWithResult(
386 file_runner_.get(), FROM_HERE,
387 base::Bind(&WriteItemsToFile, items_for_disk.get(), total_items_size,
388 std::move(file_ref)),
389 base::Bind(&BlobMemoryController::OnPagingComplete, this->AsWeakPtr(),
390 base::Passed(&items_for_disk), total_items_size));
391 return total_items_size;
392 }
393
394 void BlobMemoryController::SchedulePagingUntilWeCanFit(
395 size_t total_memory_needed) {
396 DCHECK_LT(total_memory_needed, max_blob_in_memory_size_);
397 while (total_memory_needed + blob_memory_used_ > max_blob_in_memory_size_) {
398 // schedule another page.
399 size_t memory_freeing = ScheduleBlobPaging();
400 DCHECK_GE(blob_memory_used_, memory_freeing);
401 }
402 }
403
404 void BlobMemoryController::OnPagingComplete(
405 std::unique_ptr<std::vector<scoped_refptr<ShareableBlobDataItem>>> items,
406 size_t total_items_size,
407 FileCreationInfo result) {
408 if (!enable_disk_) {
409 return;
410 }
411 if (result.error != File::FILE_OK) {
412 UMA_HISTOGRAM_ENUMERATION("Storage.Blob.PagingError", -result.error,
413 -File::FILE_ERROR_MAX);
414 disk_used_ -= total_items_size;
415 DisableDisk();
416 return;
417 }
418 // Switch from the data backing item to a new file backing item.
419 uint64_t offset = 0;
420 for (const scoped_refptr<ShareableBlobDataItem>& shareable_item :
421 *items.get()) {
422 scoped_refptr<BlobDataItem> new_item(new BlobDataItem(
423 base::WrapUnique(new DataElement()), result.file_reference));
424 new_item->data_element_ptr()->SetToFilePathRange(
425 result.file_reference->path(), offset, shareable_item->item()->length(),
426 result.last_modified);
427 shareable_item->item_.swap(new_item);
428 offset += shareable_item->item()->length();
429 }
430 in_flight_memory_used_ -= total_items_size;
431
432 // We want callback on blobs up to the amount we've freed.
433 MaybeScheduleWaitingBlobs();
434
435 // If we still have more blobs waiting, schedule more paging.
436 if (!blobs_waiting_for_paging_.empty()) {
437 SchedulePagingUntilWeCanFit(blobs_waiting_for_paging_size_);
438 }
439 }
440
441 void BlobMemoryController::DisableDisk() {
442 enable_disk_ = false;
443 blob_memory_used_ += in_flight_memory_used_;
444 in_flight_memory_used_ = 0;
445 for (const auto& size_callback_pair : blobs_waiting_for_paging_) {
446 size_callback_pair.second.Run(false);
447 }
448 blobs_waiting_for_paging_.clear();
449 blobs_waiting_for_paging_size_ = 0;
450 recent_item_cache_.Clear();
451 RecordTracingCounters();
452 }
453
454 void BlobMemoryController::RecordTracingCounters() {
455 TRACE_COUNTER2("Blob", "MemoryUsage", "RegularStorage", blob_memory_used_,
456 "InFlightToDisk", in_flight_memory_used_);
457 TRACE_COUNTER1("Blob", "TranfersPendingOnDisk",
458 blobs_waiting_for_paging_.size());
459 TRACE_COUNTER1("Blob", "TranfersBytesPendingOnDisk",
460 blobs_waiting_for_paging_size_);
461 }
462
463 size_t BlobMemoryController::GetAvailableMemoryForBlobs() const {
464 if (max_blob_in_memory_size_ < blob_memory_used_)
465 return 0;
466 if (enable_disk_)
467 return std::min<size_t>(0u, in_flight_space_ - in_flight_memory_used_) +
468 (max_blob_in_memory_size_ - blob_memory_used_);
469 return max_blob_memory_space_ - blob_memory_used_;
470 }
471
472 uint64_t BlobMemoryController::GetAvailableDiskSpaceForBlobs() const {
473 return enable_disk_ ? kBlobStorageMaxDiskSpace - disk_used_ : 0;
474 }
475
476 void BlobMemoryController::OnBlobFileDelete(uint64_t size,
477 const base::FilePath& path) {
478 DCHECK_LE(size, disk_used_);
479 LOG(ERROR) << "deleting file " << path.value();
480 disk_used_ -= size;
481 }
482
483 } // namespace storage
OLDNEW

Powered by Google App Engine
This is Rietveld 408576698