Break large blobs into multiple ipcs during creation.

webkit/blob/blob_storage_controller.cc

 // Copyright (c) 2011 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "webkit/blob/blob_storage_controller.h"
 
 #include "base/logging.h"
 #include "googleurl/src/gurl.h"
 #include "net/base/upload_data.h"
 #include "webkit/blob/blob_data.h"
@@ skipping 11 matching lines @@
   return url.spec().find('#') != std::string::npos;
 }
 
 GURL ClearBlobUrlRef(const GURL& url) {
   size_t hash_pos = url.spec().find('#');
   if (hash_pos == std::string::npos)
     return url;
   return GURL(url.spec().substr(0, hash_pos));
 }
 
+static const int64 kMaxMemoryUsage = 1073741824;  // 1G

[jianli, 2011/09/29 01:13:33]

How about sth like: 1024 x 1024 x 1024 to make it more readable?

[michaeln, 2011/09/29 18:59:37]

Done.

+
 }  // namespace
 
-BlobStorageController::BlobStorageController() {
+BlobStorageController::BlobStorageController()
+    : memory_usage_(0) {
 }
 
 BlobStorageController::~BlobStorageController() {
 }
 
-void BlobStorageController::RegisterBlobUrl(
-    const GURL& url, const BlobData* blob_data) {
+void BlobStorageController::RegisterUnfinalizedBlobUrl(const GURL& url) {
   DCHECK(url.SchemeIs("blob"));
   DCHECK(!BlobUrlHasRef(url));
+  BlobData* blob_data = new BlobData;
+  unfinalized_blob_map_[url.spec()] = blob_data;
+  IncrementBlobDataUsage(blob_data);
+}
 
-  scoped_refptr<BlobData> target_blob_data(new BlobData());
-  target_blob_data->set_content_type(blob_data->content_type());
-  target_blob_data->set_content_disposition(blob_data->content_disposition());
+void BlobStorageController::AppendBlobDataItem(
+    const GURL& url, const BlobData::Item& item) {
+  DCHECK(url.SchemeIs("blob"));
+  DCHECK(!BlobUrlHasRef(url));
+  BlobMap::iterator found = unfinalized_blob_map_.find(url.spec());
+  if (found == unfinalized_blob_map_.end())
+    return;
+  BlobData* target_blob_data = found->second;
+  DCHECK(target_blob_data);
+
+  memory_usage_ -= target_blob_data->GetMemoryUsage();
 
   // The blob data is stored in the "canonical" way. That is, it only contains a
   // list of Data and File items.
   // 1) The Data item is denoted by the raw data and the range.
   // 2) The File item is denoted by the file path, the range and the expected
   //    modification time.
   // All the Blob items in the passing blob data are resolved and expanded into
   // a set of Data and File items.
 
-  for (std::vector<BlobData::Item>::const_iterator iter =
-           blob_data->items().begin();
-       iter != blob_data->items().end(); ++iter) {
-    switch (iter->type()) {
-      case BlobData::TYPE_DATA: {
-        // WebBlobData does not allow partial data.
-        DCHECK(!(iter->offset()) && iter->length() == iter->data().size());
-        target_blob_data->AppendData(iter->data());
-        break;
-      }
-      case BlobData::TYPE_FILE:
-        AppendFileItem(target_blob_data,
-                       iter->file_path(),
-                       iter->offset(),
-                       iter->length(),
-                       iter->expected_modification_time());
-        break;
-      case BlobData::TYPE_BLOB: {
-        BlobData* src_blob_data = GetBlobDataFromUrl(iter->blob_url());
-        DCHECK(src_blob_data);
-        if (src_blob_data)
-          AppendStorageItems(target_blob_data.get(),
-                             src_blob_data,
-                             iter->offset(),
-                             iter->length());
-        break;
-      }
-    }
+  switch (item.type) {
+    case BlobData::TYPE_DATA:
+      // WebBlobData does not allow partial data.
+      DCHECK(!(item.offset) && item.length == item.data.size());
+      target_blob_data->AppendItem(item);
+      break;
+    case BlobData::TYPE_FILE:
+      AppendFileItem(target_blob_data,
+                     item.file_path,
+                     item.offset,
+                     item.length,
+                     item.expected_modification_time);
+      break;
+    case BlobData::TYPE_BLOB:
+      BlobData* src_blob_data = GetBlobDataFromUrl(item.blob_url);
+      DCHECK(src_blob_data);
+      if (src_blob_data)
+        AppendStorageItems(target_blob_data,
+                           src_blob_data,
+                           item.offset,
+                           item.length);
+      break;
   }
 
-  blob_map_[url.spec()] = target_blob_data;
+  memory_usage_ += target_blob_data->GetMemoryUsage();
+
+  // If we're using too much memory, drop this blob.
+  // TODO(michaeln): Blob memory storage does not yet spill over to disk,
+  // until it does, we'll prevent memory usage over a max amount.
+  if (memory_usage_ > kMaxMemoryUsage)
+    UnregisterBlobUrl(url);
+}
+
+void BlobStorageController::FinalizeBlob(
+    const GURL& url, const std::string& content_type) {
+  DCHECK(url.SchemeIs("blob"));
+  DCHECK(!BlobUrlHasRef(url));
+  BlobMap::iterator found = unfinalized_blob_map_.find(url.spec());
+  if (found == unfinalized_blob_map_.end())
+    return;
+  found->second->set_content_type(content_type);
+  blob_map_[url.spec()] = found->second;
+  unfinalized_blob_map_.erase(found);
 }
 
 void BlobStorageController::RegisterBlobUrlFrom(
     const GURL& url, const GURL& src_url) {
   DCHECK(url.SchemeIs("blob"));
   DCHECK(!BlobUrlHasRef(url));
 
   BlobData* blob_data = GetBlobDataFromUrl(src_url);
   DCHECK(blob_data);
   if (!blob_data)
     return;
 
   blob_map_[url.spec()] = blob_data;
+  IncrementBlobDataUsage(blob_data);
 }
 
 void BlobStorageController::UnregisterBlobUrl(const GURL& url) {
-  blob_map_.erase(url.spec());
+  DCHECK(url.SchemeIs("blob"));
+  DCHECK(!BlobUrlHasRef(url));
+
+  if (!RemoveFromMapHelper(&unfinalized_blob_map_, url))
+    RemoveFromMapHelper(&blob_map_, url);
 }
 
+bool BlobStorageController::RemoveFromMapHelper(
+    BlobMap* map, const GURL& url) {
+  BlobMap::iterator found = map->find(url.spec());
+  if (found == map->end())
+    return false;
+  if (DecrementBlobDataUsage(found->second))

[jianli, 2011/09/29 01:13:33]

Just want to check whether it is possible to rely on the reference count of
BlobData to decide if it is the right time to decrease the memory usage count.

[michaeln, 2011/09/29 18:59:37]

It's an option, but that would mean more tightly coupling the controller and the
data classes (to observe destruction or keep backptrs to the controller). And
also blobdata objects may outlive the controller since BlobUrlRequestJobs take a
reference to them until the request is done. I went with the more explicit book
keeping scheme for its simplicity.

+    memory_usage_ -= found->second->GetMemoryUsage();
+  map->erase(found);
+  return true;
+}
+
+
 BlobData* BlobStorageController::GetBlobDataFromUrl(const GURL& url) {
   BlobMap::iterator found = blob_map_.find(
       BlobUrlHasRef(url) ? ClearBlobUrlRef(url).spec() : url.spec());
   return (found != blob_map_.end()) ? found->second : NULL;
 }
 
 void BlobStorageController::ResolveBlobReferencesInUploadData(
     net::UploadData* upload_data) {
   DCHECK(upload_data);
 
   std::vector<net::UploadData::Element>* uploads = upload_data->elements();
   std::vector<net::UploadData::Element>::iterator iter;
   for (iter = uploads->begin(); iter != uploads->end();) {
     if (iter->type() != net::UploadData::TYPE_BLOB) {
       iter++;
       continue;
     }
 
     // Find the referred blob data.
-    webkit_blob::BlobData* blob_data = GetBlobDataFromUrl(iter->blob_url());
+    BlobData* blob_data = GetBlobDataFromUrl(iter->blob_url());
     DCHECK(blob_data);
     if (!blob_data) {
       // TODO(jianli): We should probably fail uploading the data
       iter++;
       continue;
     }
 
     // Remove this element.
     iter = uploads->erase(iter);
 
     // If there is no element in the referred blob data, continue the loop.
     // Note that we should not increase iter since it already points to the one
     // after the removed element.
     if (blob_data->items().empty())
       continue;
 
     // Insert the elements in the referred blob data.
     // Note that we traverse from the bottom so that the elements can be
     // inserted in the original order.
     for (size_t i = blob_data->items().size(); i > 0; --i) {
       iter = uploads->insert(iter, net::UploadData::Element());
 
-      const webkit_blob::BlobData::Item& item = blob_data->items().at(i - 1);
-      switch (item.type()) {
-        case webkit_blob::BlobData::TYPE_DATA:
+      const BlobData::Item& item = blob_data->items().at(i - 1);
+      switch (item.type) {
+        case BlobData::TYPE_DATA:
           // TODO(jianli): Figure out how to avoid copying the data.
           iter->SetToBytes(
-              &item.data().at(0) + static_cast<int>(item.offset()),
-              static_cast<int>(item.length()));
+              &item.data.at(0) + static_cast<int>(item.offset),
+              static_cast<int>(item.length));
           break;
-        case webkit_blob::BlobData::TYPE_FILE:
+        case BlobData::TYPE_FILE:
           // TODO(michaeln): Ensure that any temp files survive till the
           // net::URLRequest is done with the upload.
           iter->SetToFilePathRange(
-              item.file_path(),
-              item.offset(),
-              item.length(),
-              item.expected_modification_time());
+              item.file_path,
+              item.offset,
+              item.length,
+              item.expected_modification_time);
           break;
         default:
           NOTREACHED();
           break;
       }
     }
   }
 }
 
 void BlobStorageController::AppendStorageItems(
     BlobData* target_blob_data, BlobData* src_blob_data,
     uint64 offset, uint64 length) {
   DCHECK(target_blob_data && src_blob_data &&
          length != static_cast<uint64>(-1));
 
   std::vector<BlobData::Item>::const_iterator iter =
       src_blob_data->items().begin();
   if (offset) {
     for (; iter != src_blob_data->items().end(); ++iter) {
-      if (offset >= iter->length())
-        offset -= iter->length();
+      if (offset >= iter->length)
+        offset -= iter->length;
       else
         break;
     }
   }
 
   for (; iter != src_blob_data->items().end() && length > 0; ++iter) {
-    uint64 current_length = iter->length() - offset;
+    uint64 current_length = iter->length - offset;
     uint64 new_length = current_length > length ? length : current_length;
-    if (iter->type() == BlobData::TYPE_DATA) {
-      target_blob_data->AppendData(iter->data(),
-                                   static_cast<uint32>(iter->offset() + offset),
+    if (iter->type == BlobData::TYPE_DATA) {
+      target_blob_data->AppendData(iter->data,
+                                   static_cast<uint32>(iter->offset + offset),
                                    static_cast<uint32>(new_length));
     } else {
-      DCHECK(iter->type() == BlobData::TYPE_FILE);
+      DCHECK(iter->type == BlobData::TYPE_FILE);
       AppendFileItem(target_blob_data,
-                     iter->file_path(),
-                     iter->offset() + offset,
+                     iter->file_path,
+                     iter->offset + offset,
                      new_length,
-                     iter->expected_modification_time());
+                     iter->expected_modification_time);
     }
     length -= new_length;
     offset = 0;
   }
 }
 
 void BlobStorageController::AppendFileItem(
     BlobData* target_blob_data,
     const FilePath& file_path, uint64 offset, uint64 length,
     const base::Time& expected_modification_time) {
   target_blob_data->AppendFile(file_path, offset, length,
                                expected_modification_time);
 
   // It may be a temporary file that should be deleted when no longer needed.
   scoped_refptr<DeletableFileReference> deletable_file =
       DeletableFileReference::Get(file_path);
   if (deletable_file)
     target_blob_data->AttachDeletableFileReference(deletable_file);
 }
 
+void BlobStorageController::IncrementBlobDataUsage(BlobData* blob_data) {
+  blob_data_usage_count_[blob_data] += 1;
+}
+
+bool BlobStorageController::DecrementBlobDataUsage(BlobData* blob_data) {
+  BlobDataUsageMap::iterator found = blob_data_usage_count_.find(blob_data);
+  DCHECK(found != blob_data_usage_count_.end());
+  if (--(found->second))
+    return false;  // Still in use
+  blob_data_usage_count_.erase(found);
+  return true;
+}
+
 }  // namespace webkit_blob