The optional Malware Details also collects HTTP cache information. See design...

chrome/browser/safe_browsing/malware_details.cc

Index: chrome/browser/safe_browsing/malware_details.cc
===================================================================
--- chrome/browser/safe_browsing/malware_details.cc	(revision 76771)
+++ chrome/browser/safe_browsing/malware_details.cc	(working copy)
@@ -6,11 +6,21 @@
 
 #include "chrome/browser/safe_browsing/malware_details.h"
 
+#include "base/callback.h"
 #include "base/lazy_instance.h"
+#include "base/md5.h"
+#include "chrome/browser/net/chrome_url_request_context.h"
+#include "chrome/browser/profiles/profile.h"
 #include "chrome/browser/safe_browsing/safe_browsing_service.h"
 #include "chrome/browser/safe_browsing/report.pb.h"
+#include "chrome/common/net/url_request_context_getter.h"
 #include "chrome/common/render_messages.h"
 #include "chrome/common/render_messages_params.h"
+#include "net/base/io_buffer.h"
+#include "net/disk_cache/disk_cache.h"
+#include "net/http/http_cache.h"
+#include "net/http/http_response_headers.h"
+#include "net/http/http_response_info.h"
 #include "content/browser/browser_thread.h"
 #include "content/browser/renderer_host/render_view_host.h"
 #include "content/browser/tab_contents/navigation_entry.h"
@@ -21,6 +31,10 @@
 // Keep in sync with KMaxNodes in renderer/safe_browsing/malware_dom_details
 static const uint32 kMaxDomNodes = 500;
 
+// Only send tiny files for now, a better strategy would use the size
+// of the whole report and the user's bandwidth.
+static const int kMaxBodySize = 1024;
+
 // static
 MalwareDetailsFactory* MalwareDetails::factory_ = NULL;
 
@@ -64,7 +78,14 @@
     TabContents* tab_contents,
     const SafeBrowsingService::UnsafeResource resource)
     : TabContentsObserver(tab_contents),
-      resource_(resource) {
+      resource_(resource),
+      cache_state_(STATE_NONE),
+      ALLOW_THIS_IN_INITIALIZER_LIST(
+          cache_callback_(this, &MalwareDetails::CacheLoop)),
+      ALLOW_THIS_IN_INITIALIZER_LIST(
+          entry_callback_(
+              new net::CancelableCompletionCallback<MalwareDetails>(
+                  this, &MalwareDetails::CacheLoop))) {
   StartCollection();
 }
 
@@ -192,6 +213,10 @@
   // Get URLs of frames, scripts etc from the DOM.
   // OnReceivedMalwareDOMDetails will be called when the renderer replies.
   tab_contents()->render_view_host()->GetMalwareDOMDetails();
+
+  // DoOpenCache will need this later. Must be called on the UI thread, so
+  // we do it now.
+  request_context_getter_ = tab_contents()->profile()->GetRequestContext();

[lzheng, 2011/03/05 00:37:50]

Might be better to put this in the constructor.

[panayiotis, 2011/03/29 18:14:55]

Done.

 }
 
 // When the renderer is done, this is called.
@@ -208,9 +233,14 @@
 void MalwareDetails::AddDOMDetails(
     const ViewHostMsg_MalwareDOMDetails_Params& params) {
   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
+  DVLOG(1) << "Nodes from the DOM: " << params.nodes.size();
+  // If we have already started collecting data from the HTTP cache, don't

[lzheng, 2011/03/05 00:37:50]

nit: add an empty line above.

[panayiotis, 2011/03/29 18:14:55]

Done.

+  // modify our state.
+  if (cache_state_ != STATE_NONE)
+    return;
+
   // Add the urls from the DOM to |resources_|.  The renderer could be
   // sending bogus messages, so limit the number of nodes we accept.
-  DVLOG(1) << "Nodes from the DOM: " << params.nodes.size();
   for (uint32 i = 0; i < params.nodes.size() && i < kMaxDomNodes; ++i) {
     ViewHostMsg_MalwareDOMDetails_Node node = params.nodes[i];
     DVLOG(1) << node.url << ", " << node.tag_name << ", " << node.parent;
@@ -223,17 +253,266 @@
 // to take an action, we expect this to be called after
 // OnReceivedMalwareDOMDetails in most cases. If not, we don't include
 // the DOM data in our report.
-const std::string* MalwareDetails::GetSerializedReport() {
+void MalwareDetails::StartCacheCollection(MalwareDetailsCallback* callback) {
   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
-  // The |report_| protocol buffer is now generated: We add all the
-  // urls in our |resources_| maps.
+
+  // Start the data collection from the HTTP cache.
+  DVLOG(1) << "Getting disk_cache data for all urls...";
+  resources_it_ = resources_.begin();
+  cache_state_ = STATE_OPEN_CACHE;
+  callback_ = callback;
+
+  // Post a task in the message loop, so the callers don't need to
+  // check if we call their callback immediately.
+  BrowserThread::PostTask(
+      BrowserThread::IO, FROM_HERE,
+      NewRunnableMethod(
+          this, &MalwareDetails::CacheLoop, net::OK));
+}
+
+// The IO thread loop. Called once the DOM details are callected,
+// and then is passed to all the cache lookups as a callback.
+void MalwareDetails::CacheLoop(int result) {
+  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
+  DCHECK(cache_state_ != STATE_NONE);
+
+  int rv = result;
+  do {
+    DVLOG(1) << "cache_state_:" << cache_state_;
+    CacheState state = cache_state_;
+    cache_state_ = STATE_NONE;
+    switch (state) {
+      case STATE_OPEN_CACHE:
+        rv = DoOpenCache();
+        break;
+      case STATE_OPEN_CACHE_COMPLETE:
+        rv = DoOpenCacheComplete(rv);
+        break;
+      case STATE_OPEN_ENTRY:
+        rv = DoOpenEntry();
+        break;
+      case STATE_OPEN_ENTRY_COMPLETE:
+        rv = DoOpenEntryComplete(rv);
+        break;
+      case STATE_READ_RESPONSE:
+        rv = DoReadResponse();
+        break;
+      case STATE_READ_RESPONSE_COMPLETE:
+        rv = DoReadResponseComplete(rv);
+        break;
+      case STATE_READ_DATA:
+        rv = DoReadData();
+        break;
+      case STATE_READ_DATA_COMPLETE:
+        rv = DoReadDataComplete(rv);
+        break;
+      default:
+        NOTREACHED() << "Bad state:" << state;
+        rv = net::ERR_FAILED;
+        break;
+    }
+  } while (rv != net::ERR_IO_PENDING && cache_state_ != STATE_NONE);
+
+  if (rv != net::ERR_IO_PENDING) {
+    DVLOG(1) << "CacheIO done.";
+    DoCacheDone(rv);
+  }
+}
+
+net::URLRequestContext* MalwareDetails::GetURLRequestContext() {
+  if (!request_context_getter_) {
+    DVLOG(1) << "Missing request context getter";
+    return NULL;
+  }
+  return request_context_getter_->GetURLRequestContext();
+}
+
+int MalwareDetails::DoOpenCache() {
+  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
+  cache_state_ = STATE_OPEN_CACHE_COMPLETE;
+
+  net::URLRequestContext* context = GetURLRequestContext();
+  if (!context) {
+    LOG(ERROR) << "Could not get URLRequestContext";
+    cache_state_ = STATE_NONE;
+    return net::ERR_FAILED;
+  }
+
+  if (!context->http_transaction_factory()) {
+    DVLOG(1) << "Could not get HTTP transaction factory";
+    cache_state_ = STATE_NONE;
+    return net::ERR_FAILED;
+  }
+
+  net::HttpCache* http_cache =
+      context->http_transaction_factory()->GetCache();
+  if (!http_cache) {
+    DVLOG(1) << "Could not get http cache";
+    cache_state_ = STATE_NONE;
+    return net::ERR_FAILED;
+  }
+
+  return http_cache->GetBackend(&cache_, &cache_callback_);
+}
+
+int MalwareDetails::DoOpenCacheComplete(int result) {
+  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
+  cache_state_ = STATE_OPEN_ENTRY;
+
+  if (result == net::ERR_FAILED || !cache_) {
+    LOG(ERROR) << "Could not get disk cache";
+    cache_state_ = STATE_NONE;
+    return net::ERR_FAILED;
+  }
+
+  return net::OK;
+}
+
+int MalwareDetails::DoOpenEntry() {
+  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
+  cache_state_ = STATE_OPEN_ENTRY_COMPLETE;
+
+  if (resources_it_ == resources_.end()) {
+    cache_state_ = STATE_NONE;
+    return net::OK;
+  }
+
+  entry_ = NULL;  // Reset the entry.
+  return cache_->OpenEntry(resources_it_->first, &entry_, &cache_callback_);
+}
+
+int MalwareDetails::DoOpenEntryComplete(int rv) {
+  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
+  cache_state_ = STATE_READ_RESPONSE;
+  return net::OK;
+}
+
+int MalwareDetails::DoReadResponse() {
+  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
+  cache_state_ = STATE_READ_RESPONSE_COMPLETE;
+  entry_callback_->AddRef();

[lzheng, 2011/03/05 00:37:50]

Since MalwareDetails is refcounted, while waiting for callback, entry_callack_
won't go away (however, in this case, you have to delete entry_callback_ to let
it release the ref count for MalwareDetails, otherwise, both of them will stay
forever). I did not look carefully about the other callback. However, I do hope
we could avoid AddRef and Release.

[panayiotis, 2011/03/29 18:14:55]

I managed to not refcount this callback. But the cache_callback_ is not
refcounted, which requires that both MalwareDetailsCache and MalwareDetails need
to AddRef() to themselves ... I think.

I can take another look. 
 

On 2011/03/05 00:37:50, lzheng wrote:

+
+  if (!entry_) {  // No entry in the cache for this url.
+    return net::OK;
+  }
+
+  // net::HttpCache::kResponseInfoIndex = 0.
+  buf_len_ = entry_->GetDataSize(0);
+  if (!buf_len_) {
+    DVLOG(1) << "Empty response";
+    return buf_len_;
+  }
+
+  buf_ = new net::IOBuffer(buf_len_);
+  return entry_->ReadData(0, 0, buf_, buf_len_, entry_callback_);
+}
+
+int MalwareDetails::DoReadResponseComplete(int result) {
+  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
+  cache_state_ = STATE_READ_DATA;
+  entry_callback_->Release();
+
+  if (!entry_ || !result || result != buf_len_) {
+    DVLOG(1) << "No entry";
+  } else {
+    linked_ptr<safe_browsing::ClientMalwareReportRequest::Resource>
+        pb_resource = resources_it_->second;
+    net::HttpResponseInfo response;
+    bool truncated;
+    if (net::HttpCache::ParseResponseInfo(buf_->data(), buf_len_, &response,
+                                          &truncated)) {
+      if (!truncated) {
+        if (response.headers) {
+          safe_browsing::ClientMalwareReportRequest::HTTPResponse* pb_response =
+              pb_resource->mutable_response();
+          pb_response->mutable_firstline()->set_code(
+              response.headers->response_code());
+          void* iter = NULL;
+          std::string name, value;
+          while (response.headers->EnumerateHeaderLines(&iter, &name, &value)) {
+            // Strip any Set-Cookie headers.
+            if (LowerCaseEqualsASCII(name, "set-cookie")) {
+              continue;
+            }
+            safe_browsing::ClientMalwareReportRequest::HTTPHeader* pb_header =
+                pb_response->add_headers();
+            pb_header->set_name(name);
+            pb_header->set_value(value);
+          }
+        } else {
+          DVLOG(1) << "Missing response headers.";
+        }
+      } else {
+        DVLOG(1) << "Response truncated";
+      }
+    } else {
+      LOG(ERROR) << "Could not parse HTTP cache entry";
+    }
+  }
+
+  return net::OK;
+}
+
+int MalwareDetails::DoReadData() {
+  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
+  cache_state_ = STATE_READ_DATA_COMPLETE;
+  entry_callback_->AddRef();
+
+  if (!entry_) {  // No entry in the cache for this url.
+    return net::OK;
+  }
+
+  // net::HttpCache::kResponseContentIndex = 1.
+  buf_len_ = entry_->GetDataSize(1);
+  if (!buf_len_) {
+    DVLOG(1) << "Empty data";
+    return buf_len_;
+  }
+
+  buf_ = new net::IOBuffer(buf_len_);
+  return entry_->ReadData(1, 0, buf_, buf_len_, entry_callback_);
+}
+
+int MalwareDetails::DoReadDataComplete(int result) {
+  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
+  cache_state_ = STATE_OPEN_ENTRY;
+  entry_callback_->Release();
+
+  if (entry_ && result && result == buf_len_) {
+    linked_ptr<safe_browsing::ClientMalwareReportRequest::Resource>
+        pb_resource = resources_it_->second;
+    safe_browsing::ClientMalwareReportRequest::HTTPResponse* pb_response =
+        pb_resource->mutable_response();
+    if (buf_len_ <= kMaxBodySize) {  // Only send small bodies for now.
+      pb_response->set_body(buf_->data(), buf_len_);
+    }
+    pb_response->set_bodylength(buf_len_);
+    MD5Digest digest;
+    MD5Sum(buf_->data(), buf_len_, &digest);
+    pb_response->set_bodydigest(MD5DigestToBase16(digest));
+    entry_->Close();
+  }
+
+  ++resources_it_;  // Advance to the next entry.
+  return net::OK;
+}
+
+void MalwareDetails::DoCacheDone(int result) {
+  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
+  // Add all the urls in our |resources_| maps to the |report_| protocol buffer.
   for (ResourceMap::const_iterator it = resources_.begin();
        it != resources_.end(); it++) {
     ClientMalwareReportRequest::Resource* pb_resource =
         report_->add_resources();
     pb_resource->CopyFrom(*(it->second));
   }
+  report_->set_complete(result == net::OK);
 
+  callback_->Run(this);
+}
+
+const std::string* MalwareDetails::GetSerializedReport() {
+  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
   scoped_ptr<std::string> request_data(new std::string());
   if (!report_->SerializeToString(request_data.get())) {
     DLOG(ERROR) << "Unable to serialize the malware report.";