Make conditional requests when updating the appcache....

webkit/appcache/appcache_update_job.cc

 // Copyright (c) 2010 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "webkit/appcache/appcache_update_job.h"
 
 #include "base/compiler_specific.h"
 #include "base/message_loop.h"
 #include "base/string_util.h"
 #include "net/base/io_buffer.h"
 #include "net/base/load_flags.h"
 #include "net/base/net_errors.h"
 #include "net/http/http_request_headers.h"
 #include "webkit/appcache/appcache_group.h"
 #include "webkit/appcache/appcache_policy.h"
-#include "webkit/appcache/appcache_response.h"
 
 namespace appcache {
 
 static const int kBufferSize = 32768;
 static const size_t kMaxConcurrentUrlFetches = 2;
 static const int kMax503Retries = 3;
 
 // Extra info associated with requests for use during response processing.
 // This info is deleted when the URLRequest is deleted.
 class UpdateJobInfo :  public URLRequest::UserData {
  public:
   enum RequestType {
     MANIFEST_FETCH,
     URL_FETCH,
     MASTER_ENTRY_FETCH,
     MANIFEST_REFETCH,
   };
 
   explicit UpdateJobInfo(RequestType request_type)
       : type_(request_type),
         buffer_(new net::IOBuffer(kBufferSize)),
         retry_503_attempts_(0),
         update_job_(NULL),
         request_(NULL),
         ALLOW_THIS_IN_INITIALIZER_LIST(write_callback_(
             this, &UpdateJobInfo::OnWriteComplete)) {
   }
 
   void SetUpResponseWriter(AppCacheResponseWriter* writer,
-                         AppCacheUpdateJob* update,
-                         URLRequest* request) {
+                           AppCacheUpdateJob* update,
+                           URLRequest* request) {
     DCHECK(!response_writer_.get());
     response_writer_.reset(writer);
     update_job_ = update;
     request_ = request;
   }
 
   void OnWriteComplete(int result) {
     // A completed write may delete the URL request and this object.
     update_job_->OnWriteResponseComplete(result, request_, this);
   }
 
   RequestType type_;
   scoped_refptr<net::IOBuffer> buffer_;
   int retry_503_attempts_;
 
+  // The entry from the newest cache for this url, used for 304 responses.
+  AppCacheEntry existing_entry_;
+
   // Info needed to write responses to storage and process callbacks.
   scoped_ptr<AppCacheResponseWriter> response_writer_;
   AppCacheUpdateJob* update_job_;
   URLRequest* request_;
   net::CompletionCallbackImpl<UpdateJobInfo> write_callback_;
 };
 
 // Helper class for collecting hosts per frontend when sending notifications
 // so that only one notification is sent for all hosts using the same frontend.
 class HostNotifier {
@@ skipping 79 matching lines @@
   DCHECK(!inprogress_cache_);
   DCHECK(pending_master_entries_.empty());
   DCHECK(master_entry_fetches_.empty());
 
   if (group_)
     group_->SetUpdateStatus(AppCacheGroup::IDLE);
 
   policy_callback_->Cancel();
 }
 
+UpdateJobInfo* AppCacheUpdateJob::GetUpdateJobInfo(URLRequest* request) {
+  return static_cast<UpdateJobInfo*>(request->GetUserData(this));
+}
+
 void AppCacheUpdateJob::StartUpdate(AppCacheHost* host,
                                     const GURL& new_master_resource) {
   DCHECK(group_->update_job() == this);
   DCHECK(!group_->is_obsolete());
 
   bool is_new_pending_master_entry = false;
   if (!new_master_resource.is_empty()) {
     DCHECK(new_master_resource == host->pending_master_entry_url());
     DCHECK(!new_master_resource.has_ref());
     DCHECK(new_master_resource.GetOrigin() == manifest_url_.GetOrigin());
@@ skipping 106 matching lines @@
 
   // Add any necessary Http headers before sending fetch request.
   if (is_first_fetch) {
     AppCacheEntry* entry = (update_type_ == UPGRADE_ATTEMPT) ?
         group_->newest_complete_cache()->GetEntry(manifest_url_) : NULL;
     if (entry) {
       // Asynchronously load response info for manifest from newest cache.
       service_->storage()->LoadResponseInfo(manifest_url_,
                                             entry->response_id(), this);
     } else {
-      AddHttpHeadersAndFetch(manifest_url_request_, NULL);
+      manifest_url_request_->Start();
     }
   } else {
     DCHECK(internal_state_ == REFETCH_MANIFEST);
     DCHECK(manifest_response_info_.get());
-    AddHttpHeadersAndFetch(manifest_url_request_,
-                           manifest_response_info_.get());
+    AddConditionalHeaders(manifest_url_request_,
+                          manifest_response_info_.get());
+    manifest_url_request_->Start();
   }
 }
 
-void AppCacheUpdateJob::AddHttpHeadersAndFetch(
+void AppCacheUpdateJob::AddConditionalHeaders(
     URLRequest* request, const net::HttpResponseInfo* info) {
-  DCHECK(request);
-  if (info) {
-    net::HttpRequestHeaders extra_headers;
+  DCHECK(request && info);
+  net::HttpRequestHeaders extra_headers;
 
-    // Add If-Modified-Since header if response info has Last-Modified header.
-    const std::string last_modified = "Last-Modified";
-    std::string last_modified_value;
-    info->headers->EnumerateHeader(NULL, last_modified, &last_modified_value);
-    if (!last_modified_value.empty()) {
-      extra_headers.SetHeader(net::HttpRequestHeaders::kIfModifiedSince,
-                              last_modified_value);
-    }
+  // Add If-Modified-Since header if response info has Last-Modified header.
+  const std::string last_modified = "Last-Modified";
+  std::string last_modified_value;
+  info->headers->EnumerateHeader(NULL, last_modified, &last_modified_value);
+  if (!last_modified_value.empty()) {
+    extra_headers.SetHeader(net::HttpRequestHeaders::kIfModifiedSince,
+                            last_modified_value);
+  }
 
-    // Add If-None-Match header if resposne info has ETag header.
-    const std::string etag = "ETag";
-    std::string etag_value;
-    info->headers->EnumerateHeader(NULL, etag, &etag_value);
-    if (!etag_value.empty()) {
-      extra_headers.SetHeader(net::HttpRequestHeaders::kIfNoneMatch,
-                              etag_value);
-    }
-
-    if (!extra_headers.IsEmpty())
-      request->SetExtraRequestHeaders(extra_headers);
+  // Add If-None-Match header if resposne info has ETag header.
+  const std::string etag = "ETag";
+  std::string etag_value;
+  info->headers->EnumerateHeader(NULL, etag, &etag_value);
+  if (!etag_value.empty()) {
+    extra_headers.SetHeader(net::HttpRequestHeaders::kIfNoneMatch,
+                            etag_value);
   }
-  request->Start();
+  if (!extra_headers.IsEmpty())
+    request->SetExtraRequestHeaders(extra_headers);
 }
 
 void AppCacheUpdateJob::OnResponseStarted(URLRequest *request) {
   if (request->status().is_success() &&
       (request->GetResponseCode() / 100) == 2) {
     // Write response info to storage for URL fetches. Wait for async write
     // completion before reading any response data.
-    UpdateJobInfo* info =
-        static_cast<UpdateJobInfo*>(request->GetUserData(this));
+    UpdateJobInfo* info = GetUpdateJobInfo(request);
     if (info->type_ == UpdateJobInfo::URL_FETCH ||
         info->type_ == UpdateJobInfo::MASTER_ENTRY_FETCH) {
       info->SetUpResponseWriter(
           service_->storage()->CreateResponseWriter(manifest_url_),
           this, request);
       stored_response_ids_.push_back(info->response_writer_->response_id());
       scoped_refptr<HttpResponseInfoIOBuffer> io_buffer =
           new HttpResponseInfoIOBuffer(
               new net::HttpResponseInfo(request->response_info()));
       info->response_writer_->WriteInfo(io_buffer, &info->write_callback_);
     } else {
       ReadResponseData(request);
     }
   } else {
     OnResponseCompleted(request);
   }
 }
 
 void AppCacheUpdateJob::ReadResponseData(URLRequest* request) {
   if (internal_state_ == CACHE_FAILURE || internal_state_ == CANCELLED ||
       internal_state_ == COMPLETED) {
     return;
   }
 
   int bytes_read = 0;
-  UpdateJobInfo* info =
-      static_cast<UpdateJobInfo*>(request->GetUserData(this));
+  UpdateJobInfo* info = GetUpdateJobInfo(request);
   request->Read(info->buffer_, kBufferSize, &bytes_read);
   OnReadCompleted(request, bytes_read);
 }
 
 void AppCacheUpdateJob::OnReadCompleted(URLRequest* request, int bytes_read) {
   bool data_consumed = true;
   if (request->status().is_success() && bytes_read > 0) {
-    UpdateJobInfo* info =
-        static_cast<UpdateJobInfo*>(request->GetUserData(this));
-
+    UpdateJobInfo* info = GetUpdateJobInfo(request);
     data_consumed = ConsumeResponseData(request, info, bytes_read);
     if (data_consumed) {
       bytes_read = 0;
       while (request->Read(info->buffer_, kBufferSize, &bytes_read)) {
         if (bytes_read > 0) {
           data_consumed = ConsumeResponseData(request, info, bytes_read);
           if (!data_consumed)
             break;  // wait for async data processing, then read more
         } else {
           break;
@@ skipping 50 matching lines @@
 }
 
 void AppCacheUpdateJob::OnResponseCompleted(URLRequest* request) {
   // Retry for 503s where retry-after is 0.
   if (request->status().is_success() &&
       request->GetResponseCode() == 503 &&
       RetryRequest(request)) {
     return;
   }
 
-  UpdateJobInfo* info =
-      static_cast<UpdateJobInfo*>(request->GetUserData(this));
+  UpdateJobInfo* info = GetUpdateJobInfo(request);
   switch (info->type_) {
     case UpdateJobInfo::MANIFEST_FETCH:
       HandleManifestFetchCompleted(request);
       break;
     case UpdateJobInfo::URL_FETCH:
       HandleUrlFetchCompleted(request);
       break;
     case UpdateJobInfo::MASTER_ENTRY_FETCH:
       HandleMasterEntryFetchCompleted(request);
       break;
     case UpdateJobInfo::MANIFEST_REFETCH:
       HandleManifestRefetchCompleted(request);
       break;
     default:
       NOTREACHED();
   }
 
   delete request;
 }
 
 bool AppCacheUpdateJob::RetryRequest(URLRequest* request) {
-  UpdateJobInfo* info =
-      static_cast<UpdateJobInfo*>(request->GetUserData(this));
+  UpdateJobInfo* info = GetUpdateJobInfo(request);
   if (info->retry_503_attempts_ >= kMax503Retries) {
     return false;
   }
 
   if (!request->response_headers()->HasHeaderValue("retry-after", "0"))
     return false;
 
   const GURL& url = request->original_url();
   URLRequest* retry = new URLRequest(url, this);
   UpdateJobInfo* retry_info = new UpdateJobInfo(info->type_);
   retry_info->retry_503_attempts_ = info->retry_503_attempts_ + 1;
+  retry_info->existing_entry_ = info->existing_entry_;
   retry->SetUserData(this, retry_info);
   retry->set_context(request->context());
   retry->set_load_flags(request->load_flags());
 
   switch (info->type_) {
     case UpdateJobInfo::MANIFEST_FETCH:
     case UpdateJobInfo::MANIFEST_REFETCH:
       manifest_url_request_ = retry;
       manifest_data_.clear();
       break;
@@ skipping 103 matching lines @@
 
   group_->SetUpdateStatus(AppCacheGroup::DOWNLOADING);
   NotifyAllAssociatedHosts(DOWNLOADING_EVENT);
   FetchUrls();
   FetchMasterEntries();
   MaybeCompleteUpdate();  // if not done, continues when async fetches complete
 }
 
 void AppCacheUpdateJob::HandleUrlFetchCompleted(URLRequest* request) {
   DCHECK(internal_state_ == DOWNLOADING);
+  UpdateJobInfo* info = GetUpdateJobInfo(request);
 
   const GURL& url = request->original_url();
   pending_url_fetches_.erase(url);
   NotifyAllProgress(url);
   ++url_fetches_completed_;
 
   int response_code = request->status().is_success()
       ? request->GetResponseCode() : -1;
   AppCacheEntry& entry = url_file_list_.find(url)->second;
 
-  if (request->status().is_success() && (response_code / 100 == 2)) {
+  if (response_code / 100 == 2) {
     // Associate storage with the new entry.
-    UpdateJobInfo* info =
-        static_cast<UpdateJobInfo*>(request->GetUserData(this));
     DCHECK(info->response_writer_.get());
     entry.set_response_id(info->response_writer_->response_id());
     entry.set_response_size(info->response_writer_->amount_written());
-
     if (!inprogress_cache_->AddOrModifyEntry(url, entry))
       duplicate_response_ids_.push_back(entry.response_id());
 
     // Foreign entries will be detected during cache selection.
     // Note: 6.9.4, step 17.9 possible optimization: if resource is HTML or XML
     // file whose root element is an html element with a manifest attribute
     // whose value doesn't match the manifest url of the application cache
     // being processed, mark the entry as being foreign.
   } else {
     LOG(INFO) << "Request status: " << request->status().status()
         << " os_error: " << request->status().os_error()
         << " response code: " << response_code;
     if (entry.IsExplicit() || entry.IsFallback()) {
-      const char* kFormatString = "Resource fetch failed (%d) %s";
-      const std::string message = StringPrintf(kFormatString, response_code,
-                                               request->url().spec().c_str());
-      HandleCacheFailure(message);
-      return;
+      if (response_code == 304 && info->existing_entry_.has_response_id()) {
+        // Copy the response from the newest complete cache.

[rvargas (doing something else), 2010/07/20 19:17:18]

maybe change from the newest complete cache -> from the existing entry
(same at line 644)

+        entry.set_response_id(info->existing_entry_.response_id());
+        entry.set_response_size(info->existing_entry_.response_size());
+        inprogress_cache_->AddOrModifyEntry(url, entry);
+      } else {
+        const char* kFormatString = "Resource fetch failed (%d) %s";
+        const std::string message = StringPrintf(kFormatString, response_code,
+                                                 request->url().spec().c_str());
+        HandleCacheFailure(message);
+        return;
+      }
     } else if (response_code == 404 || response_code == 410) {
       // Entry is skipped.  They are dropped from the cache.
-    } else if (update_type_ == UPGRADE_ATTEMPT) {
+    } else if (update_type_ == UPGRADE_ATTEMPT &&
+               info->existing_entry_.has_response_id()) {
       // Copy the response from the newest complete cache.
-      AppCache* cache = group_->newest_complete_cache();
-      AppCacheEntry* copy = cache->GetEntry(url);
-      if (copy) {
-        entry.set_response_id(copy->response_id());
-        entry.set_response_size(copy->response_size());
-        inprogress_cache_->AddOrModifyEntry(url, entry);
-      }
+      // TODO(michaeln): Not sure this is a good idea. This is spec compliant
+      // but the old resource may or may not be compatible with the new contents
+      // of the cache. Impossible to know one way or the other.
+      entry.set_response_id(info->existing_entry_.response_id());
+      entry.set_response_size(info->existing_entry_.response_size());
+      inprogress_cache_->AddOrModifyEntry(url, entry);
     }
   }
 
   // Fetch another URL now that one request has completed.
   DCHECK(internal_state_ != CACHE_FAILURE);
   FetchUrls();
   MaybeCompleteUpdate();
 }
 
 void AppCacheUpdateJob::HandleMasterEntryFetchCompleted(URLRequest* request) {
@@ skipping 11 matching lines @@
   int response_code = request->status().is_success()
       ? request->GetResponseCode() : -1;
 
   PendingMasters::iterator found = pending_master_entries_.find(url);
   DCHECK(found != pending_master_entries_.end());
   PendingHosts& hosts = found->second;
 
   // Section 6.9.4. No update case: step 7.3, else step 22.
   if (response_code / 100 == 2) {
     // Add fetched master entry to the appropriate cache.
-    UpdateJobInfo* info =
-        static_cast<UpdateJobInfo*>(request->GetUserData(this));
+    UpdateJobInfo* info = GetUpdateJobInfo(request);
     AppCache* cache = inprogress_cache_ ? inprogress_cache_.get() :
         group_->newest_complete_cache();
     DCHECK(info->response_writer_.get());
     AppCacheEntry master_entry(AppCacheEntry::MASTER,
                                info->response_writer_->response_id(),
                                info->response_writer_->amount_written());
     if (!cache->AddOrModifyEntry(url, master_entry))
       duplicate_response_ids_.push_back(master_entry.response_id());
 
     // In no-update case, associate host with the newest cache.
@@ skipping 245 matching lines @@
         AddUrlToFileList(it->first, AppCacheEntry::MASTER);
     }
   }
 }
 
 void AppCacheUpdateJob::AddUrlToFileList(const GURL& url, int type) {
   std::pair<AppCache::EntryMap::iterator, bool> ret = url_file_list_.insert(
       AppCache::EntryMap::value_type(url, AppCacheEntry(type)));
 
   if (ret.second)
-    urls_to_fetch_.push_back(UrlsToFetch(url, false));
+    urls_to_fetch_.push_back(UrlToFetch(url, false, NULL));
   else
     ret.first->second.add_types(type);  // URL already exists. Merge types.
 }
 
 void AppCacheUpdateJob::FetchUrls() {
   DCHECK(internal_state_ == DOWNLOADING);
 
   // Fetch each URL in the list according to section 6.9.4 step 17.1-17.3.
   // Fetch up to the concurrent limit. Other fetches will be triggered as each
   // each fetch completes.
   while (pending_url_fetches_.size() < kMaxConcurrentUrlFetches &&
          !urls_to_fetch_.empty()) {
-    const GURL url = urls_to_fetch_.front().first;
-    bool storage_checked = urls_to_fetch_.front().second;
+    UrlToFetch url_to_fetch = urls_to_fetch_.front();
     urls_to_fetch_.pop_front();
 
-    AppCache::EntryMap::iterator it = url_file_list_.find(url);
+    AppCache::EntryMap::iterator it = url_file_list_.find(url_to_fetch.url);
     DCHECK(it != url_file_list_.end());
     AppCacheEntry& entry = it->second;
     if (ShouldSkipUrlFetch(entry)) {
-      NotifyAllProgress(url);
+      NotifyAllProgress(url_to_fetch.url);
       ++url_fetches_completed_;
-    } else if (AlreadyFetchedEntry(url, entry.types())) {
-      NotifyAllProgress(url);
+    } else if (AlreadyFetchedEntry(url_to_fetch.url, entry.types())) {
+      NotifyAllProgress(url_to_fetch.url);
       ++url_fetches_completed_;  // saved a URL request
-    } else if (!storage_checked && MaybeLoadFromNewestCache(url, entry)) {
+    } else if (!url_to_fetch.storage_checked &&
+               MaybeLoadFromNewestCache(url_to_fetch.url, entry)) {
       // Continues asynchronously after data is loaded from newest cache.
     } else {
+      UpdateJobInfo* info = new UpdateJobInfo(UpdateJobInfo::URL_FETCH);
+      const net::HttpResponseInfo* http_info = NULL;
+      if (url_to_fetch.existing_response_info.get()) {
+        DCHECK(group_->newest_complete_cache());
+        AppCacheEntry* existing_entry =
+            group_->newest_complete_cache()->GetEntry(url_to_fetch.url);
+        DCHECK(existing_entry);
+        DCHECK(existing_entry->response_id() ==
+               url_to_fetch.existing_response_info->response_id());
+        info->existing_entry_ = *existing_entry;
+        http_info = url_to_fetch.existing_response_info->http_response_info();
+      }
+
       // Send URL request for the resource.
-      URLRequest* request = new URLRequest(url, this);
-      request->SetUserData(this, new UpdateJobInfo(UpdateJobInfo::URL_FETCH));
+      URLRequest* request = new URLRequest(url_to_fetch.url, this);
+      request->SetUserData(this, info);
       request->set_context(service_->request_context());
       request->set_load_flags(
           request->load_flags() | net::LOAD_DISABLE_INTERCEPT);
+      if (http_info)
+        AddConditionalHeaders(request, http_info);
       request->Start();
-      pending_url_fetches_.insert(PendingUrlFetches::value_type(url, request));
+      pending_url_fetches_.insert(
+          PendingUrlFetches::value_type(url_to_fetch.url, request));
     }
   }
 }
 
 void AppCacheUpdateJob::CancelAllUrlFetches() {
   // Cancel any pending URL requests.
   for (PendingUrlFetches::iterator it = pending_url_fetches_.begin();
        it != pending_url_fetches_.end(); ++it) {
     delete it->second;
   }
@@ skipping 161 matching lines @@
   return true;
 }
 
 void AppCacheUpdateJob::OnResponseInfoLoaded(
     AppCacheResponseInfo* response_info, int64 response_id) {
   const net::HttpResponseInfo* http_info = response_info ?
       response_info->http_response_info() : NULL;
 
   // Needed response info for a manifest fetch request.
   if (internal_state_ == FETCH_MANIFEST) {
-    AddHttpHeadersAndFetch(manifest_url_request_, http_info);
+    if (http_info)
+      AddConditionalHeaders(manifest_url_request_, http_info);
+    manifest_url_request_->Start();
     return;
   }
 
   LoadingResponses::iterator found = loading_responses_.find(response_id);
   DCHECK(found != loading_responses_.end());
   const GURL& url = found->second;
 
   if (!http_info) {
-    LoadFromNewestCacheFailed(url);  // no response found
+    LoadFromNewestCacheFailed(url, NULL);  // no response found
   } else {
     // Check if response can be re-used according to HTTP caching semantics.
     // Responses with a "vary" header get treated as expired.
     const std::string name = "vary";
     std::string value;
     void* iter = NULL;
     if (http_info->headers->RequiresValidation(http_info->request_time,
                                                http_info->response_time,
                                                base::Time::Now()) ||
         http_info->headers->EnumerateHeader(&iter, name, &value)) {
       // TODO(michaeln): Make a conditional request when we can in this case.
-      LoadFromNewestCacheFailed(url);
+      LoadFromNewestCacheFailed(url, response_info);
     } else {
       DCHECK(group_->newest_complete_cache());
       AppCacheEntry* copy_me = group_->newest_complete_cache()->GetEntry(url);
       DCHECK(copy_me);
       DCHECK(copy_me->response_id() == response_id);
 
       AppCache::EntryMap::iterator it = url_file_list_.find(url);
       DCHECK(it != url_file_list_.end());
       AppCacheEntry& entry = it->second;
       entry.set_response_id(response_id);
       entry.set_response_size(copy_me->response_size());
       inprogress_cache_->AddOrModifyEntry(url, entry);
       NotifyAllProgress(url);
       ++url_fetches_completed_;
     }
   }
   loading_responses_.erase(found);
 
   MaybeCompleteUpdate();
 }
 
-void AppCacheUpdateJob::LoadFromNewestCacheFailed(const GURL& url) {
+void AppCacheUpdateJob::LoadFromNewestCacheFailed(
+    const GURL& url, AppCacheResponseInfo* response_info) {
   if (internal_state_ == CACHE_FAILURE)
     return;
 
   // Re-insert url at front of fetch list. Indicate storage has been checked.
-  urls_to_fetch_.push_front(AppCacheUpdateJob::UrlsToFetch(url, true));
+  urls_to_fetch_.push_front(UrlToFetch(url, true, response_info));
   FetchUrls();
 }
 
 void AppCacheUpdateJob::MaybeCompleteUpdate() {
   DCHECK(internal_state_ != CACHE_FAILURE);
 
   // Must wait for any pending master entries or url fetches to complete.
   if (master_entries_completed_ != pending_master_entries_.size() ||
       url_fetches_completed_ != url_file_list_.size()) {
     DCHECK(internal_state_ != COMPLETED);
@@ skipping 120 matching lines @@
 
   // Break the connection with the group so the group cannot call delete
   // on this object after we've posted a task to delete ourselves.
   group_->SetUpdateStatus(AppCacheGroup::IDLE);
   group_ = NULL;
 
   MessageLoop::current()->DeleteSoon(FROM_HERE, this);
 }
 
 }  // namespace appcache