set content length limit if half open range request fails

content/browser/download/download_file_impl.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "content/browser/download/download_file_impl.h"
 
 #include <string>
 #include <utility>
 
 #include "base/bind.h"
@@ skipping 30 matching lines @@
 // initial delay is specified by kInitialRenameRetryDelayMs.
 const int kInitialRenameRetryDelayMs = 200;
 
 // Number of times a failing rename is retried before giving up.
 const int kMaxRenameRetries = 3;
 
 // Because DownloadSaveInfo::kLengthFullContent is 0, we should avoid using
 // 0 for length if we found that a stream can no longer write any data.
 const int kNoBytesToWrite = -1;
 
+// Default content length limit when the file size is not yet determined.
+const int kUnknownContentLengthLimit = -1;
+
 }  // namespace
 
 DownloadFileImpl::SourceStream::SourceStream(
     int64_t offset,
     int64_t length,
     std::unique_ptr<ByteStreamReader> stream_reader)
     : offset_(offset),
       length_(length),
       bytes_written_(0),
       finished_(false),
       index_(0u),
       stream_reader_(std::move(stream_reader)) {}
 
 DownloadFileImpl::SourceStream::~SourceStream() = default;
 
 void DownloadFileImpl::SourceStream::OnWriteBytesToDisk(int64_t bytes_write) {
   bytes_written_ += bytes_write;
 }
 
 void DownloadFileImpl::SourceStream::TruncateLengthWithWrittenDataBlock(
     int64_t offset,
     int64_t bytes_written) {
   DCHECK_GT(bytes_written, 0);
   if (length_ == kNoBytesToWrite)
     return;
 
   if (offset <= offset_) {
-    if (offset + bytes_written > offset_)
+    if (offset + bytes_written > offset_) {
       length_ = kNoBytesToWrite;
+      finished_ = true;
+    }
     return;
   }
 
   if (length_ == DownloadSaveInfo::kLengthFullContent ||
       length_ > offset - offset_) {
     length_ = offset - offset_;
   }
 }
 
 DownloadFileImpl::DownloadFileImpl(
     std::unique_ptr<DownloadSaveInfo> save_info,
     const base::FilePath& default_download_directory,
     std::unique_ptr<ByteStreamReader> stream_reader,
     const std::vector<DownloadItem::ReceivedSlice>& received_slices,
     const net::NetLogWithSource& download_item_net_log,
     base::WeakPtr<DownloadDestinationObserver> observer)
     : net_log_(
           net::NetLogWithSource::Make(download_item_net_log.net_log(),
                                       net::NetLogSourceType::DOWNLOAD_FILE)),
       file_(net_log_),
       save_info_(std::move(save_info)),
       default_download_directory_(default_download_directory),
+      content_length_limit_(kUnknownContentLengthLimit),
       bytes_seen_(0),
       num_active_streams_(0),
       record_stream_bandwidth_(true),
       bytes_seen_with_parallel_streams_(0),
       bytes_seen_without_parallel_streams_(0),
       received_slices_(received_slices),
       observer_(observer),
       weak_factory_(this) {
   source_streams_[save_info_->offset] = base::MakeUnique<SourceStream>(
       save_info_->offset, save_info_->length, std::move(stream_reader));
@@ skipping 225 matching lines @@
 }
 
 void DownloadFileImpl::Detach() {
   file_.Detach();
 }
 
 void DownloadFileImpl::Cancel() {
   file_.Cancel();
 }
 
+void DownloadFileImpl::SetContentLengthLimit(int64_t limit) {
+  DCHECK(content_length_limit_ == limit ||
+         content_length_limit_ == kUnknownContentLengthLimit)
+      << "Content length changed, the download might have updated.";

[xingliu, 2017/03/31 23:38:21]

nit%: Since content length is the size of the payload, can we change the comment
to file size changed?

[qinmin, 2017/04/01 05:20:01]

Done.

+
+  if (limit < content_length_limit_ ||
+      content_length_limit_ == kUnknownContentLengthLimit) {
+    content_length_limit_ = limit;

[xingliu, 2017/03/31 23:38:21]

Will we hit the DCHECK above when limit < content_length_limit_?

Maybe (limit < content_length_limit_) is only for release build, and we expect
to hit a DCHECK in debug build when file size changed?

[qinmin, 2017/04/01 05:20:02]

Yes, it will cause the DCHECK failure.
The DCHECK will tell us if this really happens in development build, so we can
analyze why this would happen. For release build, we don't want to crash the
app.

One possible scenario is that the content-length header is too large (e.g 100M).
So when the parallel request is asking for 50M+, the server returns a range
error, and we will set the content_length_limit_ here. When the initial request
completes, it founds that the file is only 20M, and SetContentLengthLimit() is
called again. In this case, we should use 20M as the file size. 

[xingliu, 2017/04/03 16:51:56]

I see. sgtm.

+  }
+
+  // TODO(qinmin): interrupt the download if the received bytes are larger
+  // than content length limit.
+  LOG_IF(ERROR, TotalBytesReceived() > content_length_limit_)
+      << "Received data is larger than the content length limit.";
+}
+
 const base::FilePath& DownloadFileImpl::FullPath() const {
   return file_.full_path();
 }
 
 bool DownloadFileImpl::InProgress() const {
   return file_.in_progress();
 }
 
 void DownloadFileImpl::WasPaused() {
   record_stream_bandwidth_ = false;
@@ skipping 82 matching lines @@
 
   RecordContiguousWriteTime(now - start);
 
   // Take care of communication with our observer.
   if (reason != DOWNLOAD_INTERRUPT_REASON_NONE) {
     HandleStreamError(source_stream, reason);
   } else if (state == ByteStreamReader::STREAM_COMPLETE || should_terminate) {
     // Signal successful completion or termination of the current stream.
     source_stream->stream_reader()->RegisterCallback(base::Closure());
     source_stream->set_finished(true);
+    if (source_stream->length() == DownloadSaveInfo::kLengthFullContent) {
+      SetContentLengthLimit(source_stream->offset() +
+                            source_stream->bytes_written());
+    }
     num_active_streams_--;
 
     // Inform observers.
     SendUpdate();
 
     // All the stream reader are completed, shut down file IO processing.
     if (IsDownloadCompleted()) {
       RecordFileBandwidth(bytes_seen_, disk_writes_time_,
                           base::TimeTicks::Now() - download_start_);
       if (IsSparseFile() && record_stream_bandwidth_) {
@@ skipping 82 matching lines @@
         source_stream->set_index(source_stream->index() + 1);
     } else if (source_stream->offset() == offset) {
       source_stream->set_index(index);
     } else {
       source_stream->TruncateLengthWithWrittenDataBlock(offset, length);
     }
   }
 }
 
 bool DownloadFileImpl::IsDownloadCompleted() {
-  SourceStream* stream_for_last_slice = nullptr;
-  int64_t last_slice_offset = 0;
   for (auto& stream : source_streams_) {
-    SourceStream* source_stream = stream.second.get();
-    if (source_stream->offset() >= last_slice_offset &&
-        source_stream->bytes_written() > 0) {
-      stream_for_last_slice = source_stream;
-      last_slice_offset = source_stream->offset();
-    }
-    if (!source_stream->is_finished())
+    if (!stream.second->is_finished())
       return false;
   }
 
   if (!IsSparseFile())
     return true;
 
   // Verify that all the file slices have been downloaded.
   std::vector<DownloadItem::ReceivedSlice> slices_to_download =
       FindSlicesToDownload(received_slices_);
   if (slices_to_download.size() > 1) {
     // If there are 1 or more holes in the file, download is not finished.
     // Some streams might not have been added to |source_streams_| yet.
     return false;
   }
-  if (stream_for_last_slice) {
-    DCHECK_EQ(slices_to_download[0].received_bytes,
-              DownloadSaveInfo::kLengthFullContent);
-    // The last stream should not have a length limit. If it has, it might
-    // not reach the end of the file.
-    if (stream_for_last_slice->length() !=
-        DownloadSaveInfo::kLengthFullContent) {
-      return false;
-    }
-    DCHECK_EQ(slices_to_download[0].offset,
-              stream_for_last_slice->offset() +
-                  stream_for_last_slice->bytes_written());
-  }
-
-  return true;
+  return TotalBytesReceived() == content_length_limit_;
 }
 
 void DownloadFileImpl::HandleStreamError(SourceStream* source_stream,
                                          DownloadInterruptReason reason) {
   DCHECK_CURRENTLY_ON(BrowserThread::FILE);
   source_stream->stream_reader()->RegisterCallback(base::Closure());
   source_stream->set_finished(true);
   num_active_streams_--;
 
   bool can_recover_from_error = false;
@@ skipping 93 matching lines @@
     const base::FilePath& new_path,
     const RenameCompletionCallback& completion_callback)
     : option(option),
       new_path(new_path),
       retries_left(kMaxRenameRetries),
       completion_callback(completion_callback) {}
 
 DownloadFileImpl::RenameParameters::~RenameParameters() {}
 
 }  // namespace content