Update the received slices vector when stream is written to disk

content/browser/download/download_file_impl.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "content/browser/download/download_file_impl.h"
 
 #include <string>
 #include <utility>
 
 #include "base/bind.h"
 #include "base/files/file_util.h"
 #include "base/memory/ptr_util.h"
 #include "base/strings/stringprintf.h"
 #include "base/time/time.h"
 #include "base/values.h"
 #include "content/browser/byte_stream.h"
 #include "content/browser/download/download_create_info.h"
 #include "content/browser/download/download_destination_observer.h"
 #include "content/browser/download/download_interrupt_reasons_impl.h"
 #include "content/browser/download/download_net_log_parameters.h"
 #include "content/browser/download/download_stats.h"
+#include "content/browser/download/parallel_download_utils.h"
 #include "content/public/browser/browser_thread.h"
 #include "crypto/secure_hash.h"
 #include "crypto/sha2.h"
 #include "net/base/io_buffer.h"
 #include "net/log/net_log.h"
 #include "net/log/net_log_event_type.h"
 #include "net/log/net_log_source.h"
 #include "net/log/net_log_source_type.h"
 
 namespace content {
 
 const int kUpdatePeriodMs = 500;
 const int kMaxTimeBlockingFileThreadMs = 1000;
 
 // These constants control the default retry behavior for failing renames. Each
 // retry is performed after a delay that is twice the previous delay. The
 // initial delay is specified by kInitialRenameRetryDelayMs.
 const int kInitialRenameRetryDelayMs = 200;
 
 // Number of times a failing rename is retried before giving up.
 const int kMaxRenameRetries = 3;
 
 DownloadFileImpl::SourceStream::SourceStream(int64_t offset, int64_t length)
-    : offset_(offset), length_(length), bytes_written_(0), finished_(false) {}
+    : offset_(offset),
+      length_(length),
+      bytes_written_(0),
+      finished_(false),
+      index_(0) {}
 
 DownloadFileImpl::SourceStream::~SourceStream() = default;
 
 void DownloadFileImpl::SourceStream::SetByteStream(
     std::unique_ptr<ByteStreamReader> stream_reader) {
   stream_reader_ = std::move(stream_reader);
 }
 
 void DownloadFileImpl::SourceStream::OnWriteBytesToDisk(int64_t bytes_write) {
   bytes_written_ += bytes_write;
@@ skipping 249 matching lines @@
             should_terminate = true;
             incoming_data_size =
                 source_stream->length() - source_stream->bytes_written();
           }
           reason = WriteDataToFile(
               source_stream->offset() + source_stream->bytes_written(),
               incoming_data.get()->data(), incoming_data_size);
           disk_writes_time_ += (base::TimeTicks::Now() - write_start);
           bytes_seen_ += incoming_data_size;
           total_incoming_data_size += incoming_data_size;
-          if (reason == DOWNLOAD_INTERRUPT_REASON_NONE)
+          if (reason == DOWNLOAD_INTERRUPT_REASON_NONE) {
+            int64_t prev_bytes_written = source_stream->bytes_written();
             source_stream->OnWriteBytesToDisk(incoming_data_size);
+            // If the write operation creates a new slice, add it to the
+            // |received_slices_| and update all the entries in
+            // |source_streams_|.
+            int64_t current_bytes_written = source_stream->bytes_written();
+            if (current_bytes_written > 0 && prev_bytes_written == 0) {
+              AddNewSlice(source_stream->offset(), current_bytes_written);
+            } else {
+              received_slices_[source_stream->index()].received_bytes =
+                  current_bytes_written;
+            }
+          }
         }
         break;
       case ByteStreamReader::STREAM_COMPLETE:
         {
           reason = static_cast<DownloadInterruptReason>(
             source_stream->stream_reader()->GetStatus());
           SendUpdate();
         }
         break;
       default:
@@ skipping 97 matching lines @@
 
 void DownloadFileImpl::WillWriteToDisk(size_t data_len) {
   if (!update_timer_->IsRunning()) {
     update_timer_->Start(FROM_HERE,
                          base::TimeDelta::FromMilliseconds(kUpdatePeriodMs),
                          this, &DownloadFileImpl::SendUpdate);
   }
   rate_estimator_.Increment(data_len);
 }
 
+void DownloadFileImpl::AddNewSlice(int64_t offset, int64_t length) {

[xingliu, 2017/03/07 22:18:13]

This in general sgtm.

Question: after adding a new slice to ReceivedSlice, when we write back to the
db, does it means we will have more db records? 

e.g, we resume a parallel download with 2 received slices, and we find slices to
download, finally we have 4 received slices.


I think we should avoid to create more and more db records.

[qinmin, 2017/03/08 18:46:33]

Yes, we will.  We can merge some of the slices together when db initializes, but
that's not a top priority item. 
If we merge new slices with existing slices, then we have to set the
bytes_received to non-zero value for each stream,  which is very wierd.
the slice info will be cleared when download finishes, so hopefully it won't be
there forever.

[xingliu, 2017/03/08 19:45:18]

Maybe we should test this in very poor network condition that it won't spawn too
many db records.

+  if (!is_sparse_file_)
+    return;
+  int index = AddReceivedSliceToSortedArray(

[xingliu, 2017/03/08 19:45:18]

nit%: Just something to consider here, is it possible to add a new slice with
existing offset? This is dangerous since it will corrupt source stream states.

[qinmin, 2017/03/08 21:59:04]

This shouldn't happen. A slice will not be added to the db unless data is
downloaded. So a new request will always be offset+data_received. If the server
doesn't support range request, then we should not use parallel downloading.

+      DownloadItem::ReceivedSlice(offset, length), received_slices_);
+  // Update the index of exising SourceStreams.
+  for (auto& stream : source_streams_) {
+    SourceStream* source_stream = stream.second.get();
+    if (source_stream->offset() > offset) {
+      if (source_stream->bytes_written() > 0)
+        source_stream->set_index(source_stream->index() + 1);
+    } else if (source_stream->offset() == offset) {
+      source_stream->set_index(index);
+    } else if (source_stream->length() ==
+                   DownloadSaveInfo::kLengthFullContent ||
+               source_stream->length() > offset - source_stream->offset()) {
+      // The newly introduced slice will impact the length of the SourceStreams
+      // preceding it.
+      source_stream->set_length(offset - source_stream->offset());
+    }
+  }
+}
+
 DownloadFileImpl::RenameParameters::RenameParameters(
     RenameOption option,
     const base::FilePath& new_path,
     const RenameCompletionCallback& completion_callback)
     : option(option),
       new_path(new_path),
       retries_left(kMaxRenameRetries),
       completion_callback(completion_callback) {}
 
 DownloadFileImpl::RenameParameters::~RenameParameters() {}
 
 }  // namespace content