Make DownloadFileImpl handle multiple byte streams.

content/browser/download/download_file_impl.h

Index: content/browser/download/download_file_impl.h
diff --git a/content/browser/download/download_file_impl.h b/content/browser/download/download_file_impl.h
index 9a275190aa5e6cbfbadc36b95839cdafc32a9084..ff211e8b5fd87cc38411ecfb104a1f7aba937ad9 100644
--- a/content/browser/download/download_file_impl.h
+++ b/content/browser/download/download_file_impl.h
@@ -12,11 +12,13 @@
 
 #include <memory>
 #include <string>
+#include <unordered_map>
 
 #include "base/files/file.h"
 #include "base/macros.h"
 #include "base/memory/ref_counted.h"
 #include "base/memory/weak_ptr.h"
+#include "base/threading/thread_checker.h"
 #include "base/time/time.h"
 #include "base/timer/timer.h"
 #include "content/browser/byte_stream.h"
@@ -31,6 +33,56 @@ class DownloadDestinationObserver;
 
 class CONTENT_EXPORT DownloadFileImpl : public DownloadFile {
  public:
+  // Wrapper of a ByteStreamReader, and the meta data needed to write to a
+  // slice of the target file.
+  //
+  // Does not require the stream reader ready when constructor is called.
+  // |stream_reader_| can be set later when the network response is handled.
+  //
+  // Multiple SourceStreams can concurrently write to the same file sink.
+  //
+  // The file IO processing is finished when all SourceStreams are finished.
+  class CONTENT_EXPORT SourceStream {

[qinmin, 2017/02/27 18:55:29]

does this needs to be public?

[xingliu, 2017/02/28 00:57:06]

Makes sense, change it to private.

The unit test can access it through friend class.

+   public:
+    SourceStream(int64_t offset, int64_t bytes_received);
+    ~SourceStream();
+
+    void SetByteStream(std::unique_ptr<ByteStreamReader> stream_reader);
+
+    // Called when successfully read a buffer from the stream and write it to
+    // disk.
+    void OnWriteBytesToDisk(int64_t bytes_write);
+
+    ByteStreamReader* stream_reader() const { return stream_reader_.get(); }
+    int64_t offset() const { return offset_; }
+    int64_t bytes_received() const { return bytes_received_; }

[qinmin, 2017/02/27 18:55:29]

I think byte_received is misleading for a stream, bytes_written is better.

[xingliu, 2017/02/28 00:57:06]

Done, changed to bytes_written.

+    bool is_finished() const { return finished_; }
+    void set_finished(bool finish) { finished_ = finish; }
+
+   private:
+    // Starting position for the current slice.
+    int64_t offset_;
+
+    // Number of bytes received for the slice. Only get updated after bytes
+    // written to disk. The initial value may be read from download history.
+    // Will write to disk at (|offset_| + |bytes_received_|).
+    int64_t bytes_received_;
+
+    // If all the data read from the stream has been successfully write to disk.
+    bool finished_;
+
+    // The stream through which data comes.
+    // TODO(rdsmith): Move this into BaseFile; requires using the same
+    // stream semantics in SavePackage.  Alternatively, replace SaveFile
+    // with DownloadFile and get rid of BaseFile.
+    std::unique_ptr<ByteStreamReader> stream_reader_;
+
+    DISALLOW_COPY_AND_ASSIGN(SourceStream);
+  };
+
+  typedef std::unordered_map<int64_t, std::unique_ptr<SourceStream>>
+      SourceStreams;
+
   // Takes ownership of the object pointed to by |request_handle|.
   // |net_log| will be used for logging the download file's events.
   // May be constructed on any thread.  All methods besides the constructor
@@ -41,14 +93,19 @@ class CONTENT_EXPORT DownloadFileImpl : public DownloadFile {
   // DownloadDestinationObserver.
   DownloadFileImpl(std::unique_ptr<DownloadSaveInfo> save_info,
                    const base::FilePath& default_downloads_directory,
-                   std::unique_ptr<ByteStreamReader> byte_stream,
+                   std::unique_ptr<ByteStreamReader> stream_reader,
                    const net::NetLogWithSource& net_log,
+                   bool is_sparse_file,
                    base::WeakPtr<DownloadDestinationObserver> observer);
 
   ~DownloadFileImpl() override;
 
   // DownloadFile functions.
   void Initialize(const InitializeCallback& callback) override;
+
+  void AddByteStream(std::unique_ptr<ByteStreamReader> stream_reader,
+                     int64_t offset) override;
+
   void RenameAndUniquify(const base::FilePath& full_path,
                          const RenameCompletionCallback& callback) override;
   void RenameAndAnnotate(const base::FilePath& full_path,
@@ -63,9 +120,17 @@ class CONTENT_EXPORT DownloadFileImpl : public DownloadFile {
 
  protected:
   // For test class overrides.
+  // Append data to the file.
+  // On OS level, it will write at current position to the file.
   virtual DownloadInterruptReason AppendDataToFile(
       const char* data, size_t data_len);
 
+  // Write data from the offset to the file.
+  // On OS level, it will seek to the |offset| and write from there.
+  DownloadInterruptReason WriteDataToFile(int64_t offset,
+                                          const char* data,
+                                          size_t data_len);
+
   virtual base::TimeDelta GetRetryDelayForFailedRename(int attempt_number);
 
   virtual bool ShouldRetryFailedRename(DownloadInterruptReason reason);
@@ -108,9 +173,20 @@ class CONTENT_EXPORT DownloadFileImpl : public DownloadFile {
   // Send an update on our progress.
   void SendUpdate();
 
-  // Called when there's some activity on stream_reader_ that needs to be
+  // Called before the data is written to disk.
+  void WillWriteToDisk(size_t data_len);
+
+  // Called when there's some activity on the byte stream that needs to be
   // handled.
-  void StreamActive();
+  void StreamActive(SourceStream* source_stream);
+
+  // Register callback and start to read data from the stream.
+  void RegisterAndActivateStream(SourceStream* source_stream);
+
+  // Return the total valid bytes received in the target file.
+  // If the file is a sparse file, return the total number of valid bytes.
+  // Otherwise, return the current file size.
+  int64_t TotalBytesReceived() const;
 
   net::NetLogWithSource net_log_;
 
@@ -125,23 +201,29 @@ class CONTENT_EXPORT DownloadFileImpl : public DownloadFile {
   // The default directory for creating the download file.
   base::FilePath default_download_directory_;
 
-  // The stream through which data comes.
-  // TODO(rdsmith): Move this into BaseFile; requires using the same
-  // stream semantics in SavePackage.  Alternatively, replace SaveFile
-  // with DownloadFile and get rid of BaseFile.
-  std::unique_ptr<ByteStreamReader> stream_reader_;
+  // Map of the offset and the source stream that represents the slice
+  // starting from offset.
+  // Must be modified on the same thread that constructs the DownloadFile.
+  // Any byte stream should have a SourceStream before added to the download
+  // file.
+  // The disk IO is completed when all source streams are finished.
+  SourceStreams source_streams_;
 
   // Used to trigger progress updates.
   std::unique_ptr<base::RepeatingTimer> update_timer_;
 
+  // Set to true when multiple byte streams write to the same file.
+  // The file may contains null bytes(holes) in between of valid data slices.
+  bool is_sparse_file_;
+
   // Statistics
   size_t bytes_seen_;
   base::TimeDelta disk_writes_time_;
   base::TimeTicks download_start_;
   RateEstimator rate_estimator_;
 
+  base::ThreadChecker thread_checker_;
   base::WeakPtr<DownloadDestinationObserver> observer_;
-
   base::WeakPtrFactory<DownloadFileImpl> weak_factory_;
 
   DISALLOW_COPY_AND_ASSIGN(DownloadFileImpl);