Multi reader/writer cache/buffer

media/blink/multibuffer_unittest.cc

Index: media/blink/multibuffer_unittest.cc
diff --git a/media/blink/multibuffer_unittest.cc b/media/blink/multibuffer_unittest.cc
new file mode 100644
index 0000000000000000000000000000000000000000..edd4abea63a683a87ceb6bc586fa828877d4af80
--- /dev/null
+++ b/media/blink/multibuffer_unittest.cc
@@ -0,0 +1,500 @@
+// Copyright 2015 The Chromium Authors. All rights reserved.

[DaleCurtis, 2015/10/19 21:45:25]

Overall there's too much use of rand() in this file. It's okay to have a single
test or two which explicitly exercise random behavior, but most tests should be
straight forward and not have implicit random results.

+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include <deque>
+#include <string>
+#include <vector>
+
+#include "base/bind.h"
+#include "base/callback_helpers.h"
+#include "media/blink/multibuffer.h"
+#include "media/blink/multibuffer_reader.h"
+#include "testing/gtest/include/gtest/gtest.h"
+
+const int kBlockSizeShift = 8;
+const size_t kBlockSize = 1UL << kBlockSizeShift;
+
+namespace media {
+class TestMultiBufferDataProvider;
+
+std::vector<TestMultiBufferDataProvider*> writers;
+
+class TestMultiBufferDataProvider : public media::MultiBuffer::DataProvider {
+ public:
+  TestMultiBufferDataProvider(MultiBufferBlockId pos,
+                              size_t file_size,
+                              int max_blocks_after_defer,
+                              bool must_read_whole_file) :
+      pos_(pos),
+      blocks_until_deferred_(1 << 30),
+      max_blocks_after_defer_(max_blocks_after_defer),
+      file_size_(file_size),
+      must_read_whole_file_(must_read_whole_file) {
+    writers.push_back(this);
+  }
+
+  ~TestMultiBufferDataProvider() override {
+    if (must_read_whole_file_) {
+      CHECK_GE(pos_.block_num() * kBlockSize, file_size_);
+    }
+    for (size_t i = 0; i < writers.size(); i++) {
+      if (writers[i] == this) {
+        writers[i] = writers.back();
+        writers.pop_back();
+        return;
+      }
+    }
+    LOG(FATAL) << "Couldn't find myself in writers!";
+  }
+
+  MultiBufferBlockId Tell() const override {
+    return pos_;
+  }
+
+  bool Available() const override {
+    return !fifo_.empty();
+  }
+
+  scoped_refptr<DataBuffer> Read() override {
+    DCHECK(Available());
+    scoped_refptr<DataBuffer> ret = fifo_.front();
+    fifo_.pop_front();
+    ++pos_;
+    return ret;
+  }
+
+  void SetAvailableCallback(const base::Closure& cb) override {
+    DCHECK(!Available());
+    cb_ = cb;
+  }
+
+  void SetDeferred(bool deferred) override {
+    if (deferred) {
+      if (max_blocks_after_defer_ > 0) {
+        blocks_until_deferred_ = rand() % max_blocks_after_defer_;

[DaleCurtis, 2015/10/19 21:45:25]

Again, don't use rand unless you're prepared to debug this on all platforms or
print out enough information to reconstruct the test on all platforms.

+      } else if (max_blocks_after_defer_ < 0) {
+        blocks_until_deferred_ = - max_blocks_after_defer_;
+      } else {
+        blocks_until_deferred_ = 0;
+      }
+    } else {
+      blocks_until_deferred_  = 1 << 30;
+    }
+  }
+
+  bool Advance() {
+    if (blocks_until_deferred_ == 0) return false;
+    --blocks_until_deferred_;
+
+    bool ret = true;
+    scoped_refptr<media::DataBuffer> block = new media::DataBuffer(kBlockSize);
+    size_t x = 0;
+    size_t byte_pos = (fifo_.size() + pos_.block_num()) * kBlockSize;
+    for (x = 0; x < kBlockSize; x++, byte_pos++) {
+      if (byte_pos >= file_size_) break;
+      block->writable_data()[x] =
+          static_cast<uint8_t>((byte_pos * 15485863) >> 16);
+    }
+    block->set_data_size(static_cast<int>(x));
+    fifo_.push_back(block);
+    if (byte_pos == file_size_) {
+      fifo_.push_back(DataBuffer::CreateEOSBuffer());
+      ret = false;
+    }
+    cb_.Run();
+    return ret;
+  }
+
+ private:
+  std::deque<scoped_refptr<media::DataBuffer> > fifo_;
+  MultiBufferBlockId pos_;
+  int32_t blocks_until_deferred_;
+  int32_t max_blocks_after_defer_;
+  size_t file_size_;
+  bool must_read_whole_file_;
+  base::Closure cb_;
+};
+
+class TestMultiBuffer : public media::MultiBuffer {
+ public:
+  explicit TestMultiBuffer(int32_t shift) :
+      media::MultiBuffer(shift),
+      create_ok_(true),
+      max_writers_(10000),
+      file_size_(1 << 30),
+      max_blocks_after_defer_(0),
+      must_read_whole_file_(false),
+      writers_created_(0) {
+  }
+
+  void SetMaxWriters(size_t max_writers) {
+    max_writers_ = max_writers;
+  }
+
+  void CheckPresentState() {
+    MultiBufferBlockId last;
+    for (DataMap::iterator i = data_.begin();
+         i != data_.end();
+         ++i) {
+      CHECK(i->second);   // Null poineters are not allowed in data_
+      CHECK_NE(!!pinned_[i->first], lru_.Contains(i->first))
+          << " i->first = " << i->first;
+
+      if (!last.SameUrl(i->first)) {
+        last = MultiBufferBlockId(i->first.url_data(), 0);
+      }
+      while (last < i->first) {
+        CHECK_EQ(0, present_[last]) << " last = " << last;
+        ++last;
+      }
+      last = i->first;
+      CHECK_EQ(1, present_[last]) << " last = " << last;
+      ++last;
+    }
+  }
+
+  void CheckLRUState()  {
+    for (DataMap::iterator i = data_.begin();
+         i != data_.end();
+         ++i) {
+      CHECK(i->second);   // Null poineters are not allowed in data_
+      CHECK_NE(!!pinned_[i->first], lru_.Contains(i->first))
+          << " i->first = " << i->first;
+      CHECK_EQ(1, present_[i->first]) << " i->first = " << i->first;
+    }
+  }
+
+  void SetFileSize(size_t file_size) {
+    file_size_ = file_size;
+  }
+
+  void SetMaxBlocksAfterDefer(int32_t max_blocks_after_defer) {
+    max_blocks_after_defer_ = max_blocks_after_defer;
+  }
+
+  void SetMustReadWholeFile(bool must_read_whole_file) {
+    must_read_whole_file_ = must_read_whole_file;
+  }
+
+  int32_t writers_created() const {
+    return writers_created_;
+  }
+ protected:
+  DataProvider* CreateWriter(const MultiBufferBlockId& pos) override {
+    DCHECK(create_ok_);
+    CHECK_LE(writers.size(), max_writers_);
+    writers_created_++;
+    return new TestMultiBufferDataProvider(
+        pos, file_size_, max_blocks_after_defer_, must_read_whole_file_);
+  }
+  void Prune(size_t max_to_free) override {
+    // Prune should not cause additional writers to be spawned.
+    create_ok_ = false;
+    MultiBuffer::Prune(max_to_free);
+    create_ok_ = true;
+  }
+ private:
+  bool create_ok_;
+  size_t max_writers_;
+  size_t file_size_;
+  int32_t max_blocks_after_defer_;
+  bool must_read_whole_file_;
+  int32_t writers_created_;
+};
+
+}
+
+class MultiBufferTest : public testing::Test {
+ public:
+  MultiBufferTest() :
+      multibuffer_(kBlockSizeShift) {
+  }
+
+  void Advance() {
+    CHECK(media::writers.size());
+    media::writers[rand() % media::writers.size()]->Advance();

[DaleCurtis, 2015/10/19 21:45:25]

Ditto.

+  }
+
+  bool AdvanceAll() {
+    bool advanced = false;
+    for (size_t i = 0; i < media::writers.size(); i++) {
+      advanced |= media::writers[i]->Advance();
+    }
+    multibuffer_.CheckLRUState();
+    return advanced;
+  }
+
+  scoped_refptr<media::UrlData> GetUrl() {
+    return url_index_.GetByUrl(GURL("http://x"), media::UrlData::kUnspecified);
+  }
+
+ protected:
+  media::TestMultiBuffer multibuffer_;
+  media::UrlIndex url_index_;
+};
+
+TEST_F(MultiBufferTest, ReadAll) {
+  multibuffer_.SetMaxWriters(1);
+  size_t pos = 0;
+  size_t end = 10000;
+  multibuffer_.SetFileSize(10000);
+  multibuffer_.SetMustReadWholeFile(true);
+  media::MultiBufferReader reader(&multibuffer_,
+                                  GetUrl(),
+                                  pos,
+                                  end,
+                                  base::Callback<void(int64_t, int64_t)>());
+  reader.SetMaxBuffer(2000, 5000);
+  reader.SetPreload(1000, 1000);
+  while (pos < end) {
+    unsigned char buffer[27];
+    buffer[17] = 17;
+    size_t to_read = std::min<size_t>(end - pos, 17);
+    int64_t bytes_read = reader.TryRead(buffer, to_read);
+    if (bytes_read) {
+      EXPECT_EQ(buffer[17], 17);
+      for (int64_t i = 0; i < bytes_read; i++) {
+        uint8_t expected = static_cast<uint8_t>((pos * 15485863) >> 16);
+        EXPECT_EQ(expected, buffer[i]) << " pos = " << pos;
+        pos++;
+      }
+    } else {
+      Advance();
+    }
+  }
+}
+
+TEST_F(MultiBufferTest, UpdateUrlData) {
+  multibuffer_.SetMaxWriters(1);
+  size_t pos = 0;
+  size_t end = 10000;
+  multibuffer_.SetFileSize(10000);
+  scoped_refptr<media::UrlData> a = GetUrl();
+  scoped_refptr<media::UrlData> b =
+      url_index_.GetByUrl(GURL("http://y"), media::UrlData::kUnspecified);
+  media::MultiBufferReader reader(&multibuffer_,
+                                  a,
+                                  pos,
+                                  end,
+                                  base::Callback<void(int64_t, int64_t)>());
+  reader.SetPreload(1000, 1000);
+  multibuffer_.UpdateUrlData(a, b);
+  EXPECT_EQ(b, reader.GetUrlData());
+}
+
+
+TEST_F(MultiBufferTest, ReadAllAdvanceFirst) {
+  multibuffer_.SetMaxWriters(1);
+  size_t pos = 0;
+  size_t end = 10000;
+  multibuffer_.SetFileSize(10000);
+  multibuffer_.SetMustReadWholeFile(true);
+  media::MultiBufferReader reader(&multibuffer_,
+                                  GetUrl(),
+                                  pos,
+                                  end,
+                                  base::Callback<void(int64_t, int64_t)>());
+  reader.SetMaxBuffer(2000, 5000);
+  reader.SetPreload(1000, 1000);
+  while (pos < end) {
+    unsigned char buffer[27];
+    buffer[17] = 17;
+    size_t to_read = std::min<size_t>(end - pos, 17);
+    while (AdvanceAll());
+    int64_t bytes = reader.TryRead(buffer, to_read);
+    EXPECT_GT(bytes, 0);
+    EXPECT_EQ(buffer[17], 17);
+    for (int64_t i = 0; i < bytes; i++) {
+      uint8_t expected = static_cast<uint8_t>((pos * 15485863) >> 16);
+      EXPECT_EQ(expected, buffer[i]) << " pos = " << pos;
+      pos++;
+    }
+  }
+}
+
+// Checks that if the data provider provides too much data after we told it
+// to defer, we kill it.
+TEST_F(MultiBufferTest, ReadAllAdvanceFirst_NeverDefer) {
+  multibuffer_.SetMaxWriters(1);
+  size_t pos = 0;
+  size_t end = 10000;
+  multibuffer_.SetFileSize(10000);
+  multibuffer_.SetMaxBlocksAfterDefer(-10000);
+  scoped_refptr<media::UrlData> url_data = GetUrl();
+  url_data->set_range_supported();
+  media::MultiBufferReader reader(&multibuffer_,
+                                  url_data,
+                                  pos,
+                                  end,
+                                  base::Callback<void(int64_t, int64_t)>());
+  reader.SetMaxBuffer(2000, 5000);
+  reader.SetPreload(1000, 1000);
+  while (pos < end) {
+    unsigned char buffer[27];
+    buffer[17] = 17;
+    size_t to_read = std::min<size_t>(end - pos, 17);
+    while (AdvanceAll());
+    int64_t bytes = reader.TryRead(buffer, to_read);
+    EXPECT_GT(bytes, 0);
+    EXPECT_EQ(buffer[17], 17);
+    for (int64_t i = 0; i < bytes; i++) {
+      uint8_t expected = static_cast<uint8_t>((pos * 15485863) >> 16);
+      EXPECT_EQ(expected, buffer[i]) << " pos = " << pos;
+      pos++;
+    }
+  }
+  EXPECT_GT(multibuffer_.writers_created(), 1);
+}
+
+// Same as ReadAllAdvanceFirst_NeverDefer, but the url doesn't support
+// ranges, so we don't destroy it no matter how much data it provides.
+TEST_F(MultiBufferTest, ReadAllAdvanceFirst_NeverDefer2) {
+  multibuffer_.SetMaxWriters(1);
+  size_t pos = 0;
+  size_t end = 10000;
+  multibuffer_.SetFileSize(10000);
+  multibuffer_.SetMustReadWholeFile(true);
+  multibuffer_.SetMaxBlocksAfterDefer(-10000);
+  scoped_refptr<media::UrlData> url_data = GetUrl();
+  media::MultiBufferReader reader(&multibuffer_,
+                                  url_data,
+                                  pos,
+                                  end,
+                                  base::Callback<void(int64_t, int64_t)>());
+  reader.SetMaxBuffer(2000, 5000);
+  reader.SetPreload(1000, 1000);
+  while (pos < end) {
+    unsigned char buffer[27];
+    buffer[17] = 17;
+    size_t to_read = std::min<size_t>(end - pos, 17);
+    while (AdvanceAll());
+    int64_t bytes = reader.TryRead(buffer, to_read);
+    EXPECT_GT(bytes, 0);
+    EXPECT_EQ(buffer[17], 17);
+    for (int64_t i = 0; i < bytes; i++) {
+      uint8_t expected = static_cast<uint8_t>((pos * 15485863) >> 16);
+      EXPECT_EQ(expected, buffer[i]) << " pos = " << pos;
+      pos++;
+    }
+  }
+}
+
+class ReadHelper {
+ public:
+  ReadHelper(scoped_refptr<media::UrlData> url,
+             size_t end,
+             size_t max_read_size,
+             media::MultiBuffer* multibuffer) :
+      pos_(0),
+      end_(end),
+      max_read_size_(max_read_size),
+      read_size_(0),
+      reader_(multibuffer,
+              url,
+              pos_,
+              end_,
+              base::Callback<void(int64_t, int64_t)>()) {
+    reader_.SetMaxBuffer(2000, 5000);
+    reader_.SetPreload(1000, 1000);
+  }
+
+  bool Read() {
+    if (read_size_ == 0) return true;
+    unsigned char buffer[4096];
+    CHECK_LE(read_size_, static_cast<int64_t>(sizeof(buffer)));
+    CHECK_EQ(pos_, reader_.Tell());
+    int64_t bytes_read = reader_.TryRead(buffer, read_size_);
+    if (bytes_read) {
+      for (int64_t i = 0; i < bytes_read; i++) {
+        unsigned char expected = (pos_ * 15485863) >> 16;
+        EXPECT_EQ(expected, buffer[i]) << " pos = " << pos_;
+        pos_++;
+      }
+      CHECK_EQ(pos_, reader_.Tell());
+      return true;
+    }
+    return false;
+  }
+
+  void StartRead() {
+    CHECK_EQ(pos_, reader_.Tell());
+    read_size_ = std::min(1 + rand() % (max_read_size_ - 1), end_ - pos_);
+    if (!Read()) {
+      reader_.Wait(read_size_,
+                    base::Bind(&ReadHelper::WaitCB, base::Unretained(this)));
+    }
+  }
+
+  void WaitCB() {
+    CHECK(Read());
+  }
+
+  void Seek() {
+    pos_ = rand() % end_;
+    reader_.Seek(pos_);
+    CHECK_EQ(pos_, reader_.Tell());
+  }
+
+ private:
+  int64_t pos_;
+  int64_t end_;
+  int64_t max_read_size_;
+  int64_t read_size_;
+  media::MultiBufferReader reader_;
+};
+
+TEST_F(MultiBufferTest, RandomTest) {
+  size_t file_size = 1000000;
+  multibuffer_.SetFileSize(file_size);
+  multibuffer_.SetMaxBlocksAfterDefer(10);
+  std::vector<ReadHelper*> read_helpers;
+  scoped_refptr<media::UrlData> url = GetUrl();
+  for (size_t i = 0; i < 20; i++) {
+    read_helpers.push_back(new ReadHelper(url, file_size, 1000, &multibuffer_));
+  }
+  for (int i = 0; i < 10000; i++) {
+    if (rand() & 1) {
+      if (!media::writers.empty()) Advance();
+    } else {
+      size_t j = rand() % read_helpers.size();
+      if (rand() % 100 < 3)
+        read_helpers[j]->Seek();
+      read_helpers[j]->StartRead();
+      multibuffer_.CheckLRUState();
+    }
+  }
+  multibuffer_.CheckPresentState();
+  while (!read_helpers.empty()) {
+    delete read_helpers.back();
+    read_helpers.pop_back();
+  }
+}
+
+TEST_F(MultiBufferTest, RandomTest_RangeSupported) {
+  size_t file_size = 1000000;
+  multibuffer_.SetFileSize(file_size);
+  multibuffer_.SetMaxBlocksAfterDefer(10);
+  std::vector<ReadHelper*> read_helpers;
+  scoped_refptr<media::UrlData> url = GetUrl();
+  url->set_range_supported();
+  for (size_t i = 0; i < 20; i++) {
+    read_helpers.push_back(new ReadHelper(url, file_size, 1000, &multibuffer_));
+  }
+  for (int i = 0; i < 10000; i++) {
+    if (rand() & 1) {
+      if (!media::writers.empty()) Advance();
+    } else {
+      size_t j = rand() % read_helpers.size();
+      if (rand() % 100 < 3)
+        read_helpers[j]->Seek();
+      read_helpers[j]->StartRead();
+      multibuffer_.CheckLRUState();
+    }
+  }
+  multibuffer_.CheckPresentState();
+  while (!read_helpers.empty()) {
+    delete read_helpers.back();
+    read_helpers.pop_back();
+  }
+}