Render inband text tracks in the media pipeline

media/filters/chunk_demuxer.cc

Index: media/filters/chunk_demuxer.cc
diff --git a/media/filters/chunk_demuxer.cc b/media/filters/chunk_demuxer.cc
index dbf1b031bec9d75e5d5cb2b82521d7c81b707366..dd09e7ad12d6eddb23ef6a98b1979602d9030081 100644
--- a/media/filters/chunk_demuxer.cc
+++ b/media/filters/chunk_demuxer.cc
@@ -12,12 +12,12 @@
 #include "base/callback_helpers.h"
 #include "base/location.h"
 #include "base/message_loop/message_loop_proxy.h"
+#include "base/stl_util.h"
 #include "media/base/audio_decoder_config.h"
 #include "media/base/bind_to_loop.h"
 #include "media/base/stream_parser_buffer.h"
 #include "media/base/video_decoder_config.h"
 #include "media/filters/stream_parser_factory.h"
-#include "media/webm/webm_webvtt_parser.h"
 
 using base::TimeDelta;
 
@@ -39,12 +39,13 @@ class SourceState {
               const CreateDemuxerStreamCB& create_demuxer_stream_cb,
               const IncreaseDurationCB& increase_duration_cb);
 
+  ~SourceState();
+
   void Init(const StreamParser::InitCB& init_cb,
             bool allow_audio,
             bool allow_video,
-            const StreamParser::NewTextBuffersCB& text_cb,
             const StreamParser::NeedKeyCB& need_key_cb,
-            const AddTextTrackCB& add_text_track_cb);
+            bool enable_text_tracks);
 
   // Appends new data to the StreamParser.
   // Returns true if the data was successfully appended. Returns false if an
@@ -66,6 +67,11 @@ class SourceState {
   }
   void set_append_window_end(TimeDelta end) { append_window_end_ = end; }
 
+  void TextStartReturningData();
+  void TextAbortReads();
+  void TextSeek(TimeDelta);

[acolwell GONE FROM CHROMIUM, 2013/10/21 20:10:40]

nit: Specify parameter name here.

[Matthew Heaney (Chromium), 2013/10/23 05:09:01]

Done.

+  void TextCompletePendingReadIfPossible();
+
  private:
   // Called by the |stream_parser_| when a new initialization segment is
   // encountered.
@@ -73,7 +79,8 @@ class SourceState {
   // processing decoder configurations.
   bool OnNewConfigs(bool allow_audio, bool allow_video,
                     const AudioDecoderConfig& audio_config,
-                    const VideoDecoderConfig& video_config);
+                    const VideoDecoderConfig& video_config,
+                    const TextTrackConfigMap& text_track_config_map);
 
   // Called by the |stream_parser_| at the beginning of a new media segment.
   void OnNewMediaSegment();
@@ -91,12 +98,12 @@ class SourceState {
                     const StreamParser::BufferQueue& video_buffers);
 
   // Called by the |stream_parser_| when new text buffers have been parsed. It
-  // applies |timestamp_offset_| to all buffers in |buffers| and then calls
-  // |new_buffers_cb| with the modified buffers.
+  // applies |timestamp_offset_| to all buffers in |buffers| and then appends
+  // the (modified) buffers to the demuxer stream associated with
+  // the track having |text_track_number|.
   // Returns true on a successful call. Returns false if an error occured while
   // processing the buffers.
-  bool OnTextBuffers(const StreamParser::NewTextBuffersCB& new_buffers_cb,
-                     TextTrack* text_track,
+  bool OnTextBuffers(int text_track_number,
                      const StreamParser::BufferQueue& buffers);
 
   // Helper function that adds |timestamp_offset_| to each buffer in |buffers|.
@@ -142,6 +149,9 @@ class SourceState {
   ChunkDemuxerStream* video_;
   bool video_needs_keyframe_;
 
+  typedef std::map<int, ChunkDemuxerStream*> TextStreamMap;
+  TextStreamMap text_stream_map_;
+
   LogCB log_cb_;
 
   DISALLOW_COPY_AND_ASSIGN(SourceState);
@@ -193,6 +203,7 @@ class ChunkDemuxerStream : public DemuxerStream {
   // Returns false if the new config should trigger an error.
   bool UpdateAudioConfig(const AudioDecoderConfig& config, const LogCB& log_cb);
   bool UpdateVideoConfig(const VideoDecoderConfig& config, const LogCB& log_cb);
+  void UpdateTextConfig(const TextTrackConfig& config, const LogCB& log_cb);
 
   void MarkEndOfStream();
   void UnmarkEndOfStream();
@@ -203,6 +214,7 @@ class ChunkDemuxerStream : public DemuxerStream {
   virtual void EnableBitstreamConverter() OVERRIDE;
   virtual AudioDecoderConfig audio_decoder_config() OVERRIDE;
   virtual VideoDecoderConfig video_decoder_config() OVERRIDE;
+  virtual TextTrackConfig text_track_config() OVERRIDE;
 
   void set_memory_limit_for_testing(int memory_limit) {
     stream_->set_memory_limit_for_testing(memory_limit);
@@ -227,7 +239,7 @@ class ChunkDemuxerStream : public DemuxerStream {
   bool GetNextBuffer_Locked(DemuxerStream::Status* status,
                             scoped_refptr<StreamParserBuffer>* buffer);
 
-  // Specifies the type of the stream (must be AUDIO or VIDEO for now).
+  // Specifies the type of the stream.
   Type type_;
 
   scoped_ptr<SourceBufferStream> stream_;
@@ -258,12 +270,15 @@ SourceState::SourceState(scoped_ptr<StreamParser> stream_parser,
   DCHECK(!increase_duration_cb_.is_null());
 }
 
+SourceState::~SourceState() {
+  STLDeleteValues(&text_stream_map_);

[acolwell GONE FROM CHROMIUM, 2013/10/21 20:10:40]

I believe you may need to call Shutdown() on each of the streams to cancel any
pending reads they might have.

[Matthew Heaney (Chromium), 2013/10/23 05:09:01]

Done.

+}
+
 void SourceState::Init(const StreamParser::InitCB& init_cb,
                        bool allow_audio,
                        bool allow_video,
-                       const StreamParser::NewTextBuffersCB& text_cb,
                        const StreamParser::NeedKeyCB& need_key_cb,
-                       const AddTextTrackCB& add_text_track_cb) {
+                       bool enable_text_tracks) {
   StreamParser::NewBuffersCB audio_cb;
 
   stream_parser_->Init(init_cb,
@@ -274,9 +289,9 @@ void SourceState::Init(const StreamParser::InitCB& init_cb,
                        base::Bind(&SourceState::OnNewBuffers,
                                   base::Unretained(this)),
                        base::Bind(&SourceState::OnTextBuffers,
-                                  base::Unretained(this), text_cb),
+                                  base::Unretained(this)),
                        need_key_cb,
-                       add_text_track_cb,
+                       enable_text_tracks,
                        base::Bind(&SourceState::OnNewMediaSegment,
                                   base::Unretained(this)),
                        base::Bind(&SourceState::OnEndOfMediaSegment,
@@ -303,6 +318,35 @@ void SourceState::Abort() {
   can_update_offset_ = true;
 }
 
+
+void SourceState::TextStartReturningData() {
+  for (TextStreamMap::iterator itr = text_stream_map_.begin();
+       itr != text_stream_map_.end(); ++itr) {
+    itr->second->StartReturningData();
+  }
+}
+
+void SourceState::TextAbortReads() {
+  for (TextStreamMap::iterator itr = text_stream_map_.begin();
+       itr != text_stream_map_.end(); ++itr) {
+    itr->second->AbortReads();
+  }
+}
+
+void SourceState::TextSeek(TimeDelta time) {
+  for (TextStreamMap::iterator itr = text_stream_map_.begin();
+       itr != text_stream_map_.end(); ++itr) {
+    itr->second->Seek(time);
+  }
+}
+
+void SourceState::TextCompletePendingReadIfPossible() {
+  for (TextStreamMap::iterator itr = text_stream_map_.begin();
+       itr != text_stream_map_.end(); ++itr) {
+    itr->second->CompletePendingReadIfPossible();
+  }
+}
+
 void SourceState::AdjustBufferTimestamps(
     const StreamParser::BufferQueue& buffers) {
   if (timestamp_offset_ == TimeDelta())
@@ -318,7 +362,8 @@ void SourceState::AdjustBufferTimestamps(
 
 bool SourceState::OnNewConfigs(bool allow_audio, bool allow_video,
                                const AudioDecoderConfig& audio_config,
-                               const VideoDecoderConfig& video_config) {
+                               const VideoDecoderConfig& video_config,
+                               const TextTrackConfigMap& text_config) {

[acolwell GONE FROM CHROMIUM, 2013/10/21 20:10:40]

nit: s/test_config/text_configs/

[Matthew Heaney (Chromium), 2013/10/23 05:09:01]

Done.

   DVLOG(1) << "OnNewConfigs(" << allow_audio << ", " << allow_video
            << ", " << audio_config.IsValidConfig()
            << ", " << video_config.IsValidConfig() << ")";
@@ -377,6 +422,52 @@ bool SourceState::OnNewConfigs(bool allow_audio, bool allow_video,
     success &= video_->UpdateVideoConfig(video_config, log_cb_);
   }
 
+  if (text_stream_map_.empty()) {
+    for (TextTrackConfigMap::const_iterator itr = text_config.begin();
+         itr != text_config.end(); ++itr) {
+      ChunkDemuxerStream* const text_stream =
+          create_demuxer_stream_cb_.Run(DemuxerStream::TEXT);
+      text_stream->UpdateTextConfig(itr->second, log_cb_);
+      text_stream_map_[itr->first] = text_stream;
+    }
+    // TODO(matthewjheaney): must callback chunk demuxer, so it can notify
+    // its host to add text stream(s).
+  } else {
+    const TextStreamMap::size_type text_count = text_stream_map_.size();

[acolwell GONE FROM CHROMIUM, 2013/10/21 20:10:40]

nit: Please just use size_t here.

[Matthew Heaney (Chromium), 2013/10/23 05:09:01]

Done.

+    if (text_config.size() != text_count) {
+      success &= false;
+    } else if (text_count == 1) {
+      TextTrackConfigMap::const_iterator config_itr = text_config.begin();
+      const TextTrackConfig& new_config = config_itr->second;
+      TextStreamMap::iterator stream_itr = text_stream_map_.begin();
+      ChunkDemuxerStream* stream = stream_itr->second;
+      TextTrackConfig old_config = stream->text_track_config();
+      if (!new_config.Matches(old_config)) {
+        success &= false;
+      } else {
+        // TODO(matthewjheaney): replace old with new

[acolwell GONE FROM CHROMIUM, 2013/10/21 20:10:40]

I think this is just

text_stream_map_.clear().
text_stream_map_[config_itr->first] = text_stream;

Right?

[Matthew Heaney (Chromium), 2013/10/23 05:09:01]

Done.

+      }
+    } else {
+      for (TextTrackConfigMap::const_iterator config_itr = text_config.begin();
+           config_itr != text_config.end(); ++config_itr) {
+        TextStreamMap::iterator stream_itr =
+            text_stream_map_.find(config_itr->first);
+        if (stream_itr == text_stream_map_.end()) {
+          success &= false;

[acolwell GONE FROM CHROMIUM, 2013/10/21 20:10:40]

nit: Break out of the loop here and then drop the else below & deindent.

[Matthew Heaney (Chromium), 2013/10/23 05:09:01]

Done.

+        } else {
+          const TextTrackConfig& new_config = config_itr->second;
+          ChunkDemuxerStream* stream = stream_itr->second;
+          TextTrackConfig old_config = stream->text_track_config();
+          if (!new_config.Matches(old_config)) {
+            success &= false;
+          }
+        }
+      }
+
+      // TODO(matthewjheaney): if all configs match, then replace old with new

[acolwell GONE FROM CHROMIUM, 2013/10/21 20:10:40]

I don't think you need to do anything here. If they all match then everything
should be able to proceed as normal right?

[Matthew Heaney (Chromium), 2013/10/23 05:09:01]

Done.

+    }
+  }
+
   DVLOG(1) << "OnNewConfigs() : " << (success ? "success" : "failed");
   return success;
 }
@@ -432,6 +523,11 @@ bool SourceState::OnNewBuffers(const StreamParser::BufferQueue& audio_buffers,
 
     if (video_)
       video_->OnNewMediaSegment(segment_timestamp);
+
+    for (TextStreamMap::iterator itr = text_stream_map_.begin();
+         itr != text_stream_map_.end(); ++itr) {
+      itr->second->OnNewMediaSegment(segment_timestamp);
+    }
   }
 
   if (!filtered_audio.empty()) {
@@ -450,15 +546,17 @@ bool SourceState::OnNewBuffers(const StreamParser::BufferQueue& audio_buffers,
 }
 
 bool SourceState::OnTextBuffers(
-    const StreamParser::NewTextBuffersCB& new_buffers_cb,
-    TextTrack* text_track,
+    int text_track_number,
     const StreamParser::BufferQueue& buffers) {
-  if (new_buffers_cb.is_null())
+  DCHECK(!buffers.empty());
+
+  TextStreamMap::iterator itr = text_stream_map_.find(text_track_number);
+  if (itr == text_stream_map_.end())
     return false;
 
   AdjustBufferTimestamps(buffers);
 
-  return new_buffers_cb.Run(text_track, buffers);
+  return itr->second->Append(buffers);
 }
 
 void SourceState::FilterWithAppendWindow(
@@ -649,6 +747,17 @@ bool ChunkDemuxerStream::UpdateVideoConfig(const VideoDecoderConfig& config,
   return stream_->UpdateVideoConfig(config);
 }
 
+void ChunkDemuxerStream::UpdateTextConfig(const TextTrackConfig& config,
+                                          const LogCB& log_cb) {
+  DCHECK_EQ(type_, TEXT);
+  base::AutoLock auto_lock(lock_);
+
+  if (!stream_) {

[acolwell GONE FROM CHROMIUM, 2013/10/21 20:10:40]

Since we don't update TextTrackConfigs, you should be able to just
DCHECK(!stream_) here instead.

[Matthew Heaney (Chromium), 2013/10/23 05:09:01]

Done.

+    DCHECK_EQ(state_, UNINITIALIZED);
+    stream_.reset(new SourceBufferStream(config, log_cb));
+  }
+}
+
 void ChunkDemuxerStream::MarkEndOfStream() {
   base::AutoLock auto_lock(lock_);
   stream_->MarkEndOfStream();
@@ -685,6 +794,12 @@ VideoDecoderConfig ChunkDemuxerStream::video_decoder_config() {
   return stream_->GetCurrentVideoDecoderConfig();
 }
 
+TextTrackConfig ChunkDemuxerStream::text_track_config() {
+  CHECK_EQ(type_, TEXT);
+  base::AutoLock auto_lock(lock_);
+  return stream_->GetCurrentTextTrackConfig();
+}
+
 void ChunkDemuxerStream::ChangeState_Locked(State state) {
   lock_.AssertAcquired();
   DVLOG(1) << "ChunkDemuxerStream::ChangeState_Locked() : "
@@ -744,14 +859,14 @@ void ChunkDemuxerStream::CompletePendingReadIfPossible_Locked() {
 
 ChunkDemuxer::ChunkDemuxer(const base::Closure& open_cb,
                            const NeedKeyCB& need_key_cb,
-                           const AddTextTrackCB& add_text_track_cb,
+                           bool enable_text,
                            const LogCB& log_cb)
     : state_(WAITING_FOR_INIT),
       cancel_next_seek_(false),
       host_(NULL),
       open_cb_(open_cb),
       need_key_cb_(need_key_cb),
-      add_text_track_cb_(add_text_track_cb),
+      enable_text_(enable_text),
       log_cb_(log_cb),
       duration_(kNoTimestamp()),
       user_specified_duration_(-1) {
@@ -821,6 +936,7 @@ void ChunkDemuxer::OnAudioRendererDisabled() {
 
 // Demuxer implementation.
 DemuxerStream* ChunkDemuxer::GetStream(DemuxerStream::Type type) {
+  DCHECK_NE(type, DemuxerStream::TEXT);
   base::AutoLock auto_lock(lock_);
   if (type == DemuxerStream::VIDEO)
     return video_.get();
@@ -910,9 +1026,8 @@ ChunkDemuxer::Status ChunkDemuxer::AddId(const std::string& id,
       base::Bind(&ChunkDemuxer::OnSourceInitDone, base::Unretained(this)),
       has_audio,
       has_video,
-      base::Bind(&ChunkDemuxer::OnTextBuffers, base::Unretained(this)),
       need_key_cb_,
-      add_text_track_cb_);
+      enable_text_);
 
   source_state_map_[id] = source_state.release();
   return kOk;
@@ -936,6 +1051,8 @@ void ChunkDemuxer::RemoveId(const std::string& id) {
       video_->Shutdown();
     source_id_video_.clear();
   }
+
+  // TODO(matthewjheaney): delete the text streams associated with this id

[acolwell GONE FROM CHROMIUM, 2013/10/21 20:10:40]

nit: No longer needed since the streams get deleted with the SourceState.

[Matthew Heaney (Chromium), 2013/10/23 05:09:01]

Done.

 }
 
 Ranges<TimeDelta> ChunkDemuxer::GetBufferedRanges(const std::string& id) const {
@@ -1342,6 +1459,11 @@ ChunkDemuxer::CreateDemuxerStream(DemuxerStream::Type type) {
       video_.reset(new ChunkDemuxerStream(DemuxerStream::VIDEO));
       return video_.get();
       break;
+    case DemuxerStream::TEXT: {
+      // TODO(matthewjheaney): we need to destroy this text stream
+      return new ChunkDemuxerStream(DemuxerStream::TEXT);
+      break;
+    }
     case DemuxerStream::UNKNOWN:
     case DemuxerStream::NUM_TYPES:
       NOTREACHED();
@@ -1351,31 +1473,18 @@ ChunkDemuxer::CreateDemuxerStream(DemuxerStream::Type type) {
   return NULL;
 }
 
-bool ChunkDemuxer::OnTextBuffers(
-  TextTrack* text_track,
-  const StreamParser::BufferQueue& buffers) {
+#if 0

[acolwell GONE FROM CHROMIUM, 2013/10/21 20:10:40]

remove this block if it is no longer needed.

[Matthew Heaney (Chromium), 2013/10/23 05:09:01]

I restored the callback, so that the demuxer can notify the host.

+// TODO(matthewjheaney): must be called from source state
+void ChunkDemuxer::OnNewTextTrack(ChunkDemuxerStream* text_stream,
+                                  TextKind kind,
+                                  const std::string& name,
+                                  const std::string& language) {
   lock_.AssertAcquired();
   DCHECK_NE(state_, SHUTDOWN);
-
-  // TODO(matthewjheaney): IncreaseDurationIfNecessary
-
-  for (StreamParser::BufferQueue::const_iterator itr = buffers.begin();
-       itr != buffers.end(); ++itr) {
-    const StreamParserBuffer* const buffer = itr->get();
-    const TimeDelta start = buffer->timestamp();
-    const TimeDelta end = start + buffer->duration();
-
-    std::string id, settings, content;
-
-    WebMWebVTTParser::Parse(buffer->data(),
-                            buffer->data_size(),
-                            &id, &settings, &content);
-
-    text_track->addWebVTTCue(start, end, id, content, settings);
-  }
-
-  return true;
+  DCHECK(text_stream_set_.find(text_stream) != text_stream_set_.end());
+  host_->AddTextStream(text_stream, kind, name, language);
 }
+#endif
 
 bool ChunkDemuxer::IsValidId(const std::string& source_id) const {
   lock_.AssertAcquired();
@@ -1428,6 +1537,11 @@ void ChunkDemuxer::StartReturningData() {
 
   if (video_)
     video_->StartReturningData();
+
+  for (SourceStateMap::iterator itr = source_state_map_.begin();
+       itr != source_state_map_.end(); ++itr) {
+    itr->second->TextStartReturningData();
+  }
 }
 
 void ChunkDemuxer::AbortPendingReads() {
@@ -1436,6 +1550,11 @@ void ChunkDemuxer::AbortPendingReads() {
 
   if (video_)
     video_->AbortReads();
+
+  for (SourceStateMap::iterator itr = source_state_map_.begin();
+       itr != source_state_map_.end(); ++itr) {
+    itr->second->TextAbortReads();
+  }
 }
 
 void ChunkDemuxer::SeekAllSources(TimeDelta seek_time) {
@@ -1444,6 +1563,11 @@ void ChunkDemuxer::SeekAllSources(TimeDelta seek_time) {
 
   if (video_)
     video_->Seek(seek_time);
+
+  for (SourceStateMap::iterator itr = source_state_map_.begin();
+       itr != source_state_map_.end(); ++itr) {
+    itr->second->TextSeek(seek_time);
+  }
 }
 
 void ChunkDemuxer::CompletePendingReadsIfPossible() {
@@ -1452,6 +1576,11 @@ void ChunkDemuxer::CompletePendingReadsIfPossible() {
 
   if (video_)
     video_->CompletePendingReadIfPossible();
+
+  for (SourceStateMap::iterator itr = source_state_map_.begin();
+       itr != source_state_map_.end(); ++itr) {
+    itr->second->TextCompletePendingReadIfPossible();
+  }
 }
 
 }  // namespace media