Support multiple media tracks in MSE / ChunkDemuxer

media/filters/chunk_demuxer.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "media/filters/chunk_demuxer.h"
 
 #include <algorithm>
 #include <limits>
-#include <list>
 #include <utility>
 
 #include "base/bind.h"
 #include "base/callback_helpers.h"
 #include "base/location.h"
 #include "base/macros.h"
 #include "base/metrics/histogram_macros.h"
 #include "base/stl_util.h"
 #include "base/strings/string_number_conversions.h"
 #include "media/base/audio_decoder_config.h"
 #include "media/base/bind_to_current_loop.h"
 #include "media/base/media_tracks.h"
+#include "media/base/mime_util.h"
 #include "media/base/stream_parser_buffer.h"
 #include "media/base/timestamp_constants.h"
 #include "media/base/video_codecs.h"
 #include "media/base/video_decoder_config.h"
 #include "media/filters/frame_processor.h"
 #include "media/filters/stream_parser_factory.h"
 
 using base::TimeDelta;
 
 namespace media {
@@ skipping 370 matching lines @@
     const EncryptedMediaInitDataCB& encrypted_media_init_data_cb,
     const scoped_refptr<MediaLog>& media_log,
     bool splice_frames_enabled)
     : state_(WAITING_FOR_INIT),
       cancel_next_seek_(false),
       host_(NULL),
       open_cb_(open_cb),
       encrypted_media_init_data_cb_(encrypted_media_init_data_cb),
       enable_text_(false),
       media_log_(media_log),
-      pending_source_init_done_count_(0),
       duration_(kNoTimestamp),
       user_specified_duration_(-1),
       liveness_(DemuxerStream::LIVENESS_UNKNOWN),
       splice_frames_enabled_(splice_frames_enabled),
       detected_audio_track_count_(0),
       detected_video_track_count_(0),
       detected_text_track_count_(0) {
   DCHECK(!open_cb_.is_null());
   DCHECK(!encrypted_media_init_data_cb_.is_null());
 }
@@ skipping 61 matching lines @@
 }
 
 // Demuxer implementation.
 base::Time ChunkDemuxer::GetTimelineOffset() const {
   return timeline_offset_;
 }
 
 DemuxerStream* ChunkDemuxer::GetStream(DemuxerStream::Type type) {
   DCHECK_NE(type, DemuxerStream::TEXT);
   base::AutoLock auto_lock(lock_);
-  if (type == DemuxerStream::VIDEO)
-    return video_.get();
 
   if (type == DemuxerStream::AUDIO)
-    return audio_.get();
+    for (const auto& s : audio_streams_)
+      if (s->enabled())

[wolenetz, 2016/09/13 21:03:13]

Add TODO and crbug (if missing) w.r.t. mixing multiple enabled audio tracks.
Maybe "GetFirstEnabledStreamOfType()" vs "GetAllStreamsOfType()" (one for
current usage by renderer {with TODO+crbug to support audio mixing of multiple
enabled audio tracks}, and the latter for usage in UMA reporting). WDYT?

[servolk, 2016/09/14 18:15:24]

We should probably keep the GetStream name for now, since that's part of the
DemuxerStreamProvider interface and is overridden in 11 places and called in ~22
places according to code search. I agree that going forward we'll probably need
to do some overhaul of the DemuxerStreamProvider interface so that it allows
handling multiple demuxer streams of each type. I have some ideas around that,
but it'll need to be a separate CL since that's going to be a pretty big change.
Filed a bug and added a TODO to explain this.

[wolenetz, 2016/09/14 23:31:21]

Acknowledged.

+        return s.get();
+
+  if (type == DemuxerStream::VIDEO)
+    for (const auto& s : video_streams_)
+      if (s->enabled())
+        return s.get();
 
   return NULL;
 }
 
 TimeDelta ChunkDemuxer::GetStartTime() const {
   return TimeDelta();
 }
 
 int64_t ChunkDemuxer::GetMemoryUsage() const {
   base::AutoLock auto_lock(lock_);
-  return (audio_ ? audio_->GetBufferedSize() : 0) +
-         (video_ ? video_->GetBufferedSize() : 0);
+  int64_t mem = 0;
+  for (const auto& s : audio_streams_)
+    mem += s->GetBufferedSize();
+  for (const auto& s : video_streams_)
+    mem += s->GetBufferedSize();
+  return mem;
 }
 
 void ChunkDemuxer::AbortPendingReads() {
   base::AutoLock auto_lock(lock_);
   DCHECK(state_ == INITIALIZED || state_ == ENDED || state_ == SHUTDOWN ||
          state_ == PARSE_ERROR)
       << state_;
 
   if (state_ == SHUTDOWN || state_ == PARSE_ERROR)
     return;
@@ skipping 33 matching lines @@
   if (seek_cb_.is_null()) {
     cancel_next_seek_ = true;
     return;
   }
 
   base::ResetAndReturn(&seek_cb_).Run(PIPELINE_OK);
 }
 
 ChunkDemuxer::Status ChunkDemuxer::AddId(const std::string& id,
                                          const std::string& type,
-                                         std::vector<std::string>& codecs) {
+                                         const std::string& codecs) {
+  DVLOG(1) << __func__ << " id=" << id << " mime_type=" << type
+           << " codecs=" << codecs;
   base::AutoLock auto_lock(lock_);
 
   if ((state_ != WAITING_FOR_INIT && state_ != INITIALIZING) || IsValidId(id))
     return kReachedIdLimit;
 
-  bool has_audio = false;
-  bool has_video = false;
+  std::vector<std::string> parsed_codec_ids;
+  media::ParseCodecString(codecs, &parsed_codec_ids, false);
+
   std::unique_ptr<media::StreamParser> stream_parser(
-      StreamParserFactory::Create(type, codecs, media_log_, &has_audio,
-                                  &has_video));
+      StreamParserFactory::Create(type, parsed_codec_ids, media_log_));
 
   if (!stream_parser)
     return ChunkDemuxer::kNotSupported;
 
-  if ((has_audio && !source_id_audio_.empty()) ||
-      (has_video && !source_id_video_.empty()))
-    return kReachedIdLimit;
-
-  if (has_audio)
-    source_id_audio_ = id;
-
-  if (has_video)
-    source_id_video_ = id;
-
   std::unique_ptr<FrameProcessor> frame_processor(
       new FrameProcessor(base::Bind(&ChunkDemuxer::IncreaseDurationIfNecessary,
                                     base::Unretained(this)),
                          media_log_));
 
-  std::unique_ptr<MediaSourceState> source_state(new MediaSourceState(
-      std::move(stream_parser), std::move(frame_processor),
-      base::Bind(&ChunkDemuxer::CreateDemuxerStream, base::Unretained(this)),
-      media_log_));
+  std::unique_ptr<MediaSourceState> source_state(
+      new MediaSourceState(std::move(stream_parser), std::move(frame_processor),
+                           base::Bind(&ChunkDemuxer::CreateDemuxerStream,
+                                      base::Unretained(this), id),
+                           media_log_));
 
   MediaSourceState::NewTextTrackCB new_text_track_cb;
 
   if (enable_text_) {
     new_text_track_cb = base::Bind(&ChunkDemuxer::OnNewTextTrack,
                                    base::Unretained(this));
   }
 
-  pending_source_init_done_count_++;
+  pending_source_init_ids_.insert(id);
+
+  std::string expected_mss_codecs = codecs;
+  if (codecs == "" && type == "audio/aac")
+    expected_mss_codecs = "aac";
+  if (codecs == "" && (type == "audio/mpeg" || type == "audio/mp3"))
+    expected_mss_codecs = "mp3";
 
   source_state->Init(
-      base::Bind(&ChunkDemuxer::OnSourceInitDone, base::Unretained(this)),
-      has_audio, has_video, encrypted_media_init_data_cb_, new_text_track_cb);
+      base::Bind(&ChunkDemuxer::OnSourceInitDone, base::Unretained(this), id),
+      expected_mss_codecs, encrypted_media_init_data_cb_, new_text_track_cb);
 
   source_state_map_[id] = source_state.release();
   return kOk;
 }
 
 void ChunkDemuxer::SetTracksWatcher(
     const std::string& id,
     const MediaTracksUpdatedCB& tracks_updated_cb) {
   base::AutoLock auto_lock(lock_);
   CHECK(IsValidId(id));
   source_state_map_[id]->SetTracksWatcher(tracks_updated_cb);
 }
 
 void ChunkDemuxer::RemoveId(const std::string& id) {
+  DVLOG(1) << __func__ << " id=" << id;
   base::AutoLock auto_lock(lock_);
   CHECK(IsValidId(id));
 
   delete source_state_map_[id];
   source_state_map_.erase(id);
-
-  if (source_id_audio_ == id)
-    source_id_audio_.clear();
-
-  if (source_id_video_ == id)
-    source_id_video_.clear();
+  pending_source_init_ids_.erase(id);
+  // Remove demuxer streams created for this id.
+  for (const ChunkDemuxerStream* s : id_to_streams_map_[id]) {
+    for (size_t i = 0; i < audio_streams_.size(); ++i) {

[wolenetz, 2016/09/13 21:03:13]

nit: CHECK that we actually found and moved all the streams for id into
removed_streams_. Not DCHECK, since a raw pointer in demuxer client might still
be used to deref something that's been deleted if not correctly moved into
removed_streams_.

[servolk, 2016/09/14 18:15:23]

Done.

+      if (audio_streams_[i].get() == s) {
+        removed_streams_.push_back(std::move(audio_streams_[i]));
+        audio_streams_.erase(audio_streams_.begin() + i);
+        break;
+      }
+    }
+    for (size_t i = 0; i < video_streams_.size(); ++i) {
+      if (video_streams_[i].get() == s) {
+        removed_streams_.push_back(std::move(video_streams_[i]));
+        video_streams_.erase(video_streams_.begin() + i);
+        break;
+      }
+    }
+  }
+  id_to_streams_map_.erase(id);
 }
 
 Ranges<TimeDelta> ChunkDemuxer::GetBufferedRanges(const std::string& id) const {
   base::AutoLock auto_lock(lock_);
   DCHECK(!id.empty());
 
   MediaSourceStateMap::const_iterator itr = source_state_map_.find(id);
 
   DCHECK(itr != source_state_map_.end());
   return itr->second->GetBufferedRanges(duration_, state_ == ENDED);
 }
 
 base::TimeDelta ChunkDemuxer::GetHighestPresentationTimestamp(
     const std::string& id) const {
   base::AutoLock auto_lock(lock_);
   DCHECK(!id.empty());
 
   MediaSourceStateMap::const_iterator itr = source_state_map_.find(id);
 
   DCHECK(itr != source_state_map_.end());
   return itr->second->GetHighestPresentationTimestamp();
 }
 
 void ChunkDemuxer::OnEnabledAudioTracksChanged(
     const std::vector<MediaTrack::Id>& track_ids,
     base::TimeDelta currTime) {
-  // Note: We intentionally don't lock here, since we are not accessing any
-  // members directly.
-  DemuxerStream* audio_stream = GetStream(DemuxerStream::AUDIO);
-  bool enabled = false;
-  CHECK(audio_stream);
-  DCHECK_LE(track_ids.size(), 1u);
-  if (track_ids.size() > 0) {
-#if DCHECK_IS_ON()
-    base::AutoLock auto_lock(lock_);
-    DCHECK(track_id_to_demux_stream_map_[track_ids[0]] == audio_stream);
-#endif
-    enabled = true;
+  base::AutoLock auto_lock(lock_);
+  std::set<DemuxerStream*> enabled_streams;
+  for (const auto& id : track_ids) {
+    DemuxerStream* stream = track_id_to_demux_stream_map_[id];
+    DCHECK(stream);
+    DCHECK_EQ(DemuxerStream::AUDIO, stream->type());
+    enabled_streams.insert(stream);
   }
-  DVLOG(1) << __func__ << ": " << (enabled ? "enabling" : "disabling")
-           << " audio stream";
-  audio_stream->set_enabled(enabled, currTime);
+
+  // First disable all streams that need to be disabled and then enable streams
+  // that are enabled.
+  for (const auto& stream : audio_streams_) {
+    if (enabled_streams.find(stream.get()) == enabled_streams.end()) {
+      DVLOG(1) << __func__ << ": disabling stream " << stream.get();
+      stream->set_enabled(false, currTime);
+    }
+  }
+  for (const auto& stream : enabled_streams) {
+    DVLOG(1) << __func__ << ": enabling stream " << stream;
+    stream->set_enabled(true, currTime);
+  }
 }
 
 void ChunkDemuxer::OnSelectedVideoTrackChanged(
     const std::vector<MediaTrack::Id>& track_ids,
     base::TimeDelta currTime) {
-  // Note: We intentionally don't lock here, since we are not accessing any
-  // members directly.
-  DemuxerStream* video_stream = GetStream(DemuxerStream::VIDEO);
-  bool enabled = false;
-  CHECK(video_stream);
   DCHECK_LE(track_ids.size(), 1u);
-  if (track_ids.size() > 0) {
-#if DCHECK_IS_ON()
-    base::AutoLock auto_lock(lock_);
-    DCHECK(track_id_to_demux_stream_map_[track_ids[0]] == video_stream);
-#endif
-    enabled = true;
+
+  base::AutoLock auto_lock(lock_);
+  DemuxerStream* selected_stream = nullptr;
+  if (!track_ids.empty()) {
+    selected_stream = track_id_to_demux_stream_map_[track_ids[0]];
+    DCHECK(selected_stream);
+    DCHECK_EQ(DemuxerStream::VIDEO, selected_stream->type());
   }
-  DVLOG(1) << __func__ << ": " << (enabled ? "enabling" : "disabling")
-           << " video stream";
-  video_stream->set_enabled(enabled, currTime);
+
+  // First disable all streams that need to be disabled and then enable the
+  // stream that needs to be enabled (if any).
+  for (const auto& stream : video_streams_) {
+    if (stream->type() == DemuxerStream::VIDEO &&
+        stream.get() != selected_stream) {
+      DVLOG(1) << __func__ << ": disabling stream " << stream.get();
+      stream->set_enabled(false, currTime);
+    }
+  }
+  if (selected_stream) {
+    DVLOG(1) << __func__ << ": enabling stream " << selected_stream;
+    selected_stream->set_enabled(true, currTime);
+  }
 }
 
 bool ChunkDemuxer::EvictCodedFrames(const std::string& id,
                                     base::TimeDelta currentMediaTime,
                                     size_t newDataSize) {
   DVLOG(1) << __func__ << "(" << id << ")"
            << " media_time=" << currentMediaTime.InSecondsF()
            << " newDataSize=" << newDataSize;
   base::AutoLock auto_lock(lock_);
 
@@ skipping 329 matching lines @@
   for (MediaSourceStateMap::const_iterator itr = source_state_map_.begin();
        itr != source_state_map_.end(); ++itr) {
     if (itr->second->IsSeekWaitingForData())
       return true;
   }
 
   return false;
 }
 
 void ChunkDemuxer::OnSourceInitDone(
+    const std::string& source_id,
     const StreamParser::InitParameters& params) {
-  DVLOG(1) << "OnSourceInitDone(" << params.duration.InSecondsF() << ")";
+  DVLOG(1) << "OnSourceInitDone source_id=" << source_id
+           << " duration=" << params.duration.InSecondsF();
   lock_.AssertAcquired();
   DCHECK_EQ(state_, INITIALIZING);
-  if (!audio_ && !video_) {
+  DCHECK(pending_source_init_ids_.find(source_id) !=
+         pending_source_init_ids_.end());
+  if (audio_streams_.empty() && video_streams_.empty()) {
     ReportError_Locked(DEMUXER_ERROR_COULD_NOT_OPEN);
     return;
   }
 
   if (!params.duration.is_zero() && duration_ == kNoTimestamp)
     UpdateDuration(params.duration);
 
   if (!params.timeline_offset.is_null()) {
     if (!timeline_offset_.is_null() &&
         params.timeline_offset != timeline_offset_) {
       MEDIA_LOG(ERROR, media_log_)
           << "Timeline offset is not the same across all SourceBuffers.";
       ReportError_Locked(DEMUXER_ERROR_COULD_NOT_OPEN);
       return;
     }
 
     timeline_offset_ = params.timeline_offset;
   }
 
   if (params.liveness != DemuxerStream::LIVENESS_UNKNOWN) {
-    if (audio_)
-      audio_->SetLiveness(params.liveness);
-    if (video_)
-      video_->SetLiveness(params.liveness);
+    for (const auto& s : audio_streams_)
+      s->SetLiveness(params.liveness);
+    for (const auto& s : video_streams_)
+      s->SetLiveness(params.liveness);
   }
 
   detected_audio_track_count_ += params.detected_audio_track_count;
   detected_video_track_count_ += params.detected_video_track_count;
   detected_text_track_count_ += params.detected_text_track_count;
 
   // Wait until all streams have initialized.
-  pending_source_init_done_count_--;
-
-  if (pending_source_init_done_count_ > 0)
+  pending_source_init_ids_.erase(source_id);
+  if (!pending_source_init_ids_.empty())
     return;
 
-  DCHECK_EQ(0, pending_source_init_done_count_);
-  DCHECK((source_id_audio_.empty() == !audio_) &&
-         (source_id_video_.empty() == !video_));
-
   // Record detected track counts by type corresponding to an MSE playback.
   // Counts are split into 50 buckets, capped into [0,100] range.
   UMA_HISTOGRAM_COUNTS_100("Media.MSE.DetectedTrackCount.Audio",
                            detected_audio_track_count_);
   UMA_HISTOGRAM_COUNTS_100("Media.MSE.DetectedTrackCount.Video",
                            detected_video_track_count_);
   UMA_HISTOGRAM_COUNTS_100("Media.MSE.DetectedTrackCount.Text",
                            detected_text_track_count_);
 
-  if (video_) {
+  for (const auto& s : video_streams_) {
     media_log_->RecordRapporWithSecurityOrigin(
         "Media.OriginUrl.MSE.VideoCodec." +
-        GetCodecName(video_->video_decoder_config().codec()));
+        GetCodecName(s->video_decoder_config().codec()));
   }
 
   SeekAllSources(GetStartTime());
   StartReturningData();
 
   if (duration_ == kNoTimestamp)
     duration_ = kInfiniteDuration;
 
   // The demuxer is now initialized after the |start_timestamp_| was set.
   ChangeState_Locked(INITIALIZED);
   base::ResetAndReturn(&init_cb_).Run(PIPELINE_OK);
 }
 
 // static
 MediaTrack::Id ChunkDemuxer::GenerateMediaTrackId() {
   static unsigned g_track_count = 0;
   return base::UintToString(++g_track_count);
 }
 
 ChunkDemuxerStream* ChunkDemuxer::CreateDemuxerStream(
+    const std::string& source_id,
     DemuxerStream::Type type) {
   // New ChunkDemuxerStreams can be created only during initialization segment
   // processing, which happens when a new chunk of data is appended and the
   // lock_ must be held by ChunkDemuxer::AppendData.
   lock_.AssertAcquired();
 
   MediaTrack::Id media_track_id = GenerateMediaTrackId();
 
   switch (type) {
-    case DemuxerStream::AUDIO:
-      if (audio_)
-        return NULL;
-      audio_.reset(new ChunkDemuxerStream(
+    case DemuxerStream::AUDIO: {
+      std::unique_ptr<ChunkDemuxerStream> audio_stream(new ChunkDemuxerStream(
           DemuxerStream::AUDIO, splice_frames_enabled_, media_track_id));
       DCHECK(track_id_to_demux_stream_map_.find(media_track_id) ==
              track_id_to_demux_stream_map_.end());
-      track_id_to_demux_stream_map_[media_track_id] = audio_.get();
-      return audio_.get();
-      break;
-    case DemuxerStream::VIDEO:
-      if (video_)
-        return NULL;
-      video_.reset(new ChunkDemuxerStream(
+      track_id_to_demux_stream_map_[media_track_id] = audio_stream.get();
+      id_to_streams_map_[source_id].push_back(audio_stream.get());
+      audio_streams_.push_back(std::move(audio_stream));
+      return audio_streams_.back().get();
+    }
+    case DemuxerStream::VIDEO: {
+      std::unique_ptr<ChunkDemuxerStream> video_stream(new ChunkDemuxerStream(
           DemuxerStream::VIDEO, splice_frames_enabled_, media_track_id));
       DCHECK(track_id_to_demux_stream_map_.find(media_track_id) ==
              track_id_to_demux_stream_map_.end());
-      track_id_to_demux_stream_map_[media_track_id] = video_.get();
-      return video_.get();
-      break;
+      track_id_to_demux_stream_map_[media_track_id] = video_stream.get();
+      id_to_streams_map_[source_id].push_back(video_stream.get());
+      video_streams_.push_back(std::move(video_stream));
+      return video_streams_.back().get();
+    }
     case DemuxerStream::TEXT: {
-      return new ChunkDemuxerStream(DemuxerStream::TEXT, splice_frames_enabled_,
-                                    media_track_id);
-      break;
+      ChunkDemuxerStream* text_stream = new ChunkDemuxerStream(
+          DemuxerStream::TEXT, splice_frames_enabled_, media_track_id);
+      id_to_streams_map_[source_id].push_back(text_stream);
+      return text_stream;
     }
     case DemuxerStream::UNKNOWN:
     case DemuxerStream::NUM_TYPES:
       NOTREACHED();
       return NULL;
   }
   NOTREACHED();
   return NULL;
 }
 
@@ skipping 103 matching lines @@
 }
 
 void ChunkDemuxer::ShutdownAllStreams() {
   for (MediaSourceStateMap::iterator itr = source_state_map_.begin();
        itr != source_state_map_.end(); ++itr) {
     itr->second->Shutdown();
   }
 }
 
 }  // namespace media