Allow MP4 parser to handle multiple audio and video tracks

media/formats/mp4/mp4_stream_parser.cc

 // Copyright 2014 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "media/formats/mp4/mp4_stream_parser.h"
 
 #include <stddef.h>
 
 #include <limits>
 #include <memory>
@@ skipping 23 matching lines @@
 namespace mp4 {
 
 MP4StreamParser::MP4StreamParser(const std::set<int>& audio_object_types,
                                  bool has_sbr)
     : state_(kWaitingForInit),
       moof_head_(0),
       mdat_tail_(0),
       highest_end_offset_(0),
       has_audio_(false),
       has_video_(false),
-      audio_track_id_(0),
-      video_track_id_(0),
       audio_object_types_(audio_object_types),
       has_sbr_(has_sbr),
-      is_audio_track_encrypted_(false),
-      is_video_track_encrypted_(false),
       num_top_level_box_skipped_(0) {
 }
 
 MP4StreamParser::~MP4StreamParser() {}
 
 void MP4StreamParser::Init(
     const InitCB& init_cb,
     const NewConfigCB& config_cb,
     const NewBuffersCB& new_buffers_cb,
     bool /* ignore_text_tracks */,
@@ skipping 119 matching lines @@
   }
 
   queue_.Pop(reader->size());
   return !(*err);
 }
 
 bool MP4StreamParser::ParseMoov(BoxReader* reader) {
   moov_.reset(new Movie);
   RCHECK(moov_->Parse(reader));
   runs_.reset();
+  audio_track_ids_.clear();
+  video_track_ids_.clear();
+  is_track_encrypted_.clear();
 
   has_audio_ = false;
   has_video_ = false;
 
   std::unique_ptr<MediaTracks> media_tracks(new MediaTracks());
   AudioDecoderConfig audio_config;
   VideoDecoderConfig video_config;
   int detected_audio_track_count = 0;
   int detected_video_track_count = 0;
   int detected_text_track_count = 0;
 
   for (std::vector<Track>::const_iterator track = moov_->tracks.begin();
        track != moov_->tracks.end(); ++track) {
     // TODO(strobe): Only the first audio and video track present in a file are

[wolenetz, 2016/08/23 22:51:42]

Comment is obsolete.

[servolk, 2016/08/24 00:53:49]

Done.

     // used. (Track selection is better accomplished via Source IDs, though, so
     // adding support for track selection within a stream is low-priority.)
     const SampleDescription& samp_descr =
         track->media.information.sample_table.description;
 
     // TODO(strobe): When codec reconfigurations are supported, detect and send
     // a codec reconfiguration for fragments using a sample description index
     // different from the previous one
     size_t desc_idx = 0;
     for (size_t t = 0; t < moov_->extends.tracks.size(); t++) {
       const TrackExtends& trex = moov_->extends.tracks[t];
       if (trex.track_id == track->header.track_id) {
         desc_idx = trex.default_sample_description_index;
         break;
       }
     }
     RCHECK(desc_idx > 0);
     desc_idx -= 1;  // BMFF descriptor index is one-based
 
     if (track->media.handler.type == kAudio) {
       detected_audio_track_count++;
-      if (audio_config.IsValidConfig())
-        continue;  // Skip other audio tracks once we found a supported one.
 
       RCHECK(!samp_descr.audio_entries.empty());
 
       // It is not uncommon to find otherwise-valid files with incorrect sample
       // description indices, so we fail gracefully in that case.
       if (desc_idx >= samp_descr.audio_entries.size())
         desc_idx = 0;
       const AudioSampleEntry& entry = samp_descr.audio_entries[desc_idx];
       const AAC& aac = entry.esds.aac;
 
@@ skipping 68 matching lines @@
         sample_format = kSampleFormatU8;
       } else if (entry.samplesize == 16) {
         sample_format = kSampleFormatS16;
       } else if (entry.samplesize == 32) {
         sample_format = kSampleFormatS32;
       } else {
         LOG(ERROR) << "Unsupported sample size.";
         return false;
       }
 
-      is_audio_track_encrypted_ = entry.sinf.info.track_encryption.is_encrypted;
-      DVLOG(1) << "is_audio_track_encrypted_: " << is_audio_track_encrypted_;
+      uint32_t audio_track_id = track->header.track_id;
+      bool is_track_encrypted = entry.sinf.info.track_encryption.is_encrypted;
+      is_track_encrypted_[audio_track_id] = is_track_encrypted;
       audio_config.Initialize(
           codec, sample_format, channel_layout, sample_per_second, extra_data,
-          is_audio_track_encrypted_ ? AesCtrEncryptionScheme() : Unencrypted(),
+          is_track_encrypted ? AesCtrEncryptionScheme() : Unencrypted(),
           base::TimeDelta(), 0);
+      DVLOG(1) << "audio_track_id=" << audio_track_id
+               << " config=" << audio_config.AsHumanReadableString();
       if (!audio_config.IsValidConfig()) {
         MEDIA_LOG(ERROR, media_log_) << "Invalid audio decoder config: "
                                      << audio_config.AsHumanReadableString();
         return false;
       }
       has_audio_ = true;
-      audio_track_id_ = track->header.track_id;
-      media_tracks->AddAudioTrack(audio_config, audio_track_id_, "main",
+      audio_track_ids_.insert(audio_track_id);
+      media_tracks->AddAudioTrack(audio_config, audio_track_id, "main",

[wolenetz, 2016/08/23 22:51:42]

Are all audio tracks "main" in a multi-track audio mp4? IIRC, the inband
sourcing spec may help answer this.
Consult: https://dev.w3.org/html5/html-sourcing-inband-tracks/#mpeg4 "Track
Attributes for sourced Audio and Video Tracks" section.

[servolk, 2016/08/24 00:53:49]

Done.

                                   track->media.handler.name,
                                   track->media.header.language());
       continue;
     }
 
     if (track->media.handler.type == kVideo) {
       detected_video_track_count++;
-      if (video_config.IsValidConfig())
-        continue;  // Skip other video tracks once we found a supported one.
 
       RCHECK(!samp_descr.video_entries.empty());
       if (desc_idx >= samp_descr.video_entries.size())
         desc_idx = 0;
       const VideoSampleEntry& entry = samp_descr.video_entries[desc_idx];
 
       if (!entry.IsFormatValid()) {
         MEDIA_LOG(ERROR, media_log_) << "Unsupported video format 0x"
                                      << std::hex << entry.format
                                      << " in stsd box.";
@@ skipping 10 matching lines @@
       if (entry.pixel_aspect.h_spacing != 1 ||
           entry.pixel_aspect.v_spacing != 1) {
         natural_size =
             GetNaturalSize(visible_rect.size(), entry.pixel_aspect.h_spacing,
                            entry.pixel_aspect.v_spacing);
       } else if (track->header.width && track->header.height) {
         natural_size =
             gfx::Size(track->header.width, track->header.height);
       }
 
-      is_video_track_encrypted_ = entry.sinf.info.track_encryption.is_encrypted;
-      DVLOG(1) << "is_video_track_encrypted_: " << is_video_track_encrypted_;
+      uint32_t video_track_id = track->header.track_id;
+      bool is_track_encrypted = entry.sinf.info.track_encryption.is_encrypted;
+      is_track_encrypted_[video_track_id] = is_track_encrypted;

[chcunningham1, 2016/08/23 01:54:32]

Maybe capture the return value and check that its not already present in the set
(i.e. confirm media does not re-use track ids)?

[wolenetz, 2016/08/23 22:51:42]

+1 to unique track id checking (here and in the audio case)

[servolk, 2016/08/24 00:53:49]

Done.

       video_config.Initialize(
           entry.video_codec, entry.video_codec_profile, PIXEL_FORMAT_YV12,
           COLOR_SPACE_HD_REC709, coded_size, visible_rect, natural_size,
           // No decoder-specific buffer needed for AVC;
           // SPS/PPS are embedded in the video stream
           EmptyExtraData(),
-          is_video_track_encrypted_ ? AesCtrEncryptionScheme() : Unencrypted());
+          is_track_encrypted ? AesCtrEncryptionScheme() : Unencrypted());
+      DVLOG(1) << "video_track_id=" << video_track_id
+               << " config=" << video_config.AsHumanReadableString();
       if (!video_config.IsValidConfig()) {
         MEDIA_LOG(ERROR, media_log_) << "Invalid video decoder config: "
                                      << video_config.AsHumanReadableString();
         return false;
       }
       has_video_ = true;
-      video_track_id_ = track->header.track_id;
-      media_tracks->AddVideoTrack(video_config, video_track_id_, "main",
+      video_track_ids_.insert(video_track_id);
+      media_tracks->AddVideoTrack(video_config, video_track_id, "main",

[wolenetz, 2016/08/23 22:51:42]

ditto: "main" is only for the first video track.

[servolk, 2016/08/24 00:53:49]

Done.

                                   track->media.handler.name,
                                   track->media.header.language());
       continue;
     }
 
     // TODO(wolenetz): Investigate support in MSE and Chrome MSE for CEA 608/708
     // embedded caption data in video track. At time of init segment parsing, we
     // don't have this data (unless maybe by SourceBuffer's mimetype).
     // See https://crbug.com/597073
     if (track->media.handler.type == kText)
@@ skipping 116 matching lines @@
     return true;
   }
 
   DCHECK(!(*err));
 
   const uint8_t* buf;
   int buf_size;
   queue_.Peek(&buf, &buf_size);
   if (!buf_size) return false;
 
-  bool audio = has_audio_ && audio_track_id_ == runs_->track_id();
-  bool video = has_video_ && video_track_id_ == runs_->track_id();
+  bool audio =
+      audio_track_ids_.find(runs_->track_id()) != audio_track_ids_.end();
+  bool video =
+      video_track_ids_.find(runs_->track_id()) != video_track_ids_.end();
 
   // Skip this entire track if it's not one we're interested in
   if (!audio && !video) {
     runs_->AdvanceRun();
     return true;
   }
 
   // Attempt to cache the auxiliary information first. Aux info is usually
   // placed in a contiguous block before the sample data, rather than being
   // interleaved. If we didn't cache it, this would require that we retain the
@@ skipping 49 matching lines @@
 
   if (decrypt_config) {
     if (!subsamples.empty()) {
     // Create a new config with the updated subsamples.
     decrypt_config.reset(new DecryptConfig(
         decrypt_config->key_id(),
         decrypt_config->iv(),
         subsamples));
     }
     // else, use the existing config.
-  } else if ((audio && is_audio_track_encrypted_) ||
-             (video && is_video_track_encrypted_)) {
+  } else if (is_track_encrypted_[runs_->track_id()]) {
     // The media pipeline requires a DecryptConfig with an empty |iv|.
     // TODO(ddorwin): Refactor so we do not need a fake key ID ("1");
     decrypt_config.reset(
         new DecryptConfig("1", "", std::vector<SubsampleEntry>()));
   }
 
   StreamParserBuffer::Type buffer_type = audio ? DemuxerStream::AUDIO :
       DemuxerStream::VIDEO;
 
-  // TODO(wolenetz/acolwell): Validate and use a common cross-parser TrackId
-  // type and allow multiple tracks for same media type, if applicable. See
-  // https://crbug.com/341581.
   scoped_refptr<StreamParserBuffer> stream_buf = StreamParserBuffer::CopyFrom(
       &frame_buf[0], frame_buf.size(), runs_->is_keyframe(), buffer_type,
       runs_->track_id());
 
   if (decrypt_config)
     stream_buf->set_decrypt_config(std::move(decrypt_config));
 
   stream_buf->set_duration(runs_->duration());
   stream_buf->set_timestamp(runs_->cts());
   stream_buf->SetDecodeTimestamp(runs_->dts());
 
-  DVLOG(3) << "Pushing frame: aud=" << audio
+  DVLOG(3) << "Emit " << (audio ? "audio" : "video") << " frame: "

[chcunningham1, 2016/08/23 01:54:32]

what if the track is text? will you say "video" here?

[wolenetz, 2016/08/23 22:51:42]

At the moment, Chrome MSE mp4 parser doesn't parse text tracks.

[servolk, 2016/08/24 00:53:49]

Yup, text tracks in .mp4 are not supported for now.

+           << " track_id=" << runs_->track_id()
            << ", key=" << runs_->is_keyframe()
            << ", dur=" << runs_->duration().InMilliseconds()
            << ", dts=" << runs_->dts().InMilliseconds()
            << ", cts=" << runs_->cts().InMilliseconds()
            << ", size=" << runs_->sample_size();
 
   (*buffers)[runs_->track_id()].push_back(stream_buf);
   runs_->AdvanceSample();
   return true;
 }
@@ skipping 66 matching lines @@
       runs.AdvanceSample();
     }
     runs.AdvanceRun();
   }
 
   return true;
 }
 
 }  // namespace mp4
 }  // namespace media