Add Common Encryption support to BMFF, including subsample decryption.

media/mp4/mp4_stream_parser.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "media/mp4/mp4_stream_parser.h"
 
 #include "base/callback.h"
 #include "base/callback_helpers.h"
 #include "base/logging.h"
 #include "base/time.h"
@@ skipping 139 matching lines @@
        track != moov_->tracks.end(); ++track) {
     // TODO(strobe): Only the first audio and video track present in a file are
     // used. (Track selection is better accomplished via Source IDs, though, so
     // adding support for track selection within a stream is low-priority.)
     const SampleDescription& samp_descr =
         track->media.information.sample_table.description;
     if (track->media.handler.type == kAudio && !audio_config.IsValidConfig()) {
       RCHECK(!samp_descr.audio_entries.empty());
       const AudioSampleEntry& entry = samp_descr.audio_entries[0];
 
-      // TODO(strobe): We accept all format values, pending clarification on
-      // the formats used for encrypted media (http://crbug.com/132351).
-      //  RCHECK(entry.format == FOURCC_MP4A ||
-      //         (entry.format == FOURCC_ENCA &&
-      //          entry.sinf.format.format == FOURCC_MP4A));
+      RCHECK(entry.format == FOURCC_MP4A ||

[ddorwin, 2012/07/17 01:14:21]

Is this worth logging since it means the file format is incompatible (i.e.
problem with user input)?

[strobe_, 2012/07/19 02:43:35]

I'd prefer to leave it in. DLOG() is cheaper than playing "find the right
'return false';" later on. (Additionally, we may later decide to provide
informative messages to the application.)

[ddorwin, 2012/07/19 06:11:10]

I think I meant should we ALWAYS log it.

[strobe_, 2012/07/19 16:55:02]

Oh! RCHECK() does a DLOG(ERROR) on false. Should we promote that to a LOG() (it
pretty much always indicates user/file error), or just add a separate LOG() here
and below for these cases?

[ddorwin, 2012/07/24 01:00:09]

I was thinking a separate LOG() for these cases since they are valid files that
just aren't supported. Most of the others probably mean the file is somehow
invalid.
For example, if I try to play my file in Chrome and it doesn't work, this log
would help me figure it out.

+             (entry.format == FOURCC_ENCA &&
+              entry.sinf.format.format == FOURCC_MP4A));
+      RCHECK(EmitKeyNeeded(entry.sinf.info.track_encryption));
 
       // Check if it is MPEG4 AAC defined in ISO 14496 Part 3.
       RCHECK(entry.esds.object_type == kISO_14496_3);

[ddorwin, 2012/07/17 01:14:21]

same

       aac_ = entry.esds.aac;
       audio_config.Initialize(kCodecAAC, entry.samplesize,
                               aac_.channel_layout(), aac_.frequency(),
                               NULL, 0, false);
 
       has_audio_ = true;
       audio_track_id_ = track->header.track_id;
     }
     if (track->media.handler.type == kVideo && !video_config.IsValidConfig()) {
       RCHECK(!samp_descr.video_entries.empty());
       const VideoSampleEntry& entry = samp_descr.video_entries[0];
 
-      //  RCHECK(entry.format == FOURCC_AVC1 ||
-      //         (entry.format == FOURCC_ENCV &&
-      //          entry.sinf.format.format == FOURCC_AVC1));
+      RCHECK(entry.format == FOURCC_AVC1 ||

[ddorwin, 2012/07/17 01:14:21]

same

+             (entry.format == FOURCC_ENCV &&
+              entry.sinf.format.format == FOURCC_AVC1));
+      RCHECK(EmitKeyNeeded(entry.sinf.info.track_encryption));
 
       // TODO(strobe): Recover correct crop box
       video_config.Initialize(kCodecH264, H264PROFILE_MAIN,  VideoFrame::YV12,
                               gfx::Size(entry.width, entry.height),
                               gfx::Rect(0, 0, entry.width, entry.height),
                               // Framerate of zero is provided to signal that
                               // the decoder should trust demuxer timestamps
                               0, 1,

[ddorwin, 2012/07/17 01:14:21]

what is the 1?
if it's not paired with the 0 (or even if), probably better to put on separate
line since it's the only one that's doubled up.

[strobe_, 2012/07/19 02:43:35]

Framerate numerator and denominator.

                               entry.pixel_aspect.h_spacing,
                               entry.pixel_aspect.v_spacing,
                               // No decoder-specific buffer needed for AVC;
                               // SPS/PPS are embedded in the video stream
                               NULL, 0, false);
       has_video_ = true;
       video_track_id_ = track->header.track_id;
 
       size_of_nalu_length_ = entry.avcc.length_size;
     }
   }
 
   // TODO(strobe): For now, we avoid sending new configs on a new
   // reinitialization segment, and instead simply embed the updated parameter
   // sets into the video stream. The conditional should be removed when
   // http://crbug.com/122913 is fixed.
   if (!init_cb_.is_null())

[ddorwin, 2012/07/17 01:14:21]

checking wrong callback?

[strobe_, 2012/07/19 02:43:35]

No, I think this is correct. We currently use nullness of init_cb (which should
only be called once ever in the current model, and is set to null after that
first call) as a signal to indicate that config_cb should not be called again.
We don't null config_cb directly because it will eventually be the right way to
send this information. This is covered by a test.

[ddorwin, 2012/07/19 06:11:10]

Hmm, might be worth a comment explaining that here since the pattern is
unexpected.

[strobe_, 2012/07/19 16:55:02]

Done.

     RCHECK(config_cb_.Run(audio_config, video_config));
 
   base::TimeDelta duration;
   if (moov_->extends.header.fragment_duration > 0) {
     duration = TimeDeltaFromFrac(moov_->extends.header.fragment_duration,
                                  moov_->header.timescale);
   } else if (moov_->header.duration > 0) {
     duration = TimeDeltaFromFrac(moov_->header.duration,
                                  moov_->header.timescale);
   } else {
     duration = kInfiniteDuration();
   }
 
   if (!init_cb_.is_null())
     base::ResetAndReturn(&init_cb_).Run(true, duration);
   return true;
 }
 
 bool MP4StreamParser::ParseMoof(BoxReader* reader) {
   RCHECK(moov_.get());  // Must already have initialization segment
   MovieFragment moof;
   RCHECK(moof.Parse(reader));
   RCHECK(runs_.Init(*moov_, moof));
   new_segment_cb_.Run(runs_.GetMinDecodeTimestamp());
   ChangeState(kEmittingSamples);
   return true;
 }
 
+bool MP4StreamParser::EmitKeyNeeded(const TrackEncryption& track_encryption) {
+  // TODO(strobe): Send the correct value for initData. The format of initData
+  // has not yet been defined; see
+  // https://www.w3.org/Bugs/Public/show_bug.cgi?id=17673.
+  if (!track_encryption.is_encrypted) return true;
+  scoped_array<uint8> kid(new uint8[track_encryption.default_kid.size()]);
+  memcpy(kid.get(), &track_encryption.default_kid[0],
+         track_encryption.default_kid.size());
+  return need_key_cb_.Run(kid.Pass(), track_encryption.default_kid.size());
+}
+
+bool MP4StreamParser::PrepareAVCBuffer(
+    std::vector<uint8>* frame_buf,
+    std::vector<SubsampleEntry>* subsamples) {
+  // Convert the AVC NALU length fields to Annex B headers, as expected by
+  // decoding libraries. Since this may enlarge the size of the buffer, we also
+  // update the clear byte count for each subsample if encryption is used.
+  RCHECK(AVC::ConvertToAnnexB(size_of_nalu_length_, frame_buf));

[ddorwin, 2012/07/17 01:14:21]

Is there a [D]CHECK that size_of_nalu_length_ <= 4 somewhere?

[strobe_, 2012/07/19 02:43:35]

Yes, in ConvertToAnnexB.

+  if (!subsamples->empty()) {
+    const int nalu_size_diff = 4 - size_of_nalu_length_;
+    size_t expected_size = runs_.sample_size() +
+        subsamples->size() * nalu_size_diff;
+    RCHECK(frame_buf->size() == expected_size);
+    for (size_t i = 0; i < subsamples->size(); i++)
+      (*subsamples)[i].clear_bytes += nalu_size_diff;
+  }
+
+  if (runs_.is_keyframe()) {
+    // If this is a keyframe, we (re-)inject SPS and PPS headers at the start of
+    // a frame. If subsample info is present, we also update the clear byte

[ddorwin, 2012/07/17 01:14:21]

update to what?

[strobe_, 2012/07/19 02:43:35]

Commented.

[ddorwin, 2012/07/19 06:11:10]

Where?

[ddorwin, 2012/07/24 01:00:09]

Seem to have missed this one. :)

[strobe_, 2012/07/25 01:05:13]

Done.

+    // count for that first subsample.
+    const AVCDecoderConfigurationRecord* avc_config = NULL;
+    for (size_t t = 0; t < moov_->tracks.size(); t++) {
+      if (moov_->tracks[t].header.track_id == runs_.track_id()) {
+        avc_config = &moov_->tracks[t].media.information.

[ddorwin, 2012/07/17 01:14:21]

strange line break. I think everything after = should be on a new line(s).

[strobe_, 2012/07/19 02:43:35]

Rebased away.

[ddorwin, 2012/07/19 06:11:10]

Still here.

[strobe_, 2012/07/19 16:55:02]

Sorry, *really* fixed. Thanks for your dilligence!

+            sample_table.description.video_entries[0].avcc;

[ddorwin, 2012/07/17 01:14:21]

is video_entries size guaranteed to be >= 1?

[strobe_, 2012/07/19 02:43:35]

Rebased away.

[ddorwin, 2012/07/19 06:11:10]

Still here.

[strobe_, 2012/07/19 16:55:02]

Done.

+        break;
+      }
+    }
+    RCHECK(avc_config != NULL);
+    std::vector<uint8> param_sets;
+    RCHECK(AVC::ConvertParameterSets(*avc_config, &param_sets));

[ddorwin, 2012/07/17 01:14:21]

ConvertConfigToParameterSets?

[strobe_, 2012/07/19 02:43:35]

Done.

+    frame_buf->insert(frame_buf->begin(),
+                      param_sets.begin(), param_sets.end());
+    if (!subsamples->empty())
+      (*subsamples)[0].clear_bytes += param_sets.size();
+  }
+  return true;
+}
+
 bool MP4StreamParser::EnqueueSample(BufferQueue* audio_buffers,
                                     BufferQueue* video_buffers,
                                     bool* err) {
-  if (!runs_.RunValid()) {
+  if (!runs_.RunIsValid()) {

[ddorwin, 2012/07/17 01:14:21]

Why isn't this IsRunValid? Unless you really are running "is valid".

[strobe_, 2012/07/19 02:43:35]

Evidence of my functional bias. Renamed.

     // Flush any buffers we've gotten in this chunk so that buffers don't
     // cross NewSegment() calls
     *err = !SendAndFlushSamples(audio_buffers, video_buffers);
     if (*err) return false;
     ChangeState(kParsingBoxes);
     return true;
   }
 
-  if (!runs_.SampleValid()) {
+  if (!runs_.SampleIsValid()) {

[ddorwin, 2012/07/17 01:14:21]

same

[strobe_, 2012/07/19 02:43:35]

Done.

     runs_.AdvanceRun();
     return true;
   }
 
   DCHECK(!(*err));
 
   const uint8* buf;
-  int size;
-  queue_.Peek(&buf, &size);
-  if (!size) return false;
+  int buf_size;
+  queue_.Peek(&buf, &buf_size);
+  if (!buf_size) return false;
 
   bool audio = has_audio_ && audio_track_id_ == runs_.track_id();
   bool video = has_video_ && video_track_id_ == runs_.track_id();
 
   // Skip this entire track if it's not one we're interested in
   if (!audio && !video) runs_.AdvanceRun();
 
   // Attempt to cache the auxiliary information first. Aux info is usually
   // placed in a contiguous block before the sample data, rather than being
   // interleaved. If we didn't cache it, this would require that we retain the
   // start of the segment buffer while reading samples. Aux info is typically
   // quite small compared to sample data, so this pattern is useful on
   // memory-constrained devices where the source buffer consumes a substantial
   // portion of the total system memory.
-  if (runs_.NeedsCENC()) {
-    queue_.PeekAt(runs_.cenc_offset() + moof_head_, &buf, &size);
-    return runs_.CacheCENC(buf, size);
+  if (runs_.AuxInfoNeedsToBeCached()) {
+    queue_.PeekAt(runs_.aux_info_offset() + moof_head_, &buf, &buf_size);
+    if (buf_size < runs_.aux_info_size()) return false;
+    *err = !runs_.CacheAuxInfo(buf, buf_size);
+    return !*err;
   }
 
-  queue_.PeekAt(runs_.offset() + moof_head_, &buf, &size);
-  if (size < runs_.size()) return false;
+  queue_.PeekAt(runs_.sample_offset() + moof_head_, &buf, &buf_size);
+  if (buf_size < runs_.sample_size()) return false;
 
-  std::vector<uint8> frame_buf(buf, buf + runs_.size());
+  scoped_ptr<DecryptConfig> decrypt_config;
+  if (runs_.is_encrypted())
+    decrypt_config = runs_.GetDecryptConfig();
+
+  std::vector<uint8> frame_buf(buf, buf + runs_.sample_size());
   if (video) {
-    RCHECK(AVC::ConvertToAnnexB(size_of_nalu_length_, &frame_buf));
-    if (runs_.is_keyframe()) {
-      const AVCDecoderConfigurationRecord* avc_config = NULL;
-      for (size_t t = 0; t < moov_->tracks.size(); t++) {
-        if (moov_->tracks[t].header.track_id == runs_.track_id()) {
-          avc_config = &moov_->tracks[t].media.information.
-              sample_table.description.video_entries[0].avcc;
-          break;
-        }
-      }
-      RCHECK(avc_config != NULL);
-      RCHECK(AVC::InsertParameterSets(*avc_config, &frame_buf));
+    std::vector<SubsampleEntry> subsamples;
+    if (decrypt_config.get())
+      subsamples = decrypt_config->subsamples();
+    RCHECK(PrepareAVCBuffer(&frame_buf, &subsamples));

[ddorwin, 2012/07/17 01:14:21]

If else above, we want to run this for the key frame logic?

[strobe_, 2012/07/19 02:43:35]

Sorry, can you clarify this comment?

[ddorwin, 2012/07/19 06:11:10]

If line 340 - if (decrypt_config.get()) - is not true, subsamples in null and
PrepareAVCBuffer() just does something related to key frames. Is that correct?

[strobe_, 2012/07/19 16:55:02]

No. H.264 NALUs are stored in "AVC" format (length-prefixed) in BMFF, but most
decoders expect data in Annex B format (start-code-prefixed). That conversion
happens for every sample regardless of crypto or keyframe status.

[ddorwin, 2012/07/24 01:00:09]

Ahh, I overlooked the conversion call.

+    if (!subsamples.empty()) {
+      decrypt_config.reset(new DecryptConfig(
+          decrypt_config->key_id(),
+          decrypt_config->key_id_size(),
+          decrypt_config->iv(),
+          subsamples));
     }
   }
 
   if (audio) {
     aac_.ConvertEsdsToADTS(&frame_buf);
   }
 
   scoped_refptr<StreamParserBuffer> stream_buf =
     StreamParserBuffer::CopyFrom(&frame_buf[0], frame_buf.size(),
                                  runs_.is_keyframe());
 
+  if (runs_.is_encrypted())
+    stream_buf->SetDecryptConfig(decrypt_config.Pass());
+
   stream_buf->SetDuration(runs_.duration());
   stream_buf->SetTimestamp(runs_.cts());
   stream_buf->SetDecodeTimestamp(runs_.dts());
 
   DVLOG(3) << "Pushing frame: aud=" << audio
            << ", key=" << runs_.is_keyframe()
            << ", dur=" << runs_.duration().InMilliseconds()
            << ", dts=" << runs_.dts().InMilliseconds()
            << ", cts=" << runs_.cts().InMilliseconds()

[ddorwin, 2012/07/17 01:14:21]

set cts before dts above. should it appear that way here too?

[strobe_, 2012/07/19 02:43:35]

Timestamp is the more important field, since DecodeTimestamp is only required
due to an implementation detail of the source buffer and may be removed at a
later time. However, 'dts' is monotonically increasing, making output traces
easier to read if it is first since subsequent lines don't flick back and forth
as much due to uneven field size.

-           << ", size=" << runs_.size();
+           << ", size=" << runs_.sample_size();
 
   if (audio) {
     audio_buffers->push_back(stream_buf);
   } else {
     video_buffers->push_back(stream_buf);
   }
 
   runs_.AdvanceSample();
   return true;
 }
 
 bool MP4StreamParser::SendAndFlushSamples(BufferQueue* audio_buffers,
                                           BufferQueue* video_buffers) {
   if (!audio_buffers->empty()) {
     if (audio_cb_.is_null() || !audio_cb_.Run(*audio_buffers))
       return false;
     audio_buffers->clear();

[ddorwin, 2012/07/17 01:14:21]

Don't clear the buffers if there is no cb or it fails? When do they get cleared?

[strobe_, 2012/07/19 02:43:35]

Fixed.

[ddorwin, 2012/07/19 06:11:10]

Where?

[strobe_, 2012/07/19 16:55:02]

Apparently the fix was also lost in the failed rebase, but on consideration the
fix wasn't even necessary: these objects are allocated on the stack in
MP4StreamParser::Parse.

[ddorwin, 2012/07/24 01:00:09]

I don't understand. Are you intentionally keeping them around in the case of no
callback or a failed Run()? If so, that should probably be a comment somewhere.
If not, I don't understand why you wouldn't always clear them.

[strobe_, 2012/07/25 01:05:13]

Done.

   }
   if (!video_buffers->empty()) {
     if (video_cb_.is_null() || !video_cb_.Run(*video_buffers))
       return false;
     video_buffers->clear();
   }
   return true;
 }
 
 bool MP4StreamParser::ReadMDATsUntil(const int64 tgt_offset) {
@@ skipping 21 matching lines @@
   return true;
 }
 
 void MP4StreamParser::ChangeState(State new_state) {
   DVLOG(2) << "Changing state: " << new_state;
   state_ = new_state;
 }
 
 }  // namespace mp4
 }  // namespace media