Update WebmMuxer for audio component of MediaStream recording.

media/capture/webm_muxer.cc

 // Copyright 2015 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "media/capture/webm_muxer.h"
 
 #include "base/bind.h"
+#include "media/audio/audio_parameters.h"
 #include "media/base/limits.h"
 #include "media/base/video_frame.h"
 #include "ui/gfx/geometry/size.h"
 
 namespace media {
 
+namespace {
+
+// Comments and constants from media/filters/opus_audio_decoder.cc

[mcasas, 2015/10/28 00:42:02]

Please refactor the common constants out of both opus_audio_decoder.cc and
this file, in an new file (e.g. media/filters/opus_extradata.h or similar). It's
OK to
include it from here, but might need .DEPS adaptation -- presubmit will tell.

[ajose, 2015/10/28 23:43:47]

Done.

+enum {
+  // Size of the Opus extra data excluding optional mapping information.
+  OPUS_EXTRADATA_SIZE = 19,
+  // Offset for "OpusHead" (64 bits)
+  OPUS_EXTRADATA_LABEL_OFFSET = 0,
+  // Offset to the Opus version number (8 bits)
+  OPUS_EXTRADATA_VERSION_OFFSET = 8,
+  // Offset to the channel count byte in the Opus extra data (8 bits)
+  OPUS_EXTRADATA_CHANNELS_OFFSET = 9,
+  // Offset to the pre-skip value in the Opus extra data (16 bits)
+  OPUS_EXTRADATA_SKIP_SAMPLES_OFFSET = 10,
+  // Offset to the sampling rate value in the Opus extra data (32 bits)
+  OPUS_EXTRADATA_SAMPLE_RATE_OFFSET = 12,
+  // Offset to the gain value in the Opus extra data (16 bits)
+  OPUS_EXTRADATA_GAIN_OFFSET = 16,
+  // Offset to the channel mapping byte in the Opus extra data (8 bits)
+  OPUS_EXTRADATA_CHANNEL_MAPPING_OFFSET = 18,
+
+  // Extra Data contains a stream map, beyond the always present
+  // |OPUS_EXTRADATA_SIZE| bytes of data. The mapping data contains stream
+  // count, coupling information, and per channel mapping values:
+  //   - Byte 0: Number of streams.
+  //   - Byte 1: Number coupled.
+  //   - Byte 2: Starting at byte 2 are |extra_data->channels| uint8 mapping
+  //             values.
+  OPUS_EXTRADATA_NUM_STREAMS_OFFSET = OPUS_EXTRADATA_SIZE,
+  OPUS_EXTRADATA_NUM_COUPLED_OFFSET = OPUS_EXTRADATA_NUM_STREAMS_OFFSET + 1,
+  OPUS_EXTRADATA_STREAM_MAP_OFFSET = OPUS_EXTRADATA_NUM_STREAMS_OFFSET + 2,
+};
+
+// Opus internal to Vorbis channel order mapping written in the header.
+static const uint8_t opus_vorbis_channel_map[8][8] = {
+    {0},
+    {0, 1},
+    {0, 2, 1},
+    {0, 1, 2, 3},
+    {0, 4, 1, 2, 3},
+    {0, 4, 1, 2, 3, 5},
+    {0, 4, 1, 2, 3, 5, 6},
+    {0, 6, 1, 2, 3, 4, 5, 7},
+};
+
+void WriteOpusHeader(const media::AudioParameters& params, uint8* header) {
+  // See https://wiki.xiph.org/OggOpus#ID_Header.
+  // Set magic signature.
+  std::string label = "OpusHead";
+  memcpy(header + OPUS_EXTRADATA_LABEL_OFFSET, &label, label.size());
+  // Set Opus version.
+  header[OPUS_EXTRADATA_VERSION_OFFSET] = 1;
+  // Set channel count.
+  header[OPUS_EXTRADATA_CHANNELS_OFFSET] = params.channels();
+  // Set pre-skip
+  uint16 skip = 0;
+  memcpy(header + OPUS_EXTRADATA_SKIP_SAMPLES_OFFSET, &skip, sizeof(uint16));
+  // Set original input sample rate in Hz.
+  uint32 sample_rate = params.sample_rate();
+  memcpy(header + OPUS_EXTRADATA_SAMPLE_RATE_OFFSET, &sample_rate,
+         sizeof(uint32));
+  // Set output gain in dB.
+  uint16 gain = 0;
+  memcpy(header + OPUS_EXTRADATA_GAIN_OFFSET, &gain, 2);
+
+  // Set channel mapping.
+  if (params.channels() > 2) {
+    // Also possible to have a multistream, not supported for now.
+    DCHECK_LE(params.channels(), 8);

[mcasas, 2015/10/28 00:42:02]

Extract |8| here and in l.49 to a constant?

[ajose, 2015/10/28 23:43:47]

Done.

+    header[OPUS_EXTRADATA_CHANNEL_MAPPING_OFFSET] = 1;
+    // Assuming no coupled streams. This should actually be
+    // channels() - |coupled_streams|.
+    header[OPUS_EXTRADATA_NUM_STREAMS_OFFSET] = params.channels();
+    header[OPUS_EXTRADATA_NUM_COUPLED_OFFSET] = 0;
+    // Set the actual stream map.
+    memcpy(header + OPUS_EXTRADATA_STREAM_MAP_OFFSET,
+           opus_vorbis_channel_map[params.channels() - 1], params.channels());
+  } else {
+    header[OPUS_EXTRADATA_CHANNEL_MAPPING_OFFSET] = 0;
+  }
+}
+
 static double GetFrameRate(const scoped_refptr<VideoFrame>& video_frame) {
   const double kZeroFrameRate = 0.0;
   const double kDefaultFrameRate = 30.0;
 
   double frame_rate = kDefaultFrameRate;
   if (!video_frame->metadata()->GetDouble(
           VideoFrameMetadata::FRAME_RATE, &frame_rate) ||
       frame_rate <= kZeroFrameRate ||
       frame_rate > media::limits::kMaxFramesPerSecond) {
     frame_rate = kDefaultFrameRate;
   }
   return frame_rate;
 }
 
-WebmMuxer::WebmMuxer(VideoCodec codec, const WriteDataCB& write_data_callback)
+}  // anonymous namespace
+
+WebmMuxer::WebmMuxer(VideoCodec codec,
+                     size_t num_video_tracks,
+                     size_t num_audio_tracks,
+                     const WriteDataCB& write_data_callback)
     : use_vp9_(codec == kCodecVP9),
-      track_index_(0),
+      video_track_index_(0),
+      audio_track_index_(0),
+      has_video_(num_video_tracks > 0),
+      has_audio_(num_audio_tracks > 0),
       write_data_callback_(write_data_callback),
       position_(0) {
   DCHECK(!write_data_callback_.is_null());
   DCHECK(codec == kCodecVP8 || codec == kCodecVP9)
       << " Only Vp8 and VP9 are supported in WebmMuxer";
+

[mcasas, 2015/10/28 00:42:02]

Consider adding DCHECK(has_video_ || has_audio_);

[ajose, 2015/10/28 23:43:47]

Done.

+  segment_.Init(this);
+  segment_.set_mode(mkvmuxer::Segment::kLive);
+  segment_.OutputCues(false);
+
+  mkvmuxer::SegmentInfo* const info = segment_.GetSegmentInfo();
+  info->set_writing_app("Chrome");
+  info->set_muxing_app("Chrome");
+
   // Creation is done on a different thread than main activities.
   thread_checker_.DetachFromThread();
 }
 
 WebmMuxer::~WebmMuxer() {
   // No need to segment_.Finalize() since is not Seekable(), i.e. a live
   // stream, but is a good practice.
   DCHECK(thread_checker_.CalledOnValidThread());
   segment_.Finalize();
 }
 
 void WebmMuxer::OnEncodedVideo(const scoped_refptr<VideoFrame>& video_frame,
                                scoped_ptr<std::string> encoded_data,
                                base::TimeTicks timestamp,
                                bool is_key_frame) {
   DVLOG(1) << __FUNCTION__ << " - " << encoded_data->size() << "B";
   DCHECK(thread_checker_.CalledOnValidThread());
-  if (!track_index_) {
+
+  if (!video_track_index_) {
     // |track_index_|, cannot be zero (!), initialize WebmMuxer in that case.
     // http://www.matroska.org/technical/specs/index.html#Tracks
     AddVideoTrack(video_frame->visible_rect().size(),
                   GetFrameRate(video_frame));
-    first_frame_timestamp_ = timestamp;
+    if (first_frame_timestamp_ == base::TimeTicks())

[mcasas, 2015/10/28 00:42:02]

I'd say 
  if (first_frame_timestamp_.is_null()) 

[ajose, 2015/10/28 23:43:47]

Done.

+      first_frame_timestamp_ = timestamp;
   }
+
+  // TODO(ajose): Don't drop data. http://crbug.com/547948
+  if (has_audio_ && !audio_track_index_) {
+    DVLOG(1) << __FUNCTION__ << ": delaying until audio track ready.";
+    return;
+  }
+
+  most_recent_timestamp_ =
+      std::max(most_recent_timestamp_, timestamp - first_frame_timestamp_);
   segment_.AddFrame(reinterpret_cast<const uint8_t*>(encoded_data->data()),
-                    encoded_data->size(),
-                    track_index_,
-                    (timestamp - first_frame_timestamp_).InMicroseconds() *
+                    encoded_data->size(), video_track_index_,
+                    most_recent_timestamp_.InMicroseconds() *
                         base::Time::kNanosecondsPerMicrosecond,
                     is_key_frame);
 }
 
+void WebmMuxer::OnEncodedAudio(const media::AudioParameters& params,
+                               scoped_ptr<std::string> encoded_data,
+                               base::TimeTicks timestamp) {
+  DVLOG(1) << __FUNCTION__ << " - " << encoded_data->size() << "B";
+  DCHECK(thread_checker_.CalledOnValidThread());
+
+  if (!audio_track_index_) {
+    AddAudioTrack(params);
+    if (first_frame_timestamp_ == base::TimeTicks())
+      first_frame_timestamp_ = timestamp;
+  }
+
+  // TODO(ajose): Don't drop data. http://crbug.com/547948
+  if (has_video_ && !video_track_index_) {
+    DVLOG(1) << __FUNCTION__ << ": delaying until video track ready.";
+    return;
+  }
+
+  most_recent_timestamp_ =
+      std::max(most_recent_timestamp_, timestamp - first_frame_timestamp_);
+  segment_.AddFrame(reinterpret_cast<const uint8_t*>(encoded_data->data()),
+                    encoded_data->size(), audio_track_index_,
+                    most_recent_timestamp_.InMicroseconds() *
+                        base::Time::kNanosecondsPerMicrosecond,
+                    true /* is_key_frame -- always true for audio */);
+}
+
 void WebmMuxer::AddVideoTrack(const gfx::Size& frame_size, double frame_rate) {
   DCHECK(thread_checker_.CalledOnValidThread());
-  DCHECK_EQ(track_index_, 0u) << "WebmMuxer can only be initialised once.";
+  DCHECK_EQ(video_track_index_, 0u)

[mcasas, 2015/10/28 00:42:02]

nit: expected value goes first, DCHECK_EQ(0u, video_track_index_)

[ajose, 2015/10/28 23:43:47]

Done.

+      << "WebmMuxer can only be initialized once.";
 
-  segment_.Init(this);
-  segment_.set_mode(mkvmuxer::Segment::kLive);
-  segment_.OutputCues(false);
-
-  mkvmuxer::SegmentInfo* const info = segment_.GetSegmentInfo();
-  info->set_writing_app("Chrome");
-  info->set_muxing_app("Chrome");
-
-  track_index_ =
+  video_track_index_ =
       segment_.AddVideoTrack(frame_size.width(), frame_size.height(), 0);
-  DCHECK_GT(track_index_, 0u);
+  DCHECK_GT(video_track_index_, 0u);
 
   mkvmuxer::VideoTrack* const video_track =
       reinterpret_cast<mkvmuxer::VideoTrack*>(
-          segment_.GetTrackByNumber(track_index_));
+          segment_.GetTrackByNumber(video_track_index_));
   DCHECK(video_track);
   video_track->set_codec_id(use_vp9_ ? mkvmuxer::Tracks::kVp9CodecId
                                      : mkvmuxer::Tracks::kVp8CodecId);
   DCHECK_EQ(video_track->crop_right(), 0ull);
   DCHECK_EQ(video_track->crop_left(), 0ull);
   DCHECK_EQ(video_track->crop_top(), 0ull);
   DCHECK_EQ(video_track->crop_bottom(), 0ull);
 
   video_track->set_frame_rate(frame_rate);
   video_track->set_default_duration(base::Time::kNanosecondsPerSecond /
                                     frame_rate);
   // Segment's timestamps should be in milliseconds, DCHECK it. See
   // http://www.webmproject.org/docs/container/#muxer-guidelines
   DCHECK_EQ(segment_.GetSegmentInfo()->timecode_scale(), 1000000ull);
 }
 
+void WebmMuxer::AddAudioTrack(const media::AudioParameters& params) {
+  DCHECK(thread_checker_.CalledOnValidThread());
+  DCHECK_EQ(audio_track_index_, 0u)
+      << "WebmMuxer audio can only be initialised once.";
+
+  audio_track_index_ =
+      segment_.AddAudioTrack(params.sample_rate(), params.channels(), 0);
+  DCHECK_GT(audio_track_index_, 0u);
+
+  mkvmuxer::AudioTrack* const audio_track =
+      reinterpret_cast<mkvmuxer::AudioTrack*>(
+          segment_.GetTrackByNumber(audio_track_index_));
+  DCHECK(audio_track);
+  audio_track->set_codec_id(mkvmuxer::Tracks::kOpusCodecId);
+
+  // http://wiki.xiph.org/MatroskaOpus
+  audio_track->set_seek_pre_roll(80000000);

[mcasas, 2015/10/28 00:42:02]

Nit: let's define a constant for the magic number, what about:

  // Matroska recommended SeekPreRoll value "number of nanoseconds that must be
  // discarded, for that stream, after a seek until the decoded data is valid to
  // render." http://wiki.xiph.org/MatroskaOpus
  const uint32_t kMatroskaSeekPreRollNs = 80000000;

?

[ajose, 2015/10/28 23:43:47]

Done.

+
+  DCHECK_EQ(audio_track->sample_rate(), params.sample_rate());
+  DCHECK_EQ((int)audio_track->channels(), params.channels());

[mcasas, 2015/10/28 00:42:02]

No C-style casts.

[ajose, 2015/10/28 23:43:47]

Done.

+
+  uint8 opus_header[OPUS_EXTRADATA_SIZE];
+  WriteOpusHeader(params, opus_header);
+
+  if (!audio_track->SetCodecPrivate(opus_header, OPUS_EXTRADATA_SIZE)) {
+    LOG(ERROR) << __FUNCTION__ << ": failed to set opus header.";
+  }

[mcasas, 2015/10/28 00:42:02]

nit: no need for  {}

[ajose, 2015/10/28 23:43:47]

Done.

+
+  // Segment's timestamps should be in milliseconds, DCHECK it. See
+  // http://www.webmproject.org/docs/container/#muxer-guidelines
+  DCHECK_EQ(segment_.GetSegmentInfo()->timecode_scale(), 1000000ull);
+}
+
 mkvmuxer::int32 WebmMuxer::Write(const void* buf, mkvmuxer::uint32 len) {
   DCHECK(thread_checker_.CalledOnValidThread());
   DCHECK(buf);
   write_data_callback_.Run(
       base::StringPiece(reinterpret_cast<const char*>(buf), len));
   position_ += len;
   return 0;
 }
 
 mkvmuxer::int64 WebmMuxer::Position() const {
@@ skipping 10 matching lines @@
 }
 
 void WebmMuxer::ElementStartNotify(mkvmuxer::uint64 element_id,
                                    mkvmuxer::int64 position) {
   // This method gets pinged before items are sent to |write_data_callback_|.
   DCHECK_GE(position, position_.ValueOrDefault(0))
       << "Can't go back in a live WebM stream.";
 }
 
 }  // namespace media