Add AudioTrackRecorder for audio component of MediaStream recording.

content/renderer/media/audio_track_recorder.cc

+// Copyright 2015 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "base/bind.h"
+#include "base/stl_util.h"
+#include "content/renderer/media/audio_track_recorder.h"
+#include "media/audio/audio_parameters.h"
+#include "media/base/audio_bus.h"
+#include "media/base/bind_to_current_loop.h"
+
+#include "third_party/opus/src/include/opus.h"
+
+namespace content {
+
+namespace {
+
+enum {
+  // TODO(ajose): This likely shouldn't be hardcoded.
+  DEFAULT_FRAMES_PER_SECOND = 100,

[miu, 2015/10/29 01:26:05]

Actually, the purpose of this is to define the duration of one frame for the
purposes of Opus encoding.  It doesn't really depend on anything external.  I
suggest you just hard-code the Opus frame duration to 10 ms (with caveat; see
comment below).

[ajose, 2015/10/29 21:11:23]

Done.

+
+  // This is the recommended value, according to documentation in
+  // third_party/opus/src/include/opus.h, so that the Opus encoder does not
+  // degrade the audio due to memory constraints.
+  OPUS_MAX_PAYLOAD_SIZE = 4000,
+
+  // Support for max sampling rate of 48KHz, 2 channels, 100 ms duration.
+  MAX_SAMPLES_x_CHANNELS_PER_FRAME = 48 * 2 * 100,
+};
+
+static bool IsValidOpusFrameDuration(base::TimeDelta duration) {

[miu, 2015/10/29 01:26:05]

This function isn't necessary.  Just fix the Opus frame duration to 10 ms.

[ajose, 2015/10/29 21:11:23]

Done.

+  // See https://tools.ietf.org/html/rfc6716#section-2.1.4
+  return duration == base::TimeDelta::FromMicroseconds(2500) ||
+         duration == base::TimeDelta::FromMilliseconds(5) ||
+         duration == base::TimeDelta::FromMilliseconds(10) ||
+         duration == base::TimeDelta::FromMilliseconds(20) ||
+         duration == base::TimeDelta::FromMilliseconds(40) ||
+         duration == base::TimeDelta::FromMilliseconds(60);
+}
+
+}  // anonymous namespace
+
+// Nested class encapsulating opus-related encoding details.
+// AudioEncoder is created and destroyed on ATR's main thread (usually the
+// main render thread) but otherwise should operate entirely on
+// |encoder_thread_|, which is owned by AudioTrackRecorder.
+// Be sure to delete |encoder_thread_| before deleting the AudioEncoder using
+// it.
+class AudioTrackRecorder::AudioEncoder
+    : public base::RefCountedThreadSafe<AudioEncoder> {
+ public:
+  explicit AudioEncoder(const OnEncodedAudioCB& on_encoded_audio_cb)
+      : on_encoded_audio_cb_(on_encoded_audio_cb), opus_encoder_(nullptr) {
+    // AudioEncoder is constructed on the thread that ATR lives on, but should
+    // operate only on the encoder thread after that. Reset
+    // |encoder_thread_checker_| here, as the next call to CalledOnValidThread()
+    // will be from the encoder thread.
+    encoder_thread_checker_.DetachFromThread();
+  }
+
+  void OnSetFormat(const media::AudioParameters& params);
+
+  void EncodeAudio(scoped_ptr<media::AudioBus> audio_bus,
+                   const base::TimeTicks& capture_time);
+
+ private:
+  friend class base::RefCountedThreadSafe<AudioEncoder>;
+
+  ~AudioEncoder();
+
+  bool IsInitialized() { return !!opus_encoder_; }

[miu, 2015/10/29 01:26:05]

style: Inline accessors are in lower case, and this one should be const:

  bool is_initialized() const { return !!opus_encoder_; }

[ajose, 2015/10/29 21:11:23]

Done.

+
+  void DestroyOpus();
+
+  void TransferSamplesIntoBuffer(const media::AudioBus* audio_bus,
+                                 int source_offset,
+                                 int buffer_fill_offset,
+                                 int num_samples);
+  bool EncodeFromFilledBuffer(std::string* out);
+
+  int samples_per_frame_;
+  const OnEncodedAudioCB on_encoded_audio_cb_;
+
+  base::ThreadChecker encoder_thread_checker_;
+
+  // In the case where a call to EncodeAudio() cannot completely fill the
+  // buffer, this points to the position at which to populate data in a later
+  // call.
+  int buffer_fill_end_;
+
+  media::AudioParameters audio_params_;
+
+  // Buffer for passing AudioBus data to OpusEncoder.
+  scoped_ptr<float[]> buffer_;
+
+  OpusEncoder* opus_encoder_;
+
+  DISALLOW_COPY_AND_ASSIGN(AudioEncoder);
+};
+
+AudioTrackRecorder::AudioEncoder::~AudioEncoder() {
+  // We don't DCHECK that we're on the encoder thread here, as it should have
+  // already been deleted at this point.
+  DestroyOpus();
+}
+
+void AudioTrackRecorder::AudioEncoder::OnSetFormat(
+    const media::AudioParameters& params) {
+  DCHECK(encoder_thread_checker_.CalledOnValidThread());
+  if (audio_params_.Equals(params))
+    return;
+
+  // Check for and destroy previous OpusEncoder, if necessary.
+  DestroyOpus();
+
+  if (!params.IsValid()) {
+    DLOG(ERROR) << "Invalid audio params: " << params.AsHumanReadableString();
+    return;
+  }
+
+  samples_per_frame_ = params.sample_rate() / DEFAULT_FRAMES_PER_SECOND;
+  if (samples_per_frame_ <= 0 ||

[miu, 2015/10/29 01:26:05]

Hmm...If you were to hard-code frame durations to 10 ms, this check here would
only error-out on a few of the possible sample rates (e.g., 22,050 Hz).  So, you
may need to have some code that tries to use 10 ms frame durations and checks
that the sample rate modulo 100 is zero.  If it is not, it should try
falling-back to 20 ms frame durations (where the sample rate modulo 50 is zero).

Note:

  100 FPS = 1000 ms / 10 ms
  50 FPS = 1000 ms / 20 ms

(BTW--If you want to be really pedantic, you could also use 40 ms and 60 ms as
additional "fall-back" frame durations, since Opus supports them.)

[ajose, 2015/10/29 21:11:23]

Done.

+      params.sample_rate() % samples_per_frame_ != 0 ||
+      samples_per_frame_ * params.channels() >
+          MAX_SAMPLES_x_CHANNELS_PER_FRAME) {
+    DLOG(ERROR) << "Invalid |samples_per_frame_|: " << samples_per_frame_;
+    return;
+  }
+
+  const base::TimeDelta frame_duration = base::TimeDelta::FromMicroseconds(
+      base::Time::kMicrosecondsPerSecond * samples_per_frame_ /
+      params.sample_rate());
+  if (!IsValidOpusFrameDuration(frame_duration)) {
+    DLOG(ERROR) << "Invalid |frame_duration|: " << frame_duration;
+    return;
+  }
+
+  // Initialize AudioBus buffer for OpusEncoder.
+  buffer_fill_end_ = 0;
+  buffer_.reset(new float[params.channels() * samples_per_frame_]);
+
+  // Initialize OpusEncoder.
+  int opus_result;
+  opus_encoder_ = opus_encoder_create(params.sample_rate(), params.channels(),
+                                      OPUS_APPLICATION_AUDIO, &opus_result);
+  if (opus_result < 0) {
+    DLOG(ERROR) << "Couldn't init opus encoder: " << opus_strerror(opus_result);
+    return;
+  }
+
+  // Note: As of 2013-10-31, the encoder in "auto bitrate" mode would use a
+  // variable bitrate up to 102kbps for 2-channel, 48 kHz audio and a 10 ms
+  // frame size. The opus library authors may, of course, adjust this in
+  // later versions.
+  if (opus_encoder_ctl(opus_encoder_, OPUS_SET_BITRATE(OPUS_AUTO)) != OPUS_OK) {
+    DLOG(ERROR) << "Failed to set opus bitrate.";
+    return;
+  }
+
+  audio_params_ = params;
+}
+
+void AudioTrackRecorder::AudioEncoder::EncodeAudio(
+    scoped_ptr<media::AudioBus> audio_bus,
+    const base::TimeTicks& capture_time) {
+  DCHECK(encoder_thread_checker_.CalledOnValidThread());
+  DCHECK_EQ(audio_bus->channels(), audio_params_.channels());
+  DCHECK_EQ(audio_bus->frames(), audio_params_.frames_per_buffer());
+
+  if (!IsInitialized())
+    return;
+
+  // Encode all audio in |audio_bus| into zero or more frames.
+  int src_pos = 0;
+  while (src_pos < audio_bus->frames()) {
+    const int num_samples_to_xfer = std::min(
+        samples_per_frame_ - buffer_fill_end_, audio_bus->frames() - src_pos);
+    TransferSamplesIntoBuffer(audio_bus.get(), src_pos, buffer_fill_end_,
+                              num_samples_to_xfer);
+    src_pos += num_samples_to_xfer;
+    buffer_fill_end_ += num_samples_to_xfer;
+
+    if (buffer_fill_end_ < samples_per_frame_)
+      break;
+
+    scoped_ptr<std::string> encoded_data(new std::string());
+    if (EncodeFromFilledBuffer(encoded_data.get())) {
+      on_encoded_audio_cb_.Run(audio_params_, encoded_data.Pass(),
+                               capture_time);
+    }
+
+    // Reset the internal buffer for the next frame.
+    buffer_fill_end_ = 0;
+  }
+}
+
+void AudioTrackRecorder::AudioEncoder::DestroyOpus() {
+  // We don't DCHECK that we're on the encoder thread here, as this could be
+  // called from the dtor (main thread) or from OnSetForamt (render thread);
+  if (opus_encoder_) {
+    opus_encoder_destroy(opus_encoder_);
+    opus_encoder_ = nullptr;
+  }
+}
+
+void AudioTrackRecorder::AudioEncoder::TransferSamplesIntoBuffer(
+    const media::AudioBus* audio_bus,
+    int source_offset,
+    int buffer_fill_offset,
+    int num_samples) {
+  // TODO(ajose): Consider replacing with AudioBus::ToInterleaved().
+  // http://crbug.com/547918
+  DCHECK(encoder_thread_checker_.CalledOnValidThread());
+  DCHECK(IsInitialized());
+  // Opus requires channel-interleaved samples in a single array.
+  for (int ch = 0; ch < audio_bus->channels(); ++ch) {
+    const float* src = audio_bus->channel(ch) + source_offset;
+    const float* const src_end = src + num_samples;
+    float* dest =
+        buffer_.get() + buffer_fill_offset * audio_params_.channels() + ch;
+    for (; src < src_end; ++src, dest += audio_params_.channels())
+      *dest = *src;
+  }
+}
+
+bool AudioTrackRecorder::AudioEncoder::EncodeFromFilledBuffer(
+    std::string* out) {
+  DCHECK(encoder_thread_checker_.CalledOnValidThread());
+  DCHECK(IsInitialized());
+
+  out->resize(OPUS_MAX_PAYLOAD_SIZE);
+  const opus_int32 result = opus_encode_float(
+      opus_encoder_, buffer_.get(), samples_per_frame_,
+      reinterpret_cast<uint8*>(string_as_array(out)), OPUS_MAX_PAYLOAD_SIZE);
+  if (result > 1) {
+    // TODO(ajose): Investigate improving this. http://crbug.com/547918
+    out->resize(result);
+    return true;
+  }
+  // If |result| in {0,1}, do nothing; the documentation says that a return
+  // value of zero or one means the packet does not need to be transmitted.
+  // Otherwise, we have an error.
+  DLOG_IF(ERROR, result < 0) << __FUNCTION__
+                             << " failed: " << opus_strerror(result);
+  return false;
+}
+
+AudioTrackRecorder::AudioTrackRecorder(
+    const blink::WebMediaStreamTrack& track,
+    const OnEncodedAudioCB& on_encoded_audio_cb)
+    : track_(track),
+      encoder_(new AudioEncoder(media::BindToCurrentLoop(on_encoded_audio_cb))),
+      encoder_thread_("AudioEncoderThread") {
+  DCHECK(main_render_thread_checker_.CalledOnValidThread());
+  DCHECK(!track_.isNull());
+  DCHECK(track_.extraData());
+
+  // Start the |encoder_thread_|. From this point on, |encoder_| should work
+  // only on |encoder_thread_|, as enforced by DCHECKs.
+  DCHECK(!encoder_thread_.IsRunning());
+  encoder_thread_.Start();
+
+  // Connect the source provider to the track as a sink.
+  MediaStreamAudioSink::AddToAudioTrack(this, track_);
+}
+
+AudioTrackRecorder::~AudioTrackRecorder() {
+  DCHECK(main_render_thread_checker_.CalledOnValidThread());
+  MediaStreamAudioSink::RemoveFromAudioTrack(this, track_);
+}
+
+void AudioTrackRecorder::OnSetFormat(const media::AudioParameters& params) {
+  DCHECK(encoder_thread_.IsRunning());
+  // If the source is restarted, might have changed to another capture thread.
+  capture_thread_checker_.DetachFromThread();
+  DCHECK(capture_thread_checker_.CalledOnValidThread());
+
+  encoder_thread_.task_runner()->PostTask(
+      FROM_HERE, base::Bind(&AudioEncoder::OnSetFormat, encoder_, params));
+}
+
+void AudioTrackRecorder::OnData(const media::AudioBus& audio_bus,
+                                base::TimeTicks capture_time) {
+  DCHECK(encoder_thread_.IsRunning());
+  DCHECK(capture_thread_checker_.CalledOnValidThread());
+  DCHECK(!capture_time.is_null());
+
+  scoped_ptr<media::AudioBus> audio_data =
+      media::AudioBus::Create(audio_bus.channels(), audio_bus.frames());
+  audio_bus.CopyTo(audio_data.get());
+
+  encoder_thread_.task_runner()->PostTask(
+      FROM_HERE, base::Bind(&AudioEncoder::EncodeAudio, encoder_,
+                            base::Passed(&audio_data), capture_time));
+}
+
+}  // namespace content