Add AudioTrackRecorder for audio component of MediaStream recording.

content/renderer/media/audio_track_recorder.cc

+// Copyright 2015 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "base/bind.h"
+#include "base/stl_util.h"
+#include "base/threading/thread.h"
+#include "content/renderer/media/audio_track_recorder.h"
+#include "media/audio/audio_parameters.h"
+#include "media/base/bind_to_current_loop.h"
+
+#include "third_party/opus/src/include/opus.h"
+
+namespace content {
+
+namespace {
+
+enum {
+  // TODO(ajose): This likely shouldn't be hardcoded.
+  DEFAULT_FRAMES_PER_SECOND = 100,
+
+  // This is the recommended value, according to documentation in
+  // third_party/opus/src/include/opus.h, so that the Opus encoder does not
+  // degrade the audio due to memory constraints.
+  OPUS_MAX_PAYLOAD_SIZE = 4000,
+
+  // Support for max sampling rate of 48KHz, 2 channels, 100 ms duration.
+  MAX_SAMPLES_x_CHANNELS_PER_FRAME = 48 * 2 * 100,
+};
+
+static bool IsValidOpusFrameDuration(base::TimeDelta duration) {
+  // See https://tools.ietf.org/html/rfc6716#section-2.1.4
+  return duration == base::TimeDelta::FromMicroseconds(2500) ||
+         duration == base::TimeDelta::FromMilliseconds(5) ||
+         duration == base::TimeDelta::FromMilliseconds(10) ||
+         duration == base::TimeDelta::FromMilliseconds(20) ||
+         duration == base::TimeDelta::FromMilliseconds(40) ||
+         duration == base::TimeDelta::FromMilliseconds(60);
+}
+
+}  // anonymous namespace
+
+// Nested class encapsulating opus-related encoding details.
+// AudioEncoder is created and destroyed on ATR's main thread (usually the
+// main render thread) but otherwise should operate entirely on
+// |encoder_thread_|, which is owned by AudioTrackRecorder.
+// Be sure to delete |encoder_thread_| before deleting the AudioEncoder using
+// it.
+class AudioTrackRecorder::AudioEncoder
+    : public base::RefCountedThreadSafe<AudioEncoder> {
+ public:
+  explicit AudioEncoder(const OnEncodedAudioCB& on_encoded_audio_cb)
+      : initialized_(false),
+        on_encoded_audio_cb_(on_encoded_audio_cb),
+        opus_encoder_(nullptr) {
+    // AudioEncoder is constructed on the thread that ATR lives on, but should
+    // operate only on the encoder thread after that. Reset
+    // |encoder_thread_checker_| here, as the next call to CalledOnValidThread()
+    // will be from the encoder thread.
+    encoder_thread_checker_.DetachFromThread();
+  }
+
+  void OnSetFormat(const media::AudioParameters& params);
+
+  void EncodeAudio(scoped_ptr<media::AudioBus> audio_bus,
+                   const base::TimeTicks& capture_time);
+
+ private:
+  friend class base::RefCountedThreadSafe<AudioEncoder>;
+
+  ~AudioEncoder();
+
+  bool IsInitialized() { return initialized_; }

[miu, 2015/10/26 22:41:04]

You don't need the |initialized_| bool.  Instead, just return !!opus_encoder_;

[ajose, 2015/10/27 18:41:28]

Done.

+
+  bool InitOpus(const media::AudioParameters& params);
+  void DestroyOpus();
+
+  void TransferSamplesIntoBuffer(const media::AudioBus* audio_bus,
+                                 int source_offset,
+                                 int buffer_fill_offset,
+                                 int num_samples);
+  bool EncodeFromFilledBuffer(std::string* out);
+
+  bool initialized_;
+
+  int samples_per_frame_;
+  const OnEncodedAudioCB on_encoded_audio_cb_;
+
+  base::ThreadChecker encoder_thread_checker_;
+
+  // In the case where a call to EncodeAudio() cannot completely fill the
+  // buffer, this points to the position at which to populate data in a later
+  // call.
+  int buffer_fill_end_;
+
+  media::AudioParameters audio_params_;
+
+  // Buffer for passing AudioBus data to OpusEncoder.
+  scoped_ptr<float[]> buffer_;
+
+  OpusEncoder* opus_encoder_;
+
+  DISALLOW_COPY_AND_ASSIGN(AudioEncoder);
+};
+
+AudioTrackRecorder::AudioEncoder::~AudioEncoder() {
+  // We don't DCHECK that we're on the encoder thread here, as it should have
+  // already been deleted at this point.
+  DestroyOpus();
+}
+
+void AudioTrackRecorder::AudioEncoder::OnSetFormat(
+    const media::AudioParameters& params) {
+  DCHECK(encoder_thread_checker_.CalledOnValidThread());
+  InitOpus(params);

[miu, 2015/10/26 22:41:04]

The return value is being ignored here.  Looking at the code some more, it seems
like you don't need a separate InitOpus() method.  Just put all that code into
this method.

[ajose, 2015/10/27 18:41:28]

Done.

+}
+
+void AudioTrackRecorder::AudioEncoder::EncodeAudio(
+    scoped_ptr<media::AudioBus> audio_bus,
+    const base::TimeTicks& capture_time) {
+  DCHECK(encoder_thread_checker_.CalledOnValidThread());
+  DCHECK(IsInitialized());
+  DCHECK_EQ(audio_bus->channels(), audio_params_.channels());
+  DCHECK_EQ(audio_bus->frames(), audio_params_.frames_per_buffer());
+
+  // Encode all audio in |audio_bus| into zero or more frames.
+  int src_pos = 0;
+  while (src_pos < audio_bus->frames()) {
+    const int num_samples_to_xfer = std::min(
+        samples_per_frame_ - buffer_fill_end_, audio_bus->frames() - src_pos);
+    TransferSamplesIntoBuffer(audio_bus.get(), src_pos, buffer_fill_end_,
+                              num_samples_to_xfer);
+    src_pos += num_samples_to_xfer;
+    buffer_fill_end_ += num_samples_to_xfer;
+
+    if (buffer_fill_end_ < samples_per_frame_)
+      break;
+
+    scoped_ptr<std::string> encoded_data(new std::string());
+    if (EncodeFromFilledBuffer(encoded_data.get())) {
+      on_encoded_audio_cb_.Run(audio_params_, encoded_data.Pass(),
+                               capture_time);
+    }
+
+    // Reset the internal buffer for the next frame.
+    buffer_fill_end_ = 0;
+  }
+}
+
+bool AudioTrackRecorder::AudioEncoder::InitOpus(
+    const media::AudioParameters& params) {
+  DCHECK(encoder_thread_checker_.CalledOnValidThread());
+  if (audio_params_.Equals(params))
+    return true;
+
+  if (!params.IsValid()) {
+    DLOG(ERROR) << "Invalid audio params: " << params.AsHumanReadableString();
+    return false;
+  }
+
+  if (params.bits_per_sample() != 16) {

[miu, 2015/10/26 22:41:04]

IMO, this is irrelevant.  The AudioBuses always provide floats which,
technically, are quasi-32 bits per sample.

[ajose, 2015/10/27 18:41:28]

Done.

+    DLOG(ERROR) << "Invalid bits per sample: " << params.bits_per_sample();
+    return false;
+  }
+
+  samples_per_frame_ = params.sample_rate() / DEFAULT_FRAMES_PER_SECOND;
+  if (samples_per_frame_ <= 0 ||
+      params.sample_rate() % samples_per_frame_ != 0 ||
+      samples_per_frame_ * params.channels() >
+          MAX_SAMPLES_x_CHANNELS_PER_FRAME) {
+    DLOG(ERROR) << "Invalid |samples_per_frame_|: " << samples_per_frame_;
+    return false;
+  }
+
+  const base::TimeDelta frame_duration = base::TimeDelta::FromMicroseconds(
+      base::Time::kMicrosecondsPerSecond * samples_per_frame_ /
+      params.sample_rate());
+  if (frame_duration == base::TimeDelta() ||

[miu, 2015/10/26 22:41:04]

No need for zero frame_duration check since IsValidOpusFrameDuration() will
return false in this case.

[ajose, 2015/10/27 18:41:27]

Done.

+      !IsValidOpusFrameDuration(frame_duration)) {
+    DLOG(ERROR) << "Invalid |frame_duration|: " << frame_duration;
+    return false;
+  }
+
+  // Initialize AudioBus buffer for OpusEncoder.
+  buffer_fill_end_ = 0;
+  buffer_.reset(new float[params.channels() * samples_per_frame_]);
+
+  // Check for and destroy previous OpusEncoder, if necessary.
+  DestroyOpus();

[miu, 2015/10/26 22:41:04]

This should go at the top of the method, before any of the "early return"
statements.

[ajose, 2015/10/27 18:41:28]

Done.

+  // Initialize OpusEncoder.
+  int opus_result;
+  opus_encoder_ = opus_encoder_create(params.sample_rate(), params.channels(),
+                                      OPUS_APPLICATION_AUDIO, &opus_result);
+  if (opus_result < 0) {
+    DLOG(ERROR) << "Couldn't init opus encoder: " << opus_strerror(opus_result);
+    return false;
+  }
+
+  // Note: As of 2013-10-31, the encoder in "auto bitrate" mode would use a
+  // variable bitrate up to 102kbps for 2-channel, 48 kHz audio and a 10 ms
+  // frame size. The opus library authors may, of course, adjust this in
+  // later versions.
+  if (opus_encoder_ctl(opus_encoder_, OPUS_SET_BITRATE(OPUS_AUTO)) != OPUS_OK) {
+    DLOG(ERROR) << "Failed to set opus bitrate.";
+    return false;
+  }
+
+  audio_params_ = params;
+  initialized_ = true;
+  return true;
+}
+
+void AudioTrackRecorder::AudioEncoder::DestroyOpus() {
+  // We don't DCHECK that we're on the encoder thread here, as this could be
+  // called from the dtor (main thread) or from OnSetForamt (render thread);
+  if (opus_encoder_) {
+    opus_encoder_destroy(opus_encoder_);
+    opus_encoder_ = nullptr;
+  }
+}
+
+void AudioTrackRecorder::AudioEncoder::TransferSamplesIntoBuffer(
+    const media::AudioBus* audio_bus,
+    int source_offset,
+    int buffer_fill_offset,
+    int num_samples) {
+  // TODO(ajose): Replace with AudioBus::ToInterleaved()? crbug.com/547918
+  DCHECK(encoder_thread_checker_.CalledOnValidThread());
+  DCHECK(IsInitialized());
+  // Opus requires channel-interleaved samples in a single array.
+  for (int ch = 0; ch < audio_bus->channels(); ++ch) {
+    const float* src = audio_bus->channel(ch) + source_offset;
+    const float* const src_end = src + num_samples;
+    float* dest =
+        buffer_.get() + buffer_fill_offset * audio_params_.channels() + ch;
+    for (; src < src_end; ++src, dest += audio_params_.channels())
+      *dest = *src;
+  }
+}
+
+bool AudioTrackRecorder::AudioEncoder::EncodeFromFilledBuffer(
+    std::string* out) {
+  DCHECK(encoder_thread_checker_.CalledOnValidThread());
+  DCHECK(IsInitialized());
+
+  out->resize(OPUS_MAX_PAYLOAD_SIZE);
+  const opus_int32 result = opus_encode_float(
+      opus_encoder_, buffer_.get(), samples_per_frame_,
+      reinterpret_cast<uint8*>(string_as_array(out)), OPUS_MAX_PAYLOAD_SIZE);
+  if (result > 1) {
+    // TODO(ajose): Improve this. crbug.com/547918
+    out->resize(result);
+    return true;
+  }
+  // If |result| in {0,1}, do nothing; the documentation says that a return
+  // value of zero or one means the packet does not need to be transmitted.
+  // Otherwise, we have an error.
+  DLOG_IF(ERROR, result < 0) << __FUNCTION__
+                             << " failed: " << opus_strerror(result);
+  return false;
+}
+
+AudioTrackRecorder::AudioTrackRecorder(
+    const blink::WebMediaStreamTrack& track,
+    const OnEncodedAudioCB& on_encoded_audio_cb)
+    : track_(track),
+      encoder_(new AudioEncoder(media::BindToCurrentLoop(on_encoded_audio_cb))),
+      encoder_thread_(new base::Thread("AudioEncoderThread")) {
+  DCHECK(main_render_thread_checker_.CalledOnValidThread());
+  DCHECK(!track_.isNull());
+  DCHECK(track_.extraData());
+
+  // Start the |encoder_thread_|. From this point on, |encoder_| should work
+  // only on |encoder_thread_|, as enforced by DCHECKs.
+  DCHECK(!encoder_thread_->IsRunning());
+  encoder_thread_->Start();
+
+  // Connect the source provider to the track as a sink.
+  MediaStreamAudioSink::AddToAudioTrack(this, track_);
+}
+
+AudioTrackRecorder::~AudioTrackRecorder() {
+  DCHECK(main_render_thread_checker_.CalledOnValidThread());
+  MediaStreamAudioSink::RemoveFromAudioTrack(this, track_);
+}
+
+void AudioTrackRecorder::OnSetFormat(const media::AudioParameters& params) {
+  DCHECK(encoder_thread_->IsRunning());
+  // If the source is restarted, might have changed to another capture thread.
+  capture_thread_checker_.DetachFromThread();
+  DCHECK(capture_thread_checker_.CalledOnValidThread());
+
+  encoder_thread_->task_runner()->PostTask(
+      FROM_HERE, base::Bind(&AudioEncoder::OnSetFormat, encoder_, params));
+}
+
+void AudioTrackRecorder::OnData(const media::AudioBus& audio_bus,
+                                base::TimeTicks capture_time) {
+  DCHECK(encoder_thread_->IsRunning());
+  DCHECK(capture_thread_checker_.CalledOnValidThread());
+  DCHECK(!capture_time.is_null());
+
+  // TODO(ajose): When will audio_bus be deleted?

[miu, 2015/10/26 22:41:04]

Looks like the closure (on line 310) or the AudioEncoder::EncodeAudio method
will delete it automatically when the scoped_ptr<> goes out-of-scope.

+  scoped_ptr<media::AudioBus> audio_data =
+      media::AudioBus::Create(audio_bus.channels(), audio_bus.frames());
+  audio_bus.CopyTo(audio_data.get());
+
+  encoder_thread_->task_runner()->PostTask(
+      FROM_HERE, base::Bind(&AudioEncoder::EncodeAudio, encoder_,
+                            base::Passed(&audio_data), capture_time));
+}
+
+}  // namespace content