Add AudioTrackRecorder for audio component of MediaStream recording.

content/renderer/media/audio_track_recorder.cc

+// Copyright 2015 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "base/bind.h"
+#include "base/stl_util.h"
+#include "content/renderer/media/audio_track_recorder.h"

[minyue, 2015/11/11 13:40:10]

I know this should go before all other include as a separate line

[ajose, 2015/11/12 00:10:40]

Done.

+#include "media/audio/audio_parameters.h"
+#include "media/base/audio_bus.h"
+#include "media/base/bind_to_current_loop.h"
+

[minyue, 2015/11/11 13:40:10]

I don't know if there should be a empty line

[ajose, 2015/11/12 00:10:40]

Done.

+#include "third_party/opus/src/include/opus.h"
+
+namespace content {
+
+namespace {
+
+enum {
+  // This is the recommended value, according to documentation in
+  // third_party/opus/src/include/opus.h, so that the Opus encoder does not
+  // degrade the audio due to memory constraints.
+  OPUS_MAX_PAYLOAD_SIZE = 4000,
+
+  // Support for max sampling rate of 48KHz, 2 channels, 100 ms duration.

[minyue, 2015/11/11 13:40:10]

where dose 100 ms duration come from?

[ajose, 2015/11/12 00:10:40]

Lifted this from cast::AudioEncoder, didn't make Opus-specific. Thanks for
catching.

+  MAX_SAMPLES_x_CHANNELS_PER_FRAME = 48 * 2 * 100,
+};
+
+// Returns the Opus frame duration in milliseconds, or zero if none will work
+// for the given |sample_rate|.
+static int GetOpusFrameDuration(int sample_rate) {

[minyue, 2015/11/11 13:40:10]

Like it or not, the naming convention in Chrome media uses

frame to mean one multi-channel sample

See
https://code.google.com/p/chromium/codesearch#chromium/src/media/base/audio_b...

What is then for a trunk of frames? I am not sure. It is usually implicit. Or
you may use num_frames (or frames).

I hope you follow that convention, since audio_track_recorder resides in media.

[ajose, 2015/11/12 00:10:40]

Good point, I had unintentionally been using both meanings of "frame" here. I've
switched the "chunk of samples" definition to being called "buffer" instead
(even though this is a bit misleading in its own way) to be consistent with
media::AudioParameters [1]. Hopefully more consistent now.

[1]
https://code.google.com/p/chromium/codesearch#chromium/src/media/audio/audio_...

+  // Valid frame durations in millseconds. Note there are other valid frame
+  // durations for Opus, see https://tools.ietf.org/html/rfc6716#section-2.1.4
+  const std::vector<int> opus_valid_frame_durations_ms =
+      {10, 20, 40, 60};
+
+  // Search for a duration such that |sample_rate| % |frames_per_second| == 0,
+  // where |frames_per_second| = 1000ms / |possible_duration|.
+  for (auto possible_duration : opus_valid_frame_durations_ms) {

[minyue, 2015/11/11 13:40:10]

This says that if all are valid, we'd use the shortest, right? From compression
point of view, longer frame usually gives better performance. What are we
supposed to optimize for?

[ajose, 2015/11/12 00:10:40]

Switched to descending.

+    if ((sample_rate * possible_duration) % 1000 == 0) {
+      return possible_duration;
+    }
+  }
+
+  // Otherwise, couldn't find a good duration.
+  return 0;
+}
+
+}  // anonymous namespace
+
+// Nested class encapsulating opus-related encoding details.
+// AudioEncoder is created and destroyed on ATR's main thread (usually the
+// main render thread) but otherwise should operate entirely on
+// |encoder_thread_|, which is owned by AudioTrackRecorder.
+// Be sure to delete |encoder_thread_| before deleting the AudioEncoder using
+// it.
+class AudioTrackRecorder::AudioEncoder
+    : public base::RefCountedThreadSafe<AudioEncoder> {
+ public:
+  explicit AudioEncoder(const OnEncodedAudioCB& on_encoded_audio_cb)
+      : on_encoded_audio_cb_(on_encoded_audio_cb), opus_encoder_(nullptr) {
+    // AudioEncoder is constructed on the thread that ATR lives on, but should
+    // operate only on the encoder thread after that. Reset
+    // |encoder_thread_checker_| here, as the next call to CalledOnValidThread()
+    // will be from the encoder thread.
+    encoder_thread_checker_.DetachFromThread();
+  }
+
+  void OnSetFormat(const media::AudioParameters& params);
+
+  void EncodeAudio(scoped_ptr<media::AudioBus> audio_bus,
+                   const base::TimeTicks& capture_time);
+
+ private:
+  friend class base::RefCountedThreadSafe<AudioEncoder>;
+
+  ~AudioEncoder();
+
+  bool is_initialized() const { return !!opus_encoder_; }
+
+  void DestroyOpus();
+
+  void TransferSamplesIntoBuffer(const media::AudioBus* audio_bus,
+                                 int source_offset,
+                                 int buffer_fill_offset,
+                                 int num_samples);
+  bool EncodeFromFilledBuffer(std::string* out);
+
+  int samples_per_frame_;
+  const OnEncodedAudioCB on_encoded_audio_cb_;
+
+  base::ThreadChecker encoder_thread_checker_;
+
+  // In the case where a call to EncodeAudio() cannot completely fill the
+  // buffer, this points to the position at which to populate data in a later
+  // call.
+  int buffer_fill_end_;
+
+  media::AudioParameters audio_params_;
+
+  // Buffer for passing AudioBus data to OpusEncoder.
+  scoped_ptr<float[]> buffer_;
+
+  OpusEncoder* opus_encoder_;
+
+  DISALLOW_COPY_AND_ASSIGN(AudioEncoder);
+};
+
+AudioTrackRecorder::AudioEncoder::~AudioEncoder() {
+  // We don't DCHECK that we're on the encoder thread here, as it should have
+  // already been deleted at this point.
+  DestroyOpus();
+}
+
+void AudioTrackRecorder::AudioEncoder::OnSetFormat(
+    const media::AudioParameters& params) {
+  DCHECK(encoder_thread_checker_.CalledOnValidThread());
+  if (audio_params_.Equals(params))
+    return;
+
+  // Check for and destroy previous OpusEncoder, if necessary.

[minyue, 2015/11/11 13:40:10]

"if necessary" seems unnecessary. We seem to delete the encoder always.

[ajose, 2015/11/12 00:10:40]

Done.

+  DestroyOpus();
+
+  if (!params.IsValid()) {
+    DLOG(ERROR) << "Invalid audio params: " << params.AsHumanReadableString();
+    return;
+  }
+
+  int frame_duration = GetOpusFrameDuration(params.sample_rate());
+  if (frame_duration == 0) {
+    DLOG(ERROR) << "Could not find a valid |frame_duration| for the given "
+                << "sample rate: " << params.sample_rate();
+    return;
+  }
+
+  // This assumes |sample_rate| * |frame_duration| % 1000 == 0, which is true
+  // if |frame_duration| was calculated with GetOpusFrameDuration().
+  samples_per_frame_ = params.sample_rate() * frame_duration / 1000;
+  if (samples_per_frame_ <= 0 ||

[minyue, 2015/11/11 13:40:10]

how can "samples_per_frame_ <= 0" happen?

[ajose, 2015/11/12 00:10:40]

Should be covered by params.IsValid().

+      params.sample_rate() % samples_per_frame_ != 0 ||
+      samples_per_frame_ * params.channels() >
+          MAX_SAMPLES_x_CHANNELS_PER_FRAME) {

[minyue, 2015/11/11 13:40:10]

is MAX_SAMPLES_x_CHANNELS_PER_FRAME a good name? the lower case x makes it look
unconventional.

Also consider my suggestion on "samples x channels" -> "frames"

[ajose, 2015/11/12 00:10:40]

Changed to |MAX_SAMPLES_PER_BUFFER| to fit new naming scheme.

+    DLOG(ERROR) << "Invalid |samples_per_frame_|: " << samples_per_frame_;
+    return;
+  }
+
+  // Initialize AudioBus buffer for OpusEncoder.
+  buffer_fill_end_ = 0;
+  buffer_.reset(new float[params.channels() * samples_per_frame_]);
+
+  // Initialize OpusEncoder.
+  int opus_result;
+  opus_encoder_ = opus_encoder_create(params.sample_rate(), params.channels(),
+                                      OPUS_APPLICATION_AUDIO, &opus_result);
+  if (opus_result < 0) {
+    DLOG(ERROR) << "Couldn't init opus encoder: " << opus_strerror(opus_result);
+    return;
+  }
+
+  // Note: As of 2013-10-31, the encoder in "auto bitrate" mode would use a
+  // variable bitrate up to 102kbps for 2-channel, 48 kHz audio and a 10 ms
+  // frame size. The opus library authors may, of course, adjust this in
+  // later versions.
+  if (opus_encoder_ctl(opus_encoder_, OPUS_SET_BITRATE(OPUS_AUTO)) != OPUS_OK) {

[minyue, 2015/11/11 13:40:10]

Auto bit rate can be ok. but why not giving AudioTrackRecorder a way to set the
bit rate?

[ajose, 2015/11/12 00:10:40]

This will likely be added in the near future.

[minyue, 2015/11/12 16:52:31]

Acknowledged.

+    DLOG(ERROR) << "Failed to set opus bitrate.";
+    return;
+  }
+
+  audio_params_ = params;
+}
+
+void AudioTrackRecorder::AudioEncoder::EncodeAudio(
+    scoped_ptr<media::AudioBus> audio_bus,
+    const base::TimeTicks& capture_time) {
+  DCHECK(encoder_thread_checker_.CalledOnValidThread());
+  DCHECK_EQ(audio_bus->channels(), audio_params_.channels());
+  DCHECK_EQ(audio_bus->frames(), audio_params_.frames_per_buffer());
+
+  if (!is_initialized())
+    return;
+
+  // Encode all audio in |audio_bus| into zero or more frames.

[minyue, 2015/11/11 13:40:10]

"frames"->"packets"

[ajose, 2015/11/12 00:10:41]

Done.

+  int src_pos = 0;
+  while (src_pos < audio_bus->frames()) {
+    const int num_samples_to_xfer = std::min(
+        samples_per_frame_ - buffer_fill_end_, audio_bus->frames() - src_pos);
+    TransferSamplesIntoBuffer(audio_bus.get(), src_pos, buffer_fill_end_,
+                              num_samples_to_xfer);
+    src_pos += num_samples_to_xfer;
+    buffer_fill_end_ += num_samples_to_xfer;
+
+    if (buffer_fill_end_ < samples_per_frame_)
+      break;
+
+    scoped_ptr<std::string> encoded_data(new std::string());
+    if (EncodeFromFilledBuffer(encoded_data.get())) {
+      on_encoded_audio_cb_.Run(audio_params_, encoded_data.Pass(),
+                               capture_time);
+    }
+
+    // Reset the internal buffer for the next frame.
+    buffer_fill_end_ = 0;
+  }
+}
+
+void AudioTrackRecorder::AudioEncoder::DestroyOpus() {
+  // We don't DCHECK that we're on the encoder thread here, as this could be
+  // called from the dtor (main thread) or from OnSetForamt() (render thread);
+  if (opus_encoder_) {
+    opus_encoder_destroy(opus_encoder_);
+    opus_encoder_ = nullptr;
+  }
+}
+
+void AudioTrackRecorder::AudioEncoder::TransferSamplesIntoBuffer(
+    const media::AudioBus* audio_bus,
+    int source_offset,
+    int buffer_fill_offset,
+    int num_samples) {
+  // TODO(ajose): Consider replacing with AudioBus::ToInterleaved().
+  // http://crbug.com/547918
+  DCHECK(encoder_thread_checker_.CalledOnValidThread());
+  DCHECK(is_initialized());
+  // Opus requires channel-interleaved samples in a single array.
+  for (int ch = 0; ch < audio_bus->channels(); ++ch) {
+    const float* src = audio_bus->channel(ch) + source_offset;
+    const float* const src_end = src + num_samples;
+    float* dest =
+        buffer_.get() + buffer_fill_offset * audio_params_.channels() + ch;
+    for (; src < src_end; ++src, dest += audio_params_.channels())
+      *dest = *src;
+  }
+}
+
+bool AudioTrackRecorder::AudioEncoder::EncodeFromFilledBuffer(
+    std::string* out) {
+  DCHECK(encoder_thread_checker_.CalledOnValidThread());
+  DCHECK(is_initialized());
+
+  out->resize(OPUS_MAX_PAYLOAD_SIZE);

[minyue, 2015/11/11 13:40:10]

Unless memory size is very critical, it would be good to fix the size of out. I
don't know if this solves 547918.

[ajose, 2015/11/12 00:10:40]

Could you clarify this? Won't calling out->resize(SIZE) "fix the size" in the
sense that it will consistently be |SIZE|? Or are you suggesting that |out|
should be passed in with a fixed, known size so that this call to resize() is
unnecessary?

[minyue, 2015/11/12 16:52:31]

Ah, I see why there is a confusion. I just focused on reducing resize() as said
in the bug. But of course, we want to know the actual size on the returning of
|out|.

Without returning the actual payload size explicitly, it would be hard to
circumvent using resize().

You may just leave it as it is.

+  const opus_int32 result = opus_encode_float(
+      opus_encoder_, buffer_.get(), samples_per_frame_,
+      reinterpret_cast<uint8*>(string_as_array(out)), OPUS_MAX_PAYLOAD_SIZE);
+  if (result > 1) {
+    // TODO(ajose): Investigate improving this. http://crbug.com/547918
+    out->resize(result);
+    return true;
+  }
+  // If |result| in {0,1}, do nothing; the documentation says that a return
+  // value of zero or one means the packet does not need to be transmitted.
+  // Otherwise, we have an error.
+  DLOG_IF(ERROR, result < 0) << __FUNCTION__
+                             << " failed: " << opus_strerror(result);
+  return false;
+}
+
+AudioTrackRecorder::AudioTrackRecorder(
+    const blink::WebMediaStreamTrack& track,
+    const OnEncodedAudioCB& on_encoded_audio_cb)
+    : track_(track),
+      encoder_(new AudioEncoder(media::BindToCurrentLoop(on_encoded_audio_cb))),
+      encoder_thread_("AudioEncoderThread") {
+  DCHECK(main_render_thread_checker_.CalledOnValidThread());
+  DCHECK(!track_.isNull());
+  DCHECK(track_.extraData());
+
+  // Start the |encoder_thread_|. From this point on, |encoder_| should work
+  // only on |encoder_thread_|, as enforced by DCHECKs.
+  DCHECK(!encoder_thread_.IsRunning());
+  encoder_thread_.Start();
+
+  // Connect the source provider to the track as a sink.
+  MediaStreamAudioSink::AddToAudioTrack(this, track_);
+}
+
+AudioTrackRecorder::~AudioTrackRecorder() {
+  DCHECK(main_render_thread_checker_.CalledOnValidThread());
+  MediaStreamAudioSink::RemoveFromAudioTrack(this, track_);
+}
+
+void AudioTrackRecorder::OnSetFormat(const media::AudioParameters& params) {
+  DCHECK(encoder_thread_.IsRunning());
+  // If the source is restarted, might have changed to another capture thread.
+  capture_thread_checker_.DetachFromThread();
+  DCHECK(capture_thread_checker_.CalledOnValidThread());
+
+  encoder_thread_.task_runner()->PostTask(
+      FROM_HERE, base::Bind(&AudioEncoder::OnSetFormat, encoder_, params));
+}
+
+void AudioTrackRecorder::OnData(const media::AudioBus& audio_bus,
+                                base::TimeTicks capture_time) {
+  DCHECK(encoder_thread_.IsRunning());
+  DCHECK(capture_thread_checker_.CalledOnValidThread());
+  DCHECK(!capture_time.is_null());
+
+  scoped_ptr<media::AudioBus> audio_data =
+      media::AudioBus::Create(audio_bus.channels(), audio_bus.frames());
+  audio_bus.CopyTo(audio_data.get());
+
+  encoder_thread_.task_runner()->PostTask(
+      FROM_HERE, base::Bind(&AudioEncoder::EncodeAudio, encoder_,
+                            base::Passed(&audio_data), capture_time));
+}
+
+}  // namespace content