MediaCaptureFromElement: add support for audio captureStream().

content/renderer/media/html_audio_element_capturer_source.cc

Index: content/renderer/media/html_audio_element_capturer_source.cc
diff --git a/content/renderer/media/html_audio_element_capturer_source.cc b/content/renderer/media/html_audio_element_capturer_source.cc
new file mode 100644
index 0000000000000000000000000000000000000000..95a08f07eb3e260812872b36680de5215e001318
--- /dev/null
+++ b/content/renderer/media/html_audio_element_capturer_source.cc
@@ -0,0 +1,154 @@
+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "content/renderer/media/html_audio_element_capturer_source.h"
+
+#include "base/threading/thread_task_runner_handle.h"
+#include "base/timer/elapsed_timer.h"

[miu, 2016/05/13 23:40:35]

Seems this is not used.

[mcasas, 2016/05/14 02:23:47]

Done.

+#include "media/base/audio_fifo.h"
+#include "media/base/audio_parameters.h"
+#include "media/base/audio_renderer_sink.h"
+#include "media/blink/webmediaplayer_impl.h"
+#include "third_party/WebKit/public/platform/WebMediaPlayer.h"
+
+namespace content {
+
+namespace {
+
+// Maximum amount of buffers that can be held in the AudioFifo.
+static const size_t kMaxNumberOfBuffers = 10;

[miu, 2016/05/13 23:40:35]

This can be deleted if you agree w/ my advice about using AudioPushFifo instead.

[mcasas, 2016/05/14 02:23:47]

Done.

+
+}  // anonymous namespace
+
+//static
+scoped_refptr<HtmlAudioElementCapturerSource>
+HtmlAudioElementCapturerSource::CreateFromWebMediaPlayerImpl(
+    blink::WebMediaPlayer* player) {
+  DCHECK(player);
+  return make_scoped_refptr(new HtmlAudioElementCapturerSource(
+      static_cast<media::WebAudioSourceProviderImpl*>(
+          player->getAudioSourceProvider())
+          ->AsWeakPtr()));
+}
+
+HtmlAudioElementCapturerSource::HtmlAudioElementCapturerSource(
+    const base::WeakPtr<media::WebAudioSourceProviderImpl>& audio_source)
+    : audio_input_cb_(
+          base::Bind(&HtmlAudioElementCapturerSource::OnAudioBus, this)),
+      audio_source_(audio_source),
+      min_fifo_frames_for_conversion_(0) {
+  DCHECK(audio_source_);
+}
+
+void HtmlAudioElementCapturerSource::Initialize(
+    const media::AudioParameters& params,
+    CaptureCallback* callback,
+    int session_id) {
+  DVLOG(1) << __FUNCTION__ << params.AsHumanReadableString();
+  DCHECK(thread_checker_.CalledOnValidThread());
+
+  capture_callback_ = callback;
+  output_params_ = params;
+}
+
+void HtmlAudioElementCapturerSource::Start() {
+  DCHECK(thread_checker_.CalledOnValidThread());
+  if (audio_source_)
+    audio_source_->SetCopyAudioCallback(audio_input_cb_);

[miu, 2016/05/13 23:40:35]

By creating the closure in the ctor, HtmlAudioElementCapturerSource will hold a
ref-count to itself forever; and therefore it will never be destroyed.

Instead, can you just eliminate |audio_input_cb_|, and do the base::Bind(...)
here?

[mcasas, 2016/05/14 02:23:47]

Absolutely! Done.

+}
+
+void HtmlAudioElementCapturerSource::Stop() {
+  DCHECK(thread_checker_.CalledOnValidThread());
+  if (audio_source_)
+    audio_source_->ClearCopyAudioCallback();
+  capture_callback_ = nullptr;
+}
+
+void HtmlAudioElementCapturerSource::SetVolume(double volume) {
+  DCHECK(thread_checker_.CalledOnValidThread());
+}
+
+void HtmlAudioElementCapturerSource::SetAutomaticGainControl(bool enable) {
+  DCHECK(thread_checker_.CalledOnValidThread());
+}
+
+void HtmlAudioElementCapturerSource::OnAudioBus(
+    std::unique_ptr<media::AudioBus> audio_bus,
+    uint32_t delay_milliseconds,
+    int sample_rate) {
+  // Create |converter_| on first frame arrival.
+  // TODO(mcasas): What if the input parameters change?
+  if (!converter_) {
+    // This parameter signifies the minimum input frames to produce a successful
+    // conversion, and is usually smaller than |audio_bus.frames()|. We use it
+    // to pull one by one converted AudioBuses out of AudioConverter.
+    min_fifo_frames_for_conversion_ = output_params_.frames_per_buffer() *
+                                      sample_rate /
+                                      output_params_.sample_rate();
+
+    input_params_ = media::AudioParameters(
+        output_params_.format(),
+        media::GuessChannelLayout(audio_bus->channels()), sample_rate,
+        output_params_.bits_per_sample(), min_fifo_frames_for_conversion_);
+    DCHECK(input_params_.IsValid());
+
+    CreateConverter();
+  }
+  DCHECK_EQ(input_params_.channels(), audio_bus->channels());
+
+  fifo_->Push(audio_bus.get());

[miu, 2016/05/13 23:40:35]

All this code from here to the end of this method would be greatly simplified if
you use media::AudioPushFifo instead (I wrote it so we wouldn't have to keep
writing this code over and over). It even manages the audio delay calculation
for you, and avoids unnecessary copying when input_params==output_params.

Example usage:
https://code.google.com/p/chromium/codesearch#chromium/src/content/renderer/m...

For this case, you'd construct the AudioPushFifo to callback a method that looks
like:

  void DeliverRebufferedAudio(
      const media::AudioBus& audio_bus,
      int frame_delay) {
    converter_input_bus_ = &audio_bus;  // ProvideInput() reads from this.
    if (!converter_output_bus_) {
      converter_output_bus_ = media::AudioBus::Create(...);
    }
    const int output_block_delay = frame_delay * MillisPerSecond /
input_sample_rate;
    converter_->Convert(converter_output_bus_.get(), output_block_delay, 1.0,
false);
    DCHECK(!converter_input_bus_);  // Cleared by ProvideInput().
  }

And ProvideInput() becomes:

  void ProvideInput(media::AudioBus* audio_bus, ...) {
    DCHECK(converter_input_bus_);
    converter_input_bus_->CopyTo(audio_bus);
    converter_input_bus_ = nullptr;
    return 1.0;
  }

And somewhere prior to all this:

  fifo_.Reset(min_fifo_frames_for_conversion_);

[mcasas, 2016/05/14 02:23:47]

Done.

+
+  // Convert() while there are enough frames queued up in |fifo_| to guarantee a
+  // satisfactory conversion and retrieval of an output buffer, i.e. there are
+  // at least |min_fifo_frames_for_conversion_|.
+  const int input_block_size = audio_bus->frames();
+  while (fifo_->frames() >= min_fifo_frames_for_conversion_) {
+    // Calculated the delay for Capture() based on the position of the converted
+    // frames in the input buffer. Frequently there are frames left over from a
+    // previous |audio_bus| and this delay will be negative.
+    const int output_block_delay = (input_block_size - fifo_->frames()) *
+                                   base::Time::kMillisecondsPerSecond /
+                                   input_params_.sample_rate();
+
+    std::unique_ptr<media::AudioBus> converted_bus = media::AudioBus::Create(
+        output_params_.channels(), output_params_.frames_per_buffer());
+    converter_->Convert(converted_bus.get());
+
+    // TODO(mcasas): The conversion process might take some time that might need
+    // to be discounted: if that's the case, add a base::ElapsedTimer before
+    // Convert() and subtract the Elapsed() time.
+    DVLOG(3) << __FUNCTION__;
+    capture_callback_->Capture(converted_bus.get(), output_block_delay,
+                               1.0 /* volume */, false /* key_pressed */);
+  }
+}
+
+double HtmlAudioElementCapturerSource::ProvideInput(
+    media::AudioBus* audio_bus,
+    base::TimeDelta buffer_delay) {
+  DCHECK_EQ(min_fifo_frames_for_conversion_, audio_bus->frames());
+  fifo_->Consume(audio_bus, 0, audio_bus->frames());
+  return 1.0;  // Return volume greater than zero to indicate we have more data.
+}
+
+HtmlAudioElementCapturerSource::~HtmlAudioElementCapturerSource() {
+  DCHECK(thread_checker_.CalledOnValidThread());
+}
+
+void HtmlAudioElementCapturerSource::CreateConverter() {
+  DCHECK(output_params_.IsValid());
+  DVLOG(1) << " Converting from: " << input_params_.AsHumanReadableString()
+           << " --> " << output_params_.AsHumanReadableString();
+
+  converter_.reset(new media::AudioConverter(input_params_, output_params_,
+                                             false /* disable_fifo */));
+  converter_->AddInput(this);
+  converter_->PrimeWithSilence();
+
+  fifo_.reset(new media::AudioFifo(
+      input_params_.channels(),
+      kMaxNumberOfBuffers * input_params_.frames_per_buffer()));
+}
+
+}  // namespace content