Binding media stream audio track to speech recognition [renderer]

content/renderer/media/speech_recognition_audio_source_provider_unittest.cc

+// Copyright 2014 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "content/renderer/media/speech_recognition_audio_source_provider.h"
+
+#include "base/strings/utf_string_conversions.h"
+#include "content/renderer/media/mock_media_constraint_factory.h"
+#include "content/renderer/media/webrtc/webrtc_local_audio_track_adapter.h"
+#include "content/renderer/media/webrtc_local_audio_track.h"
+#include "media/audio/audio_parameters.h"
+#include "media/base/audio_bus.h"
+#include "testing/gmock/include/gmock/gmock.h"
+#include "testing/gtest/include/gtest/gtest.h"
+#include "third_party/WebKit/public/platform/WebMediaStreamTrack.h"
+
+namespace content {
+
+// Mocked out sockets used for Send/Receive.
+// Data is written and read from a shared buffer used as a FIFO and there is
+// no blocking. |OnSendCB| is used to trigger a |Receive| on the other socket.
+class MockSyncSocket : public base::SyncSocket {
+ public:
+  // This allows for 2 requests in queue between the |MockSyncSocket|s.
+  static const int kSharedBufferSize = 8;
+
+  // Buffer to be shared between two |MockSyncSocket|s. Allocated on heap.
+  struct SharedBuffer {
+    SharedBuffer() : start(0), length(0) {}
+
+    uint8 data[kSharedBufferSize];
+    size_t start;
+    size_t length;
+  };
+
+  // Callback used for pairing an A.Send() with B.Receieve() without blocking.
+  typedef base::Callback<void()> OnSendCB;
+
+  explicit MockSyncSocket(SharedBuffer* shared_buffer)
+      : buffer_(shared_buffer),
+        in_failure_mode_(false) { }
+
+  MockSyncSocket(SharedBuffer* shared_buffer, const OnSendCB& on_send_cb)
+      : buffer_(shared_buffer),
+        on_send_cb_(on_send_cb),
+        in_failure_mode_(false) { }
+
+  virtual size_t Send(const void* buffer, size_t length) OVERRIDE;
+  virtual size_t Receive(void* buffer, size_t length) OVERRIDE;
+
+  // When |in_failure_mode_| == true, the socket fails to send.
+  void SetFailureMode(bool in_failure_mode) {
+    in_failure_mode_ = in_failure_mode;
+  }
+
+ private:
+  SharedBuffer* buffer_;
+  const OnSendCB on_send_cb_;
+  bool in_failure_mode_;
+};
+
+size_t MockSyncSocket::Send(const void* buffer, size_t length) {
+  if (in_failure_mode_)
+    return 0;
+
+  uint8* b = static_cast<uint8*>(const_cast<void*>(buffer));
+  for (size_t i = 0; i < length; i++, buffer_->length++)

[burnik, 2014/09/23 12:39:21]

Changed to prefixed increment.

+    buffer_->data[buffer_->start + buffer_->length] = b[i];
+
+  on_send_cb_.Run();
+  return length;
+}
+
+size_t MockSyncSocket::Receive(void* buffer, size_t length) {
+  uint8* b = static_cast<uint8*>(const_cast<void*>(buffer));
+  for (size_t i = buffer_->start; i < buffer_->length; i++, buffer_->start++)

[burnik, 2014/09/23 12:39:20]

Changed to prefixed increment.

+    b[i] = buffer_->data[buffer_->start];
+
+  // Since buffer is used sequentially, we can reset the buffer indices here.
+  buffer_->start = buffer_->length = 0;
+  return length;
+}
+
+////////////////////////////////////////////////////////////////////////////////

[henrika (OOO until Aug 14), 2014/09/22 08:02:19]

Please remove these non-standard separators.

[burnik, 2014/09/22 09:17:36]

Done.

[no longer working on chromium, 2014/09/23 10:09:13]

Not done yet.

[burnik, 2014/09/23 12:39:21]

Yes, done for next patchset as advertised.

+
+class FakeSpeechRecognizer {

[henrika (OOO until Aug 14), 2014/09/22 08:02:19]

This looks like a very complex helper which is now without any comments. To me,
it seems like a risk to add such a complex class to a test since the test might
fail due to this special implementation instead of the actual code.

[burnik, 2014/09/22 09:17:36]

This is the mock consumer. Unit tests focus on the class being tested (the
producer here) and should show how that class is to be used as far as I know.
I've added more comments to explain what the class does. 

I don't really see how this unit test could fail because of this helper class.
The consumer code is covered in a different test.

[henrika (OOO until Aug 14), 2014/09/23 10:45:33]

I am not saying it will fail but that is a large helper without any comments.
Hence, it is not clear to me why the test must contain all this code instead of
using existing (possibly mock) instead.

[burnik, 2014/09/23 12:39:21]

Ok. I'll revisit existing unit tests to see if anything can be reused. 
This could possibly happen after I add new stuff to other tests to check the
consumer.
Until then, I think it is a fairly simple helper showing how the consumer should
behave.
Also, I pointed out which methods are just mocks and helpers for testing.
Generally, any code duplication should be avoided, in that sense I do agree.

+ public:
+  FakeSpeechRecognizer() : is_responsive_(true) { }
+
+  void Initialize(
+      const blink::WebMediaStreamTrack& track,
+      const media::AudioParameters& sink_params,
+      base::SharedMemoryHandle* foreign_memory_handle) {
+    // Shared memory is allocated, mapped and shared.
+    uint32 shared_memory_size =
+        sizeof(media::AudioInputBufferParameters) +
+        media::AudioBus::CalculateMemorySize(sink_params);
+    shared_memory_.reset(new base::SharedMemory());
+    ASSERT_TRUE(shared_memory_->CreateAndMapAnonymous(shared_memory_size));
+    ASSERT_TRUE(shared_memory_->ShareToProcess(base::GetCurrentProcessHandle(),
+                                               foreign_memory_handle));
+
+    // Wrap the shared memory for the audio bus.
+    media::AudioInputBuffer* buffer =
+        static_cast<media::AudioInputBuffer*>(shared_memory_->memory());
+    audio_track_bus_ = media::AudioBus::WrapMemory(sink_params, buffer->audio);
+
+    // Reference to the counter used to synchronize.
+    buffer_index_ = &(buffer->params.size);
+    *buffer_index_ = 0U;
+
+    // Create a shared buffer for the |MockSyncSocket|s.
+    shared_buffer_.reset(new MockSyncSocket::SharedBuffer());
+
+    // Local socket will receive signals from the producer.
+    local_socket_.reset(new MockSyncSocket(shared_buffer_.get()));
+
+    // We automatically trigger a Receive when data is sent over the socket.
+    foreign_socket_ = new MockSyncSocket(
+        shared_buffer_.get(),
+        base::Bind(&FakeSpeechRecognizer::EmulateReceiveThreadLoopIteration,
+                   base::Unretained(this)));
+
+    // This is usually done to pair the sockets. Here it's not effective.
+    base::SyncSocket::CreatePair(local_socket_.get(), foreign_socket_);
+  }
+
+  // Emulates a single iteraton of a thread receiving on the socket.
+  // This would normally be done on a receiving thread's task on the browser.
+  void EmulateReceiveThreadLoopIteration() {
+    // When not responsive do nothing as if the process is busy.
+    if (!is_responsive_)
+      return;
+
+    local_socket_->Receive(buffer_index_, sizeof(*buffer_index_));
+    // Notify the producer that the audio buffer has been consumed.
+    (*buffer_index_)++;

[no longer working on chromium, 2014/09/23 10:09:13]

++(*buffer_index_)

[burnik, 2014/09/23 12:39:21]

Done.

+  }
+
+  // Used to simulate an unresponsive behaviour of the consumer.
+  void SimulateResponsiveness(bool is_responsive) {
+    is_responsive_ = is_responsive;
+  }
+
+  MockSyncSocket* foreign_socket() { return foreign_socket_; }
+  media::AudioBus* audio_bus() const { return audio_track_bus_.get(); }
+  uint32 buffer_index() { return *buffer_index_; }
+
+ private:
+  bool is_responsive_;
+
+  // Shared memory for the audio and synchronization.
+  scoped_ptr<base::SharedMemory> shared_memory_;
+
+  // Fake sockets shared buffer.
+  scoped_ptr<MockSyncSocket::SharedBuffer> shared_buffer_;
+  scoped_ptr<MockSyncSocket> local_socket_;
+  MockSyncSocket* foreign_socket_;

[no longer working on chromium, 2014/09/23 10:09:13]

why this is a raw pointer?

[burnik, 2014/09/23 12:39:21]

It is owned by the recognizer and destroyed there.

+
+  // Audio bus wrapping the shared memory from the renderer.
+  scoped_ptr<media::AudioBus> audio_track_bus_;
+
+  // Used for synchronization of sent/received buffers.
+  uint32* buffer_index_;
+};
+
+////////////////////////////////////////////////////////////////////////////////

[henrika (OOO until Aug 14), 2014/09/22 08:02:19]

remove

[burnik, 2014/09/22 09:17:36]

Done.

+
+namespace {
+
+// Supported speech recognition audio parameters.
+const int kSpeechRecognitionSampleRate = 16000;
+const int kSpeechRecognitionFramesPerBuffer = 1600;
+
+// Input audio format.
+const media::AudioParameters::Format kInputFormat =
+    media::AudioParameters::AUDIO_PCM_LOW_LATENCY;
+const media::ChannelLayout kInputChannelLayout = media::CHANNEL_LAYOUT_MONO;
+const int kInputChannels = 1;
+const int kInputBitsPerSample = 16;
+
+// Output audio format.
+const media::AudioParameters::Format kOutputFormat =
+    media::AudioParameters::AUDIO_PCM_LOW_LATENCY;
+const media::ChannelLayout kOutputChannelLayout = media::CHANNEL_LAYOUT_STEREO;
+const int kOutputChannels = 2;
+const int kOutputBitsPerSample = 16;
+
+}  // namespace
+
+////////////////////////////////////////////////////////////////////////////////
+
+class SpeechRecognitionAudioSourceProviderTest : public testing::Test {
+ public:
+  SpeechRecognitionAudioSourceProviderTest() { }
+
+  // Initializes the producer and consumer with specified audio parameters.

[henrika (OOO until Aug 14), 2014/09/22 08:02:18]

Can you elaborate on what a producer and consumer is in this test.

[burnik, 2014/09/22 09:17:36]

Yes. It's explained on lines 228 - 238.

+  // Returns the minimal number of input audio buffers which need to be captured
+  // before they get sent to the consumer.
+  uint32 Initialize(int input_sample_rate,
+                    int input_frames_per_buffer,
+                    int output_sample_rate,
+                    int output_frames_per_buffer) {
+    // Audio Environment setup.
+    source_params_.Reset(kInputFormat,
+                         kInputChannelLayout,
+                         kInputChannels,
+                         input_sample_rate,
+                         kInputBitsPerSample,
+                         input_frames_per_buffer);
+    sink_params_.Reset(kOutputFormat,
+                       kOutputChannelLayout,
+                       kOutputChannels,
+                       output_sample_rate,
+                       kOutputBitsPerSample,
+                       output_frames_per_buffer);
+    source_data_.reset(new int16[input_frames_per_buffer * kInputChannels]);
+
+    // Prepare the track and audio source.
+    blink::WebMediaStreamTrack blink_track;
+    PrepareBlinkTrackOfType(MEDIA_DEVICE_AUDIO_CAPTURE, &blink_track);
+
+    // Get the native track from the blink track and initialize.
+    native_track_ =
+        static_cast<WebRtcLocalAudioTrack*>(blink_track.extraData());
+    native_track_->OnSetFormat(source_params_);
+
+    // Create and initialize the consumer.
+    recognizer_.reset(new FakeSpeechRecognizer());
+    base::SharedMemoryHandle foreign_memory_handle;
+    recognizer_->Initialize(blink_track, sink_params_, &foreign_memory_handle);
+
+    // Create the producer.
+    audio_source_provider_.reset(new SpeechRecognitionAudioSourceProvider(
+        blink_track, sink_params_, foreign_memory_handle,
+        recognizer_->foreign_socket(),
+        base::Bind(&SpeechRecognitionAudioSourceProviderTest::StoppedCallback,
+                   base::Unretained(this))));
+
+    // Return number of buffers needed to trigger resampling and consumption.
+    return static_cast<uint32>(std::ceil(
+        static_cast<double>(output_frames_per_buffer * input_sample_rate) /
+        (input_frames_per_buffer * output_sample_rate)));
+  }
+
+  // Mock callback expected to be called when the track is stopped.
+  MOCK_METHOD0(StoppedCallback, void());
+
+ protected:
+  static void PrepareBlinkTrackOfType(
+      const MediaStreamType device_type,
+      blink::WebMediaStreamTrack* blink_track) {
+    // Device info.
+    StreamDeviceInfo device_info(device_type, "Mock audio device",
+                                 "mock_audio_device_id");
+
+    // Constraints.
+    MockMediaConstraintFactory constraint_factory;
+    const blink::WebMediaConstraints constraints =
+        constraint_factory.CreateWebMediaConstraints();
+
+    // Capturer.

[henrika (OOO until Aug 14), 2014/09/22 08:02:18]

These comments does not add much. Please explain what they do in the test or
remove.

[burnik, 2014/09/22 09:17:36]

All these comments are now removed.

+    scoped_refptr<WebRtcAudioCapturer> capturer(
+        WebRtcAudioCapturer::CreateCapturer(-1, device_info, constraints, NULL,
+                                            NULL));
+
+    // Adapter.
+    scoped_refptr<WebRtcLocalAudioTrackAdapter> adapter(
+        WebRtcLocalAudioTrackAdapter::Create(std::string(), NULL));
+
+    // Native track.
+    scoped_ptr<WebRtcLocalAudioTrack> native_track(
+        new WebRtcLocalAudioTrack(adapter.get(), capturer, NULL));
+
+    // Blink audio source.
+    blink::WebMediaStreamSource blink_audio_source;
+    blink_audio_source.initialize(base::UTF8ToUTF16("dummy_source_id"),
+                                  blink::WebMediaStreamSource::TypeAudio,
+                                  base::UTF8ToUTF16("dummy_source_name"));
+    MediaStreamSource::SourceStoppedCallback cb;
+    blink_audio_source.setExtraData(
+        new MediaStreamAudioSource(-1, device_info, cb, NULL));
+
+    // Blink track.
+    blink_track->initialize(blink::WebString::fromUTF8("dummy_audio_track"),
+                            blink_audio_source);
+    blink_track->setExtraData(native_track.release());
+  }
+
+  // Emulates an audio capture device capturing data from the source.
+  inline void CaptureAudio(const uint32 buffers) {
+    for (uint32 i = 0; i < buffers; ++i)
+      native_track_->Capture(source_data_.get(),
+                             base::TimeDelta::FromMilliseconds(0), 1, false,

[henrika (OOO until Aug 14), 2014/09/22 08:02:18]

FromMilliseconds(0)?

[burnik, 2014/09/22 09:17:36]

Yes, no delay is required in the unit test.

+                             false);
+  }
+
+  // Used to simulate a problem with sockets.
+  void SetFailureModeOnForeignSocket(bool in_failure_mode) {
+    recognizer_->foreign_socket()->SetFailureMode(in_failure_mode);
+  }
+
+  // Helper method for verifying captured audio data has been consumed.
+  inline void AssertConsumedBuffers(const uint32 buffer_index) {
+    ASSERT_EQ(buffer_index, recognizer_->buffer_index());
+  }
+
+  // Helper method for providing audio data to producer and verifying it was
+  // consumed on the recognizer.
+  inline void CaptureAudioAndAssertConsumedBuffers(const uint32 buffers,
+                                                   const uint32 buffer_index) {
+    CaptureAudio(buffers);
+    AssertConsumedBuffers(buffer_index);
+  }
+
+  // Helper method to capture and assert consumption at different sample rates
+  // and audio buffer sizes.
+  inline void AssertConsumptionForAudioParameters(
+      const int input_sample_rate,
+      const int input_frames_per_buffer,
+      const int output_sample_rate,
+      const int output_frames_per_buffer,
+      const uint32 consumptions) {
+  const uint32 kBuffersPerNotification =
+        Initialize(input_sample_rate, input_frames_per_buffer,
+                   output_sample_rate, output_frames_per_buffer);
+    AssertConsumedBuffers(0U);
+
+    for (uint32 i = 1U; i <= consumptions; ++i) {
+      CaptureAudio(kBuffersPerNotification);
+      ASSERT_EQ(i, recognizer_->buffer_index())
+          << "Tested at rates: "
+          << "In(" << input_sample_rate << ", " << input_frames_per_buffer
+          << ") "
+          << "Out(" << output_sample_rate << ", " << output_frames_per_buffer
+          << ")";
+    }
+  }
+
+  // Producer.
+  scoped_ptr<SpeechRecognitionAudioSourceProvider> audio_source_provider_;
+
+  // Consumer.
+  scoped_ptr<FakeSpeechRecognizer> recognizer_;
+
+  // Audio related members.
+  scoped_ptr<int16[]> source_data_;
+  media::AudioParameters source_params_;
+  media::AudioParameters sink_params_;
+  WebRtcLocalAudioTrack* native_track_;
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+TEST_F(SpeechRecognitionAudioSourceProviderTest, CheckIsSupportedAudioTrack) {

[henrika (OOO until Aug 14), 2014/09/22 08:02:18]

Could you make the name more clear? CheckIsSupported is not clear to me.

[burnik, 2014/09/22 09:17:37]

Added comment above test.

+  typedef std::map<MediaStreamType, bool> SupportedTrackPolicy;
+
+  // This test must be aligned with the policy of supported tracks.
+  SupportedTrackPolicy p;
+  p[MEDIA_NO_SERVICE] = false;
+  p[MEDIA_DEVICE_AUDIO_CAPTURE] = true;  // The only one supported for now.
+  p[MEDIA_DEVICE_VIDEO_CAPTURE] = false;
+  p[MEDIA_TAB_AUDIO_CAPTURE] = false;
+  p[MEDIA_TAB_VIDEO_CAPTURE] = false;
+  p[MEDIA_DESKTOP_VIDEO_CAPTURE] = false;
+  p[MEDIA_LOOPBACK_AUDIO_CAPTURE] = false;
+  p[MEDIA_DEVICE_AUDIO_OUTPUT] = false;
+
+  // Ensure this test gets updated along with |content::MediaStreamType| enum.
+  EXPECT_EQ(NUM_MEDIA_TYPES, p.size());
+
+  // Check the the entire policy.
+  for (SupportedTrackPolicy::iterator it = p.begin(); it != p.end(); ++it) {
+    blink::WebMediaStreamTrack blink_track;
+    PrepareBlinkTrackOfType(it->first, &blink_track);
+    ASSERT_EQ(
+        it->second,
+        SpeechRecognitionAudioSourceProvider::IsSupportedTrack(blink_track));
+  }
+}
+
+TEST_F(SpeechRecognitionAudioSourceProviderTest, RecognizerNotifiedOnSocket) {

[henrika (OOO until Aug 14), 2014/09/22 08:02:18]

Please add some lines of comments above each test explaining some more about
what you test.

[burnik, 2014/09/22 09:17:37]

Done.

+  const size_t kNumAudioParamTuples = 22;
+  const int kAudioParams[kNumAudioParamTuples][2] = {
+      {8000, 80}, {8000, 800}, {16000, 160}, {16000, 1600},
+      {32000, 320}, {32000, 3200}, {44100, 441}, {44100, 4410},
+      {48000, 480}, {48000, 4800}, {96000, 960}, {96000, 9600},
+      {11025, 111}, {11025, 1103}, {22050, 221}, {22050, 2205},
+      {88200, 882}, {88200, 8820}, {176400, 1764}, {176400, 17640},
+      {192000, 1920}, {192000, 19200}};
+
+  // Check all listed tuples of input sample rates and buffers sizes.
+  for (size_t i = 0; i < kNumAudioParamTuples; ++i) {
+    AssertConsumptionForAudioParameters(
+        kAudioParams[i][0], kAudioParams[i][1],
+        kSpeechRecognitionSampleRate, kSpeechRecognitionFramesPerBuffer, 3U);
+  }
+}
+
+TEST_F(SpeechRecognitionAudioSourceProviderTest, AudioDataIsResampledOnSink) {

[henrika (OOO until Aug 14), 2014/09/22 08:02:19]

Lots of hardcoded values in this test. Makes it difficult to understand what you
test. Should it only work for these settings?

[burnik, 2014/09/22 09:17:37]

Added more comments.
I don't test that the resampler does a good job, since that is covered in it's
own unit test.
I use these typical values here just to make sure that the producer also does
resampling when it delivers frames to the consumer.

+  const uint32 kBuffersPerNotification = Initialize(44100, 441, 16000, 1600);
+
+  // Fill audio input frames with 0, 1, 2, 3, ..., 440.
+  const uint32 source_data_length = 441 * kInputChannels;
+  for (uint32 i = 0; i < source_data_length; ++i)
+    source_data_[i] = i;
+
+  const uint32 num_frames_to_test = 12;
+  const uint32 sink_data_length = 1600 * kOutputChannels;
+  int16 sink_data[sink_data_length];
+  media::AudioBus* sink_bus = recognizer_->audio_bus();
+
+  // Render the audio data from the recognizer.
+  sink_bus->ToInterleaved(sink_bus->frames(),
+                          sink_params_.bits_per_sample() / 8, sink_data);
+
+  // Test both channels are zeroed out before we trigger resampling.
+  for (uint32 i = 0; i < num_frames_to_test; ++i) {
+    ASSERT_EQ(0, sink_data[i * 2]);
+    ASSERT_EQ(0, sink_data[i * 2 + 1]);
+  }
+
+  // Trigger the source provider to resample the input data.
+  AssertConsumedBuffers(0U);
+  CaptureAudioAndAssertConsumedBuffers(kBuffersPerNotification, 1U);
+
+  // Render the audio data from the recognizer.
+  sink_bus->ToInterleaved(sink_bus->frames(),
+                          sink_params_.bits_per_sample() / 8, sink_data);
+
+  // Resampled data expected frames - based on |source_data_|.
+  const int16 expected_data[num_frames_to_test] = {0,  2,  5,  8,  11, 13,
+                                                   16, 19, 22, 24, 27, 30};
+
+  // Test both channels have same resampled data.
+  for (uint32 i = 0; i < num_frames_to_test; ++i) {
+    ASSERT_EQ(expected_data[i], sink_data[i * 2]);
+    ASSERT_EQ(expected_data[i], sink_data[i * 2 + 1]);
+  }
+}
+
+TEST_F(SpeechRecognitionAudioSourceProviderTest, SyncSocketFailsSendingData) {
+  const uint32 kBuffersPerNotification = Initialize(44100, 441, 16000, 1600);
+  // (1) Start with no problems on the socket.

[henrika (OOO until Aug 14), 2014/09/22 08:02:18]

Remove (1) and (2)

[burnik, 2014/09/22 09:17:37]

Done.

+  AssertConsumedBuffers(0U);
+  CaptureAudioAndAssertConsumedBuffers(kBuffersPerNotification, 1U);
+
+  // (2) A failure occurs (socket cannot to send).
+  SetFailureModeOnForeignSocket(true);
+  CaptureAudioAndAssertConsumedBuffers(kBuffersPerNotification, 1U);
+}
+
+TEST_F(SpeechRecognitionAudioSourceProviderTest, OnReadyStateChangedOccured) {
+  const uint32 kBuffersPerNotification = Initialize(44100, 441, 16000, 1600);
+  AssertConsumedBuffers(0U);
+  CaptureAudioAndAssertConsumedBuffers(kBuffersPerNotification, 1U);
+  EXPECT_CALL(*this, StoppedCallback()).Times(1);
+
+  native_track_->Stop();
+  CaptureAudioAndAssertConsumedBuffers(kBuffersPerNotification, 1U);
+}
+
+}  // namespace content