ppapi: implement PPB_AudioEncoder

content/renderer/pepper/pepper_audio_encoder_host.cc

+// Copyright 2015 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "base/bind.h"
+#include "base/memory/shared_memory.h"
+#include "content/public/renderer/renderer_ppapi_host.h"
+#include "content/renderer/pepper/host_globals.h"
+#include "content/renderer/pepper/pepper_audio_encoder_host.h"
+#include "content/renderer/render_thread_impl.h"
+#include "media/base/bind_to_current_loop.h"
+#include "ppapi/c/pp_codecs.h"
+#include "ppapi/c/pp_errors.h"
+#include "ppapi/host/dispatch_host_message.h"
+#include "ppapi/host/ppapi_host.h"
+#include "ppapi/proxy/ppapi_messages.h"
+#include "ppapi/shared_impl/media_stream_buffer.h"
+#include "third_party/opus/src/include/opus.h"
+
+using ppapi::proxy::SerializedHandle;
+
+namespace content {
+
+namespace {
+
+// Buffer up to 150ms (15 x 10ms per frame).
+const uint32_t kDefaultNumberOfAudioBuffers = 15;
+
+// Class used to pass audio data between threads.
+class AudioData {
+ public:
+  AudioData(const AudioData& other) : data(other.data), size(other.size) {}
+  AudioData(uint8_t* data, size_t size) : data(data), size(size) {}
+  ~AudioData() {}
+
+  uint8_t* data;
+  size_t size;
+};
+
+bool PP_HardwareAccelerationCompatible(bool accelerated,
+                                       PP_HardwareAcceleration requested) {
+  switch (requested) {
+    case PP_HARDWAREACCELERATION_ONLY:
+      return accelerated;
+    case PP_HARDWAREACCELERATION_NONE:
+      return !accelerated;
+    case PP_HARDWAREACCELERATION_WITHFALLBACK:
+      return true;
+      // No default case, to catch unhandled PP_HardwareAcceleration values.
+  }
+  return false;
+}
+
+AudioData GetAudioDataFromBufferManager(
+    const scoped_ptr<ppapi::MediaStreamBufferManager>& buffer_manager,
+    int32_t* buffer_id) {
+  DCHECK(buffer_manager->HasAvailableBuffer());
+  int32_t id = buffer_manager->DequeueBuffer();
+  ppapi::MediaStreamBuffer* buffer = buffer_manager->GetBufferPointer(id);
+  *buffer_id = id;
+  return AudioData(static_cast<uint8_t*>(buffer->audio.data),
+                   buffer->audio.data_size);
+}
+
+AudioData GetBitstreamDataFromBufferManager(
+    const scoped_ptr<ppapi::MediaStreamBufferManager>& buffer_manager,
+    int32_t* buffer_id) {
+  DCHECK(buffer_manager->HasAvailableBuffer());
+  int32_t id = buffer_manager->DequeueBuffer();
+  uint8_t* buffer =
+      reinterpret_cast<uint8_t*>(buffer_manager->GetBufferPointer(id));
+  *buffer_id = id;
+  return AudioData(buffer, buffer_manager->buffer_size());
+}
+
+void StopAudioEncoder(
+    scoped_ptr<PepperAudioEncoderHost::AudioEncoderImpl> encoder,
+    scoped_ptr<ppapi::MediaStreamBufferManager> audio_buffer_manager,
+    scoped_ptr<ppapi::MediaStreamBufferManager> bitstream_buffer_manager) {}
+
+}  // namespace
+
+// This class should be constructed and initialized on the main renderer
+// thread, used and destructed on the media thread.
+class PepperAudioEncoderHost::AudioEncoderImpl {
+ public:
+  // Callback used to signal encoded data. If |size| is negative, an error
+  // occured.
+  using BitstreamBufferReadyCB = base::Callback<void(int32_t size)>;
+
+  AudioEncoderImpl();
+  ~AudioEncoderImpl();
+
+  // Used on the renderer thread.
+  std::vector<PP_AudioProfileDescription> GetSupportedProfiles();
+  bool Initialize(const ppapi::proxy::PPB_AudioEncodeParameters& parameters);
+  int32_t GetNumberOfSamplesPerFrame();
+
+  // Used on the media thread.
+  void Encode(const AudioData& input,
+              const AudioData& output,
+              BitstreamBufferReadyCB callback);
+  void RequestBitrateChange(uint32_t bitrate);
+
+ private:
+  // Initialization parameters.
+  ppapi::proxy::PPB_AudioEncodeParameters parameters_;
+
+  scoped_ptr<uint8[]> encoder_memory_;
+  OpusEncoder* opus_encoder_;
+
+  DISALLOW_COPY_AND_ASSIGN(AudioEncoderImpl);
+};
+
+PepperAudioEncoderHost::AudioEncoderImpl::AudioEncoderImpl() {}

[bbudge, 2015/11/12 20:31:31]

Could you initialize opus_encoder_(NULL) and parameters here?

[llandwerlin-old, 2015/11/13 12:46:23]

Done.

+
+PepperAudioEncoderHost::AudioEncoderImpl::~AudioEncoderImpl() {}
+
+std::vector<PP_AudioProfileDescription>
+PepperAudioEncoderHost::AudioEncoderImpl::GetSupportedProfiles() {
+  std::vector<PP_AudioProfileDescription> profiles;
+  static const uint32_t sampling_rates[] = {8000, 12000, 16000, 24000, 48000};
+
+  for (uint32_t i = 0; i < arraysize(sampling_rates); ++i) {
+    PP_AudioProfileDescription profile;
+    profile.profile = PP_AUDIOPROFILE_OPUS;
+    profile.max_channels = 2;
+    profile.sample_size = PP_AUDIOBUFFER_SAMPLESIZE_16_BITS;
+    profile.sample_rate = sampling_rates[i];
+    profile.hardware_accelerated = PP_FALSE;
+    profiles.push_back(profile);
+  }
+  return profiles;
+}
+
+bool PepperAudioEncoderHost::AudioEncoderImpl::Initialize(
+    const ppapi::proxy::PPB_AudioEncodeParameters& parameters) {

[bbudge, 2015/11/12 20:31:31]

Could the body be reworked here so that fields are not left in partially
initialized state if we fail? In other words, only assign to encoder_memory_,
opus_encoder_, and parameters_ once you know Initialize succeeded.

Also, in methods, could you DCHECK(opus_encoder_) or some similar proxy for
success? Of course GetSupportedProfiles is the exception, since it is called
before Initialize. In fact, that method could be made static since it shouldn't
need any state.

[llandwerlin-old, 2015/11/13 12:46:23]

Ok, I think the DCHECK should be on encoder_memory_.
I'll add a comment in the class to let the reader know that all the variables
are only valid if encoder_memory_ is not null.

+  if (parameters.output_profile != PP_AUDIOPROFILE_OPUS)
+    return false;
+
+  DCHECK(!encoder_memory_);
+
+  int32_t encoder_size = opus_encoder_get_size(parameters.channels);
+  if (encoder_size < 1)
+    return false;
+
+  encoder_memory_.reset(new uint8[encoder_size]);
+  opus_encoder_ = reinterpret_cast<OpusEncoder*>(encoder_memory_.get());
+
+  if (opus_encoder_init(opus_encoder_, parameters.input_sample_rate,
+                        parameters.channels, OPUS_APPLICATION_AUDIO) != OPUS_OK)
+    return false;
+
+  if (opus_encoder_ctl(opus_encoder_,
+                       OPUS_SET_BITRATE(parameters.initial_bitrate <= 0
+                                            ? OPUS_AUTO
+                                            : parameters.initial_bitrate)) !=
+      OPUS_OK)
+    return false;
+
+  parameters_ = parameters;
+
+  return true;
+}
+
+int32_t PepperAudioEncoderHost::AudioEncoderImpl::GetNumberOfSamplesPerFrame() {

[bbudge, 2015/11/12 20:31:31]

DCHECK() that Initialize succeeded for sanity here and other methods.

[llandwerlin-old, 2015/11/13 12:46:23]

Done.

+  // Opus supports 2.5, 5, 10, 20, 40 or 60ms audio frames. We take
+  // 10ms by default.
+  return parameters_.input_sample_rate / 100;
+}
+
+void PepperAudioEncoderHost::AudioEncoderImpl::Encode(
+    const AudioData& input,
+    const AudioData& output,
+    BitstreamBufferReadyCB callback) {
+  int32_t result = opus_encode(
+      opus_encoder_, reinterpret_cast<opus_int16*>(input.data),
+      (input.size / parameters_.channels) / parameters_.input_sample_size,
+      output.data, output.size);
+  callback.Run(result);
+}
+
+void PepperAudioEncoderHost::AudioEncoderImpl::RequestBitrateChange(
+    uint32_t bitrate) {
+  opus_encoder_ctl(opus_encoder_, OPUS_SET_BITRATE(bitrate));
+}
+
+PepperAudioEncoderHost::PepperAudioEncoderHost(RendererPpapiHost* host,
+                                               PP_Instance instance,
+                                               PP_Resource resource)
+    : ResourceHost(host->GetPpapiHost(), instance, resource),
+      renderer_ppapi_host_(host),
+      initialized_(false),
+      encoder_last_error_(PP_ERROR_FAILED),
+      audio_buffer_manager_(new ppapi::MediaStreamBufferManager(this)),
+      bitstream_buffer_manager_(new ppapi::MediaStreamBufferManager(this)),

[bbudge, 2015/11/12 20:31:31]

It's probably better to defer constructing these until you are actually
allocating the buffers.

[llandwerlin-old, 2015/11/13 12:46:23]

Done.

+      media_task_runner_(RenderThreadImpl::current()
+                             ->GetMediaThreadTaskRunner()),
+      weak_ptr_factory_(this) {}
+
+PepperAudioEncoderHost::~PepperAudioEncoderHost() {
+  Close();
+}
+
+int32_t PepperAudioEncoderHost::OnResourceMessageReceived(
+    const IPC::Message& msg,
+    ppapi::host::HostMessageContext* context) {
+  PPAPI_BEGIN_MESSAGE_MAP(PepperAudioEncoderHost, msg)
+    PPAPI_DISPATCH_HOST_RESOURCE_CALL_0(
+        PpapiHostMsg_AudioEncoder_GetSupportedProfiles,
+        OnHostMsgGetSupportedProfiles)
+    PPAPI_DISPATCH_HOST_RESOURCE_CALL(PpapiHostMsg_AudioEncoder_Initialize,
+                                      OnHostMsgInitialize)
+    PPAPI_DISPATCH_HOST_RESOURCE_CALL(PpapiHostMsg_AudioEncoder_Encode,
+                                      OnHostMsgEncode)
+    PPAPI_DISPATCH_HOST_RESOURCE_CALL(
+        PpapiHostMsg_AudioEncoder_RecycleBitstreamBuffer,
+        OnHostMsgRecycleBitstreamBuffer)
+    PPAPI_DISPATCH_HOST_RESOURCE_CALL(
+        PpapiHostMsg_AudioEncoder_RequestBitrateChange,
+        OnHostMsgRequestBitrateChange)
+    PPAPI_DISPATCH_HOST_RESOURCE_CALL_0(PpapiHostMsg_AudioEncoder_Close,
+                                        OnHostMsgClose)
+  PPAPI_END_MESSAGE_MAP()
+  return PP_ERROR_FAILED;
+}
+
+int32_t PepperAudioEncoderHost::OnHostMsgGetSupportedProfiles(
+    ppapi::host::HostMessageContext* context) {
+  std::vector<PP_AudioProfileDescription> profiles;
+  GetSupportedProfiles(&profiles);
+
+  host()->SendReply(
+      context->MakeReplyMessageContext(),
+      PpapiPluginMsg_AudioEncoder_GetSupportedProfilesReply(profiles));
+
+  return PP_OK_COMPLETIONPENDING;
+}
+
+int32_t PepperAudioEncoderHost::OnHostMsgInitialize(
+    ppapi::host::HostMessageContext* context,
+    const ppapi::proxy::PPB_AudioEncodeParameters& parameters) {
+  if (initialized_)
+    return PP_ERROR_FAILED;
+
+  if (!IsInitializationValid(parameters))
+    return PP_ERROR_NOTSUPPORTED;
+
+  encode_parameters_ = parameters;

[bbudge, 2015/11/12 20:31:31]

nit: s/encode_parameters_/parameters_ for consistency with the AudioEncoderImpl,
since there are no other kind of parameters in the host.

[llandwerlin-old, 2015/11/13 12:46:23]

Done.

+
+  int32_t error = PP_ERROR_FAILED;

[bbudge, 2015/11/12 20:31:31]

I think the initialization code is clearer if you do the inverse tests, i.e.

if (parameters.acceleration != PP_HARDWAREACCELERATION_NONE &&
    parameters.acceleration != PP_HARDWAREACCELERATION_WITHFALLBACK)
  return PP_ERROR_FAILED;
scoped_ptr<AudioEncoderImpl> encoder(new ...
if (!encoder_->Initialize(parameters))
  return PP_ERROR_FAILED;
if (!AllocateBuffers())
  return PP_ERROR_NOMEMORY;

encode_parameters_ = parameters;
encoder_.swap(encoder);
...

[llandwerlin-old, 2015/11/13 12:46:23]

Thanks, done.

+  if (parameters.acceleration == PP_HARDWAREACCELERATION_NONE ||
+      parameters.acceleration == PP_HARDWAREACCELERATION_WITHFALLBACK) {
+    encoder_.reset(new AudioEncoderImpl);
+    if (encoder_->Initialize(parameters)) {
+      if (AllocateBuffers()) {
+        initialized_ = true;
+        encoder_last_error_ = PP_OK;
+
+        ppapi::host::ReplyMessageContext reply_context =
+            context->MakeReplyMessageContext();
+        reply_context.params.AppendHandle(SerializedHandle(
+            renderer_ppapi_host_->ShareSharedMemoryHandleWithRemote(
+                audio_buffer_manager_->shm()->handle()),
+            audio_buffer_manager_->shm()->mapped_size()));
+        reply_context.params.AppendHandle(SerializedHandle(
+            renderer_ppapi_host_->ShareSharedMemoryHandleWithRemote(
+                bitstream_buffer_manager_->shm()->handle()),
+            bitstream_buffer_manager_->shm()->mapped_size()));
+
+        host()->SendReply(reply_context,
+                          PpapiPluginMsg_AudioEncoder_InitializeReply(
+                              encoder_->GetNumberOfSamplesPerFrame(),
+                              audio_buffer_manager_->number_of_buffers(),
+                              audio_buffer_manager_->buffer_size(),
+                              bitstream_buffer_manager_->number_of_buffers(),
+                              bitstream_buffer_manager_->buffer_size()));
+
+        return PP_OK_COMPLETIONPENDING;
+      }
+      error = PP_ERROR_NOMEMORY;
+    } else
+      error = PP_ERROR_FAILED;
+  }
+
+  encoder_ = nullptr;
+
+  return error;
+}
+
+int32_t PepperAudioEncoderHost::OnHostMsgEncode(
+    ppapi::host::HostMessageContext* context,
+    int32_t buffer_id) {
+  if (encoder_last_error_)
+    return encoder_last_error_;
+
+  if (buffer_id < 0 || buffer_id >= audio_buffer_manager_->number_of_buffers())
+    return PP_ERROR_BADARGUMENT;
+
+  audio_buffer_manager_->EnqueueBuffer(buffer_id);
+
+  DoEncode();
+
+  return PP_OK_COMPLETIONPENDING;
+}
+
+int32_t PepperAudioEncoderHost::OnHostMsgRecycleBitstreamBuffer(
+    ppapi::host::HostMessageContext* context,
+    int32_t buffer_id) {
+  if (encoder_last_error_)
+    return encoder_last_error_;
+
+  if (buffer_id < 0 ||
+      buffer_id >= bitstream_buffer_manager_->number_of_buffers())
+    return PP_ERROR_BADARGUMENT;
+
+  bitstream_buffer_manager_->EnqueueBuffer(buffer_id);
+
+  DoEncode();
+
+  return PP_OK;
+}
+
+int32_t PepperAudioEncoderHost::OnHostMsgRequestBitrateChange(
+    ppapi::host::HostMessageContext* context,
+    uint32_t bitrate) {
+  if (encoder_last_error_)
+    return encoder_last_error_;
+
+  media_task_runner_->PostTask(
+      FROM_HERE, base::Bind(&AudioEncoderImpl::RequestBitrateChange,
+                            base::Unretained(encoder_.get()), bitrate));
+
+  return PP_OK;
+}
+
+int32_t PepperAudioEncoderHost::OnHostMsgClose(
+    ppapi::host::HostMessageContext* context) {
+  encoder_last_error_ = PP_ERROR_FAILED;
+  Close();
+
+  return PP_OK;
+}
+
+void PepperAudioEncoderHost::GetSupportedProfiles(
+    std::vector<PP_AudioProfileDescription>* profiles) {
+  DCHECK(RenderThreadImpl::current());
+
+  AudioEncoderImpl software_encoder;
+  *profiles = software_encoder.GetSupportedProfiles();

[bbudge, 2015/11/12 20:31:31]

Looks like this should be a static method.

[llandwerlin-old, 2015/11/13 12:46:23]

Done.

+}
+
+bool PepperAudioEncoderHost::IsInitializationValid(
+    const ppapi::proxy::PPB_AudioEncodeParameters& parameters) {
+  DCHECK(RenderThreadImpl::current());
+
+  std::vector<PP_AudioProfileDescription> profiles;
+  GetSupportedProfiles(&profiles);
+
+  for (const PP_AudioProfileDescription& profile : profiles) {
+    if (parameters.output_profile == profile.profile &&
+        parameters.input_sample_size == profile.sample_size &&
+        parameters.input_sample_rate == profile.sample_rate &&
+        parameters.channels <= profile.max_channels &&
+        PP_HardwareAccelerationCompatible(
+            profile.hardware_accelerated == PP_TRUE ? true : false,
+            parameters.acceleration))
+      return true;
+  }
+
+  return false;
+}
+
+bool PepperAudioEncoderHost::AllocateBuffers() {
+  DCHECK(RenderThreadImpl::current());
+  DCHECK(encoder_);
+
+  base::CheckedNumeric<size_t> buffer_size =
+      encoder_->GetNumberOfSamplesPerFrame();
+  buffer_size *= encode_parameters_.channels;
+  buffer_size *= encode_parameters_.input_sample_size;
+
+  return AllocateAudioBuffers(buffer_size) &&
+         AllocateBitstreamBuffers(buffer_size);

[bbudge, 2015/11/12 20:31:31]

I think it would be better to inline these function calls. As it is you may
leave one buffer initialized and another uninitialized if one call fails.

Also, it's strange to pass a base::CheckedNumeric as a function parameter. I
can't find any other instances of this in Chromium.

Doing everything inline should shorten the calculations and will also make it
clearer how the audio and bitstream buffer sizes are related.

[llandwerlin-old, 2015/11/13 12:46:23]

Done.

+}
+
+bool PepperAudioEncoderHost::AllocateAudioBuffers(
+    const base::CheckedNumeric<size_t>& buffer_size) {
+  base::CheckedNumeric<size_t> total_buffer_size = buffer_size;
+  total_buffer_size += sizeof(ppapi::MediaStreamBuffer::Audio);
+  if (!total_buffer_size.IsValid())
+    return false;
+
+  base::CheckedNumeric<size_t> total_memory_size =
+      total_buffer_size.ValueOrDie();
+  total_memory_size *= kDefaultNumberOfAudioBuffers;
+  if (!total_memory_size.IsValid())
+    return false;
+
+  if (!audio_buffer_manager_->SetBuffers(
+          kDefaultNumberOfAudioBuffers, total_buffer_size.ValueOrDie(),
+          RenderThreadImpl::current()
+              ->HostAllocateSharedMemoryBuffer(total_memory_size.ValueOrDie())

[bbudge, 2015/11/12 20:31:31]

This could return a null pointer, and SetBuffers doesn't check for that, so you
need a null check here.

[llandwerlin-old, 2015/11/13 12:46:23]

Done.

+              .Pass(),
+          false))
+    return false;
+
+  for (int32_t i = 0; i < audio_buffer_manager_->number_of_buffers(); ++i) {
+    ppapi::MediaStreamBuffer::Audio* buffer =
+        &(audio_buffer_manager_->GetBufferPointer(i)->audio);
+    buffer->header.size = total_buffer_size.ValueOrDie();
+    buffer->header.type = ppapi::MediaStreamBuffer::TYPE_AUDIO;
+    buffer->sample_rate = static_cast<PP_AudioBuffer_SampleRate>(
+        encode_parameters_.input_sample_rate);
+    buffer->number_of_channels = encode_parameters_.channels;
+    buffer->number_of_samples = encoder_->GetNumberOfSamplesPerFrame();
+    buffer->data_size = buffer_size.ValueOrDie();
+  }
+
+  return true;
+}
+
+bool PepperAudioEncoderHost::AllocateBitstreamBuffers(
+    const base::CheckedNumeric<size_t>& buffer_size) {
+  // Individual buffers are twice the size of the raw data.
+  base::CheckedNumeric<size_t> bitstream_buffer_size = buffer_size;
+  bitstream_buffer_size *= 2;
+  if (!bitstream_buffer_size.IsValid())
+    return false;
+
+  base::CheckedNumeric<size_t> total_size = bitstream_buffer_size;
+  total_size *= kDefaultNumberOfAudioBuffers;
+  if (!total_size.IsValid())
+    return false;
+
+  if (!bitstream_buffer_manager_->SetBuffers(
+          kDefaultNumberOfAudioBuffers, bitstream_buffer_size.ValueOrDie(),
+          RenderThreadImpl::current()
+              ->HostAllocateSharedMemoryBuffer(total_size.ValueOrDie())
+              .Pass(),
+          true))
+    return false;
+
+  return true;
+}
+
+void PepperAudioEncoderHost::DoEncode() {
+  if (!audio_buffer_manager_->HasAvailableBuffer() ||
+      !bitstream_buffer_manager_->HasAvailableBuffer())
+    return;
+
+  int32_t audio_buffer_id, bitstream_buffer_id;
+  AudioData input(
+      GetAudioDataFromBufferManager(audio_buffer_manager_, &audio_buffer_id));
+  AudioData output(GetBitstreamDataFromBufferManager(bitstream_buffer_manager_,
+                                                     &bitstream_buffer_id));

[bbudge, 2015/11/12 20:31:31]

I think this would be clearer if you just inline the functions here.

Also, I would consider eliminating the AudioData struct and passing the
parameters separately. Is there an issue with parameter count in the binding
code?

[llandwerlin-old, 2015/11/13 12:46:23]

Done.

+
+  media_task_runner_->PostTask(
+      FROM_HERE, base::Bind(&AudioEncoderImpl::Encode,
+                            base::Unretained(encoder_.get()), input, output,
+                            media::BindToCurrentLoop(base::Bind(
+                                &PepperAudioEncoderHost::BitstreamBufferReady,
+                                weak_ptr_factory_.GetWeakPtr(), audio_buffer_id,
+                                bitstream_buffer_id))));
+}
+
+void PepperAudioEncoderHost::BitstreamBufferReady(int32_t audio_buffer_id,
+                                                  int32_t bitstream_buffer_id,
+                                                  int32_t size) {
+  DCHECK(RenderThreadImpl::current());
+
+  if (encoder_last_error_)
+    return;
+
+  host()->SendUnsolicitedReply(
+      pp_resource(), PpapiPluginMsg_AudioEncoder_EncodeReply(audio_buffer_id));
+
+  if (size < 0) {
+    NotifyPepperError(PP_ERROR_FAILED);
+    return;
+  }
+
+  host()->SendUnsolicitedReply(
+      pp_resource(), PpapiPluginMsg_AudioEncoder_BitstreamBufferReady(
+                         bitstream_buffer_id, static_cast<uint32_t>(size)));
+}
+
+void PepperAudioEncoderHost::NotifyPepperError(int32_t error) {
+  DCHECK(RenderThreadImpl::current());
+
+  encoder_last_error_ = error;
+  Close();
+  host()->SendUnsolicitedReply(
+      pp_resource(),
+      PpapiPluginMsg_AudioEncoder_NotifyError(encoder_last_error_));
+}
+
+void PepperAudioEncoderHost::Close() {
+  DCHECK(RenderThreadImpl::current());
+
+  // Destroy the encoder and the audio/bitstream buffers on the media thread
+  // to avoid freeing memory the encoder might still be working on.
+  media_task_runner_->PostTask(
+      FROM_HERE, base::Bind(&StopAudioEncoder, base::Passed(encoder_.Pass()),
+                            base::Passed(audio_buffer_manager_.Pass()),
+                            base::Passed(bitstream_buffer_manager_.Pass())));
+}
+
+}  // namespace content