ppapi: add AudioEncode example

ppapi/examples/audio_encode/audio_encode.cc

+// Copyright 2015 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include <stdlib.h>
+#include <string.h>
+
+#include <algorithm>
+#include <limits>
+#include <vector>
+
+#include "ppapi/cpp/audio_buffer.h"
+#include "ppapi/cpp/audio_encoder.h"
+#include "ppapi/cpp/instance.h"
+#include "ppapi/cpp/logging.h"
+#include "ppapi/cpp/media_stream_audio_track.h"
+#include "ppapi/cpp/module.h"
+#include "ppapi/cpp/var_array_buffer.h"
+#include "ppapi/cpp/var_dictionary.h"
+#include "ppapi/utility/completion_callback_factory.h"
+
+// When compiling natively on Windows, PostMessage can be #define-d to
+// something else.
+#ifdef PostMessage
+#undef PostMessage
+#endif
+
+// This example demonstrates receiving audio samples from an audio
+// track and encoding them.
+
+namespace {
+
+class AudioEncoderInstance : public pp::Instance {
+ public:
+  explicit AudioEncoderInstance(PP_Instance instance);
+  virtual ~AudioEncoderInstance();
+
+ private:
+  struct FrameDescription {

[bbudge, 2015/11/24 19:20:58]

I'm not sure I see the utility of this struct. You don't really pass these
around, so it would be less code to just have 4 member fields on the instance
class (i.e. channels_, sample_rate_, sample_size_, and samples_per_frame_.)

[llandwerlin-old, 2015/11/25 18:11:46]

Done.

+    FrameDescription()
+        : channels(0), sample_rate(0), sample_size(0), samples_per_frame(0) {}
+    FrameDescription(uint32_t channels,
+                     uint32_t sample_rate,
+                     uint32_t sample_size,
+                     uint32_t samples_per_frame)
+        : channels(channels),
+          sample_rate(sample_rate),
+          sample_size(sample_size),
+          samples_per_frame(samples_per_frame) {}
+    ~FrameDescription() {}
+
+    uint32_t channels;
+    uint32_t sample_rate;
+    uint32_t sample_size;
+    uint32_t samples_per_frame;
+  };
+
+  //

[bbudge, 2015/11/24 19:20:58]

delete

[llandwerlin-old, 2015/11/25 18:11:47]

Done.

+  virtual void HandleMessage(const pp::Var& var_message);
+
+  void AddAudioProfile(PP_AudioProfile profile, const std::string& profile_str);
+  void InitializeVideoProfiles();

[bbudge, 2015/11/24 19:20:58]

InitializeAudioProfiles

[llandwerlin-old, 2015/11/25 18:11:46]

Done.

+  PP_AudioProfile ProfileFromString(const std::string& string);
+  std::string ProfileToString(PP_AudioProfile profile);
+
+  void ProbeEncoder();

[bbudge, 2015/11/24 19:20:58]

Maybe a better name is GetProfiles or GetSupportedProfiles?

[llandwerlin-old, 2015/11/25 18:11:46]

Done.

+  void OnEncoderProbed(int32_t result,
+                       const std::vector<PP_AudioProfileDescription> profiles);
+
+  void StartAudioTrack(const pp::Resource& resource_track);
+  void StopAudioTrack();
+  void OnGetFirstBuffer(int32_t result, pp::AudioBuffer buffer);
+  void OnGetBuffer(int32_t result, pp::AudioBuffer buffer);
+
+  void InitializeEncoder();
+  void OnEncoderInitialized(int32_t result);
+  void OnAudioTrackReconfigured(int32_t result);
+  void GetEncoderBuffer(const pp::AudioBuffer& track_buffer);
+  void OnGetEncoderBuffer(int32_t result,
+                          pp::AudioBuffer buffer,
+                          pp::AudioBuffer track_buffer);
+  void OnEncodeDone(int32_t result);
+  void OnBitstreamBuffer(int32_t result, const PP_AudioBitstreamBuffer& buffer);

[bbudge, 2015/11/24 19:20:58]

OnGetBitstreamBuffer for consistency?

[llandwerlin-old, 2015/11/25 18:11:47]

Done.

+
+  pp::VarDictionary NewCommand(const std::string& type);
+
+  void Log(const std::string& message);
+  void LogError(int32_t error, const std::string& message);
+
+  void PostAudioFormat();
+  void PostDataMessage(const void* data, uint32_t size);
+
+  pp::CompletionCallbackFactory<AudioEncoderInstance> callback_factory_;
+
+  bool no_encoding_;

[bbudge, 2015/11/24 19:20:59]

A comment explaining what this is for?

[llandwerlin-old, 2015/11/25 18:11:46]

Done.

+  bool is_encoding_;
+  PP_AudioProfile encoding_profile_;
+
+  FrameDescription frame_description_;
+
+  pp::MediaStreamAudioTrack audio_track_;
+  pp::AudioEncoder audio_encoder_;
+
+  typedef std::map<std::string, PP_AudioProfile> AudioProfileFromStringMap;
+  AudioProfileFromStringMap profile_from_string_;
+
+  typedef std::map<PP_AudioProfile, std::string> AudioProfileToStringMap;
+  AudioProfileToStringMap profile_to_string_;
+};
+
+AudioEncoderInstance::AudioEncoderInstance(PP_Instance instance)
+    : pp::Instance(instance),
+      callback_factory_(this),
+      no_encoding_(true),
+      is_encoding_(false),
+      encoding_profile_(PP_AUDIOPROFILE_OPUS) {
+  InitializeVideoProfiles();
+  ProbeEncoder();
+}
+
+AudioEncoderInstance::~AudioEncoderInstance() {}
+
+void AudioEncoderInstance::HandleMessage(const pp::Var& var_message) {
+  if (!var_message.is_dictionary()) {
+    LogError(PP_ERROR_FAILED,
+             "Cannot handle incoming message, not a dictionary");
+    return;
+  }
+
+  pp::VarDictionary dict_message(var_message);
+  std::string command = dict_message.Get("command").AsString();
+
+  if (command == "start") {
+    pp::Var var_track = dict_message.Get("track");
+    if (!var_track.is_resource()) {
+      LogError(PP_ERROR_FAILED, "Given track is not a resource");
+      return;
+    }
+    std::string profile = dict_message.Get("profile").AsString();
+    if (profile == "wav")
+      no_encoding_ = true;
+    else {
+      no_encoding_ = false;
+      encoding_profile_ = ProfileFromString(profile);
+    }
+    StartAudioTrack(var_track.AsResource());
+  } else if (command == "stop") {
+    StopAudioTrack();
+  } else {
+    LogError(PP_ERROR_FAILED, "Invalid command");
+  }
+}
+
+void AudioEncoderInstance::AddAudioProfile(PP_AudioProfile profile,
+                                           const std::string& profile_str) {
+  profile_to_string_.insert(std::make_pair(profile, profile_str));
+  profile_from_string_.insert(std::make_pair(profile_str, profile));
+}
+
+void AudioEncoderInstance::InitializeVideoProfiles() {
+  AddAudioProfile(PP_AUDIOPROFILE_OPUS, "opus");
+}
+
+PP_AudioProfile AudioEncoderInstance::ProfileFromString(
+    const std::string& string) {
+  AudioProfileFromStringMap::iterator it = profile_from_string_.find(string);
+  if (it == profile_from_string_.end())
+    return PP_AUDIOPROFILE_OPUS;
+  return it->second;
+}
+
+std::string AudioEncoderInstance::ProfileToString(PP_AudioProfile profile) {
+  AudioProfileToStringMap::iterator it = profile_to_string_.find(profile);
+  if (it == profile_to_string_.end())
+    return "unknown";
+  return it->second;
+}
+
+void AudioEncoderInstance::ProbeEncoder() {
+  audio_encoder_ = pp::AudioEncoder(this);
+  audio_encoder_.GetSupportedProfiles(callback_factory_.NewCallbackWithOutput(
+      &AudioEncoderInstance::OnEncoderProbed));
+}
+
+void AudioEncoderInstance::OnEncoderProbed(
+    int32_t result,
+    const std::vector<PP_AudioProfileDescription> profiles) {
+  pp::VarDictionary dictionary = NewCommand("supportedProfiles");
+  pp::VarArray js_profiles;
+  dictionary.Set(pp::Var("profiles"), js_profiles);
+
+  if (result < 0) {
+    LogError(result, "Cannot get supported profiles");
+    PostMessage(dictionary);
+  }
+
+  int32_t idx = 0;
+  for (std::vector<PP_AudioProfileDescription>::const_iterator it =
+           profiles.begin();
+       it != profiles.end(); ++it) {
+    pp::VarDictionary js_profile;
+    js_profile.Set(pp::Var("name"), pp::Var(ProfileToString(it->profile)));
+    js_profile.Set(pp::Var("sample_size"),
+                   pp::Var(static_cast<int32_t>(it->sample_size)));
+    js_profile.Set(pp::Var("sample_rate"),
+                   pp::Var(static_cast<int32_t>(it->sample_rate)));
+    js_profiles.Set(idx++, js_profile);
+  }
+  PostMessage(dictionary);
+}
+
+void AudioEncoderInstance::StartAudioTrack(const pp::Resource& resource_track) {
+  audio_track_ = pp::MediaStreamAudioTrack(resource_track);
+  audio_track_.GetBuffer(callback_factory_.NewCallbackWithOutput(
+      &AudioEncoderInstance::OnGetFirstBuffer));
+}
+
+void AudioEncoderInstance::StopAudioTrack() {
+  Log("StopAudioTrack!");
+  audio_encoder_.Close();
+  audio_encoder_ = pp::AudioEncoder();
+  audio_track_.Close();
+  audio_track_ = pp::MediaStreamAudioTrack();
+}
+
+void AudioEncoderInstance::OnGetFirstBuffer(int32_t result,

[bbudge, 2015/11/24 19:20:59]

This is very similar to OnGetBuffer. Could you use a sentinel to detect when you
need to initialize frame_description? This could be a boolean (e.g.
got_first_buffer_) or you could detect an unitialized FrameDescription (e.g.
frame_description_.channels == 0).

[llandwerlin-old, 2015/11/25 18:11:46]

Done.

+                                            pp::AudioBuffer buffer) {
+  if (result == PP_ERROR_ABORTED)
+    return;
+  if (result != PP_OK) {
+    LogError(result, "Cannot get audio track buffer");
+    return;
+  }
+
+  frame_description_ =
+      FrameDescription(buffer.GetNumberOfChannels(), buffer.GetSampleRate(),
+                       buffer.GetSampleSize(), buffer.GetNumberOfSamples());
+
+  if (no_encoding_) {
+    PostAudioFormat();
+    PostDataMessage(buffer.GetDataBuffer(), buffer.GetDataBufferSize());
+    audio_track_.GetBuffer(callback_factory_.NewCallbackWithOutput(
+        &AudioEncoderInstance::OnGetBuffer));
+  } else {
+    InitializeEncoder();
+  }
+
+  audio_track_.RecycleBuffer(buffer);
+}
+
+void AudioEncoderInstance::OnGetBuffer(int32_t result, pp::AudioBuffer buffer) {
+  if (result == PP_ERROR_ABORTED)
+    return;

[bbudge, 2015/12/04 00:59:12]

see comment below.

[llandwerlin-old, 2015/12/08 12:29:22]

This is to avoid to report an error that isn't one. This was just aborted by the
user.

+  if (result != PP_OK) {
+    LogError(result, "Cannot get audio track buffer");
+    return;
+  }
+
+  if (no_encoding_) {
+    PostDataMessage(buffer.GetDataBuffer(), buffer.GetDataBufferSize());
+    audio_track_.RecycleBuffer(buffer);
+  } else
+    GetEncoderBuffer(buffer);
+
+  audio_track_.GetBuffer(callback_factory_.NewCallbackWithOutput(
+      &AudioEncoderInstance::OnGetBuffer));
+}
+
+void AudioEncoderInstance::InitializeEncoder() {
+  audio_encoder_ = pp::AudioEncoder(this);
+  audio_encoder_.Initialize(
+      frame_description_.channels,
+      static_cast<PP_AudioBuffer_SampleRate>(frame_description_.sample_rate),
+      static_cast<PP_AudioBuffer_SampleSize>(frame_description_.sample_size),
+      encoding_profile_, 100000, PP_HARDWAREACCELERATION_WITHFALLBACK,
+      callback_factory_.NewCallback(
+          &AudioEncoderInstance::OnEncoderInitialized));
+}
+
+void AudioEncoderInstance::OnEncoderInitialized(int32_t result) {
+  if (result == PP_ERROR_ABORTED)
+    return;
+  if (result != PP_OK) {
+    LogError(result, "Cannot initialize encoder");
+    return;
+  }
+
+  int32_t attribs[] = {// Duration in milliseconds.
+                       PP_MEDIASTREAMAUDIOTRACK_ATTRIB_DURATION,
+                       1000 / (frame_description_.sample_rate /
+                               audio_encoder_.GetNumberOfSamples()),
+                       PP_MEDIASTREAMAUDIOTRACK_ATTRIB_NONE};
+  audio_track_.Configure(attribs,
+                         callback_factory_.NewCallback(
+                             &AudioEncoderInstance::OnAudioTrackReconfigured));

[bbudge, 2015/11/24 19:20:58]

OnAudioTrackConfigured seems like a better name here.

[llandwerlin-old, 2015/11/25 18:11:46]

Done.

+
+  frame_description_.samples_per_frame = audio_encoder_.GetNumberOfSamples();
+  PostAudioFormat();
+}
+
+void AudioEncoderInstance::OnAudioTrackReconfigured(int32_t result) {
+  if (result == PP_ERROR_ABORTED)
+    return;

[bbudge, 2015/11/24 19:20:59]

Is it necessary to handle this differently than other errors here?

[llandwerlin-old, 2015/11/25 18:11:46]

I'm not sure what you think is different from the other errors here.

[bbudge, 2015/12/04 00:59:12]

This is from the MediaStreamAudioTrack. Does it normally return PP_ERROR_ABORTED
like audio/video decoder/encoder resources can when Closing? Normally Pepper
resources only abort when they are destroyed.

Same thing for the first lines of AudioEncoderInstance::OnGetBuffer.

[llandwerlin-old, 2015/12/08 12:29:22]

I get it now. I want to avoid to display an error in the aborted case because
it's triggered by the user when the encoding/recording stops.

+  if (result != PP_OK) {
+    LogError(result, "Cannot reconfigure audio track buffer duration");
+    return;
+  }
+
+  is_encoding_ = true;
+  audio_encoder_.GetBitstreamBuffer(callback_factory_.NewCallbackWithOutput(
+      &AudioEncoderInstance::OnBitstreamBuffer));
+  audio_track_.GetBuffer(callback_factory_.NewCallbackWithOutput(
+      &AudioEncoderInstance::OnGetBuffer));

[bbudge, 2015/11/24 19:20:58]

nit: It seems clearer and more consistent with encoder flow to request the audio
buffer before the bitstream buffer.

[llandwerlin-old, 2015/11/25 18:11:46]

Done.

+}
+
+void AudioEncoderInstance::GetEncoderBuffer(
+    const pp::AudioBuffer& track_buffer) {
+  audio_encoder_.GetBuffer(callback_factory_.NewCallbackWithOutput(
+      &AudioEncoderInstance::OnGetEncoderBuffer, track_buffer));
+}
+
+void AudioEncoderInstance::OnGetEncoderBuffer(int32_t result,
+                                              pp::AudioBuffer buffer,
+                                              pp::AudioBuffer track_buffer) {
+  if (result == PP_ERROR_ABORTED) {
+    audio_track_.RecycleBuffer(track_buffer);
+    return;
+  }
+  if (result != PP_OK) {
+    audio_track_.RecycleBuffer(track_buffer);
+    LogError(result, "Cannot get encoder buffer");
+    return;
+  }
+
+  if (buffer.GetDataBufferSize() != track_buffer.GetDataBufferSize()) {

[bbudge, 2015/11/24 19:20:59]

This seems like a good defensive check, but isn't this a pretty serious failure,
and shouldn't we stop encoding in this case?

[llandwerlin-old, 2015/11/25 18:11:46]

Done.

+    audio_track_.RecycleBuffer(track_buffer);
+    LogError(PP_ERROR_FAILED, "Invalid buffer size");
+    return;
+  }
+
+  memcpy(buffer.GetDataBuffer(), track_buffer.GetDataBuffer(),
+         buffer.GetDataBufferSize());
+  audio_track_.RecycleBuffer(track_buffer);
+
+  audio_encoder_.Encode(buffer, callback_factory_.NewCallback(
+                                    &AudioEncoderInstance::OnEncodeDone));

[bbudge, 2015/11/24 19:20:58]

I think you could eliminate a lot of code duplication in this method by not
trying to return early in the error case. Like this:

if (buffer.GetDataBufferSize() != track_buffer.GetDataBufferSize())
  result = PP_ERROR_FAILED;

if (result == PP_OK) {
  memcpy...
  audio_encoder_.Encode...
} else {
  LogError...
}

audio_track_.RecycleBuffer(track_buffer);

If you want the special error message, you could introduce a char* error_message
local and set it appropriately.

[llandwerlin-old, 2015/11/25 18:11:46]

Done.

+}
+
+void AudioEncoderInstance::OnEncodeDone(int32_t result) {}
+
+void AudioEncoderInstance::OnBitstreamBuffer(
+    int32_t result,
+    const PP_AudioBitstreamBuffer& buffer) {
+  if (result == PP_ERROR_ABORTED)
+    return;
+  if (result != PP_OK) {
+    LogError(result, "Cannot get bitstream buffer");
+    return;
+  }
+
+  audio_encoder_.GetBitstreamBuffer(callback_factory_.NewCallbackWithOutput(
+      &AudioEncoderInstance::OnBitstreamBuffer));
+
+  PostDataMessage(buffer.buffer, buffer.size);
+  audio_encoder_.RecycleBitstreamBuffer(buffer);
+}
+
+pp::VarDictionary AudioEncoderInstance::NewCommand(const std::string& type) {
+  pp::VarDictionary dictionary;
+  dictionary.Set(pp::Var("command"), pp::Var(type));
+  return dictionary;
+}
+
+void AudioEncoderInstance::Log(const std::string& message) {
+  pp::VarDictionary dictionary = NewCommand("log");
+  dictionary.Set(pp::Var("message"), pp::Var(message));
+  PostMessage(dictionary);
+}
+
+void AudioEncoderInstance::LogError(int32_t error, const std::string& message) {
+  pp::VarDictionary dictionary = NewCommand("error");
+  std::string msg("Error: ");
+  msg.append(message);
+  msg.append(" : ");
+  msg.append(pp::Var(error).DebugString());
+  dictionary.Set(pp::Var("message"), pp::Var(msg));
+  PostMessage(dictionary);
+}
+
+void AudioEncoderInstance::PostAudioFormat() {
+  pp::VarDictionary dictionary = NewCommand("format");
+  dictionary.Set(pp::Var("channels"),
+                 pp::Var(static_cast<int32_t>(frame_description_.channels)));
+  dictionary.Set(pp::Var("sample_rate"),
+                 pp::Var(static_cast<int32_t>(frame_description_.sample_rate)));
+  dictionary.Set(pp::Var("sample_size"),
+                 pp::Var(static_cast<int32_t>(frame_description_.sample_size)));
+  dictionary.Set(
+      pp::Var("sample_per_frame"),
+      pp::Var(static_cast<int32_t>(frame_description_.samples_per_frame)));
+  PostMessage(dictionary);
+}
+
+void AudioEncoderInstance::PostDataMessage(const void* data, uint32_t size) {
+  pp::VarDictionary dictionary = NewCommand("data");
+  uint8_t* buffer;
+
+  pp::VarArrayBuffer array_buffer = pp::VarArrayBuffer(size);
+  buffer = static_cast<uint8_t*>(array_buffer.Map());
+  memcpy(buffer, data, size);
+  array_buffer.Unmap();
+
+  dictionary.Set(pp::Var("buffer"), array_buffer);
+
+  PostMessage(dictionary);
+}
+
+class AudioEncoderModule : public pp::Module {
+ public:
+  virtual pp::Instance* CreateInstance(PP_Instance instance) {
+    return new AudioEncoderInstance(instance);
+  }
+};
+
+}  // namespace
+
+namespace pp {
+
+// Factory function for your specialization of the Module object.
+Module* CreateModule() {
+  return new AudioEncoderModule();
+}
+
+}  // namespace pp