Implement Pepper PPB_VideoDecoder interface.

ppapi/examples/video_decode/video_decode.cc

+// Copyright (c) 2014 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include <stdio.h>
+#include <string.h>
+
+#include <iostream>
+#include <queue>
+#include <sstream>
+
+#include "ppapi/c/pp_errors.h"
+#include "ppapi/c/ppb_console.h"
+#include "ppapi/c/ppb_opengles2.h"
+#include "ppapi/cpp/graphics_3d.h"
+#include "ppapi/cpp/graphics_3d_client.h"
+#include "ppapi/cpp/input_event.h"
+#include "ppapi/cpp/instance.h"
+#include "ppapi/cpp/module.h"
+#include "ppapi/cpp/rect.h"
+#include "ppapi/cpp/var.h"
+#include "ppapi/cpp/video_decoder.h"
+#include "ppapi/examples/video_decode/testdata.h"
+#include "ppapi/lib/gl/include/GLES2/gl2.h"
+#include "ppapi/lib/gl/include/GLES2/gl2ext.h"
+#include "ppapi/utility/completion_callback_factory.h"
+
+// Use assert as a poor-man's CHECK, even in non-debug mode.
+// Since <assert.h> redefines assert on every inclusion (it doesn't use
+// include-guards), make sure this is the last file #include'd in this file.
+#undef NDEBUG
+#include <assert.h>
+
+// Assert |context_| isn't holding any GL Errors.  Done as a macro instead of a
+// function to preserve line number information in the failure message.
+#define assertNoGLError() assert(!gles2_if_->GetError(context_->pp_resource()));
+
+namespace {
+
+struct Shader {
+  Shader() : program(0), texcoord_scale_location(0) {}
+  ~Shader() {}
+
+  GLuint program;
+  GLint texcoord_scale_location;
+};
+
+class Decoder;
+class MyInstance;
+
+class MyInstance : public pp::Instance, public pp::Graphics3DClient {
+ public:
+  MyInstance(PP_Instance instance, pp::Module* module);
+  virtual ~MyInstance();
+
+  // pp::Instance implementation.
+  virtual void DidChangeView(const pp::Rect& position,
+                             const pp::Rect& clip_ignored);
+
+  // pp::Graphics3DClient implementation.
+  virtual void Graphics3DContextLost() {
+    // TODO(vrk/fischman): Properly reset after a lost graphics context.  In
+    // particular need to delete context_ and re-create textures.
+    // Probably have to recreate the decoder from scratch, because old textures
+    // can still be outstanding in the decoder!
+    assert(false && "Unexpectedly lost graphics context");
+  }
+
+  void PaintPicture(Decoder* decoder, const PP_VideoPicture& picture);
+
+ private:
+  // Log an error to the developer console and stderr by creating a temporary
+  // object of this type and streaming to it.  Example usage:
+  // LogError(this).s() << "Hello world: " << 42;
+  class LogError {
+   public:
+    LogError(MyInstance* instance) : instance_(instance) {}
+    ~LogError() {
+      const std::string& msg = stream_.str();
+      instance_->console_if_->Log(
+          instance_->pp_instance(), PP_LOGLEVEL_ERROR, pp::Var(msg).pp_var());
+      std::cerr << msg << std::endl;
+    }
+    // Impl note: it would have been nicer to have LogError derive from
+    // std::ostringstream so that it can be streamed to directly, but lookup
+    // rules turn streamed string literals to hex pointers on output.
+    std::ostringstream& s() { return stream_; }
+
+   private:
+    MyInstance* instance_;
+    std::ostringstream stream_;
+  };
+
+  void InitializeDecoders();
+
+  // GL-related functions.
+  void InitGL();
+  void CreateGLObjects();
+  void Create2DProgramOnce();
+  void CreateRectangleARBProgramOnce();
+  Shader CreateProgram(const char* vertex_shader, const char* fragment_shader);
+  void CreateShader(GLuint program, GLenum type, const char* source, int size);
+  void PaintFinished(int32_t result, Decoder* decoder, PP_VideoPicture picture);
+
+  pp::Size plugin_size_;
+  bool is_painting_;
+  // When decode outpaces render, we queue up decoded pictures for later
+  // painting.  Elements are <decoder,picture>.
+  typedef std::queue<std::pair<Decoder*, PP_VideoPicture> > PictureQueue;
+  PictureQueue pictures_pending_paint_;
+
+  int num_frames_rendered_;
+  PP_TimeTicks first_frame_delivered_ticks_;
+  PP_TimeTicks last_swap_request_ticks_;
+  PP_TimeTicks swap_ticks_;
+  pp::CompletionCallbackFactory<MyInstance> callback_factory_;
+
+  // Unowned pointers.
+  const PPB_Console* console_if_;
+  const PPB_Core* core_if_;
+  const PPB_OpenGLES2* gles2_if_;
+
+  // Owned data.
+  pp::Graphics3D* context_;
+  typedef std::vector<Decoder*> DecoderList;
+  DecoderList video_decoders_;
+
+  // Shader program to draw GL_TEXTURE_2D target.
+  Shader shader_2d_;
+  // Shader program to draw GL_TEXTURE_RECTANGLE_ARB target.
+  Shader shader_rectangle_arb_;
+};
+
+class Decoder {
+ public:
+  Decoder(MyInstance* instance, int id, const pp::Graphics3D& graphics_3d);
+  ~Decoder();
+
+  int id() const { return id_; }
+  bool ready() const { return !flushing_ && !resetting_; }
+
+  void Seek(int frame);
+  void RecyclePicture(const PP_VideoPicture& picture);
+
+ private:
+  void InitializeDone(int32_t result);
+  void Start(int frame);
+  void DecodeNextFrames();
+  void DecodeDone(int32_t result);
+  void PictureReady(int32_t result, PP_VideoPicture picture);
+  void FlushDone(int32_t result);
+  void ResetDone(int32_t result);
+
+  MyInstance* instance_;
+  int id_;
+
+  pp::VideoDecoder* decoder_;
+  pp::CompletionCallbackFactory<Decoder> callback_factory_;
+
+  size_t encoded_data_next_pos_to_decode_;
+  int next_picture_id_;
+  int seek_frame_;
+  bool flushing_;
+  bool resetting_;
+};
+
+// Returns true if the current position is at the start of a NAL unit.
+static bool LookingAtNAL(const unsigned char* encoded, size_t pos) {
+  return pos + 3 < kDataLen && encoded[pos] == 0 && encoded[pos + 1] == 0 &&
+         encoded[pos + 2] == 0 && encoded[pos + 3] == 1;
+}
+
+// Find the start and end of the next frame.
+static void GetNextFrame(size_t* start_pos, size_t* end_pos) {
+  // H264 frames start with 0, 0, 0, 1.
+  assert(LookingAtNAL(kData, *start_pos));
+  *end_pos = *start_pos;
+  *end_pos += 4;
+  while (*end_pos + 3 < kDataLen && !LookingAtNAL(kData, *end_pos)) {
+    ++*end_pos;
+  }
+  if (*end_pos + 3 >= kDataLen) {
+    *end_pos = kDataLen;
+  }
+}
+
+Decoder::Decoder(MyInstance* instance,
+                 int id,
+                 const pp::Graphics3D& graphics_3d)
+    : instance_(instance),
+      id_(id),
+      decoder_(new pp::VideoDecoder(instance)),

[yzshen1, 2014/05/14 18:08:23]

It seems this is leaked.

[bbudge, 2014/05/14 19:35:04]

Good catch. Deleted in dtor.

+      callback_factory_(this),
+      encoded_data_next_pos_to_decode_(0),
+      next_picture_id_(0),
+      seek_frame_(0),
+      flushing_(false),
+      resetting_(false) {
+  assert(!decoder_->is_null());
+  const PP_VideoProfile profile = PP_VIDEOPROFILE_H264MAIN;
+  decoder_->Initialize(graphics_3d,
+                       profile,
+                       PP_TRUE /* allow_software_fallback */,
+                       callback_factory_.NewCallback(&Decoder::Start));

[yzshen1, 2014/05/14 18:08:23]

InitializeDone?

[bbudge, 2014/05/14 19:35:04]

Done.

+}
+
+Decoder::~Decoder() {
+}
+
+void Decoder::InitializeDone(int32_t result) {

[yzshen1, 2014/05/14 18:08:23]

Currently this method is not used.

[bbudge, 2014/05/14 19:35:04]

Changed callback to call this.

+  assert(decoder_);
+  assert(result == PP_OK);
+  assert(ready());
+  Start(0);
+}
+
+void Decoder::Start(int frame) {
+  assert(decoder_);
+
+  // Skip to |frame|.
+  size_t start_pos = 0;
+  size_t end_pos = 0;
+  for (int i = frame; i > 0; i--)
+    GetNextFrame(&start_pos, &end_pos);
+  encoded_data_next_pos_to_decode_ = end_pos;
+
+  DecodeNextFrames();
+  // Register callback to get the first picture. We call GetPicture
+  // again in PictureReady to keep getting pictures.
+  decoder_->GetPicture(
+      callback_factory_.NewCallbackWithOutput(&Decoder::PictureReady));
+}
+
+void Decoder::Seek(int frame) {
+  assert(decoder_);
+  seek_frame_ = frame;
+  resetting_ = true;
+  decoder_->Reset(callback_factory_.NewCallback(&Decoder::ResetDone));
+}
+
+void Decoder::RecyclePicture(const PP_VideoPicture& picture) {
+  assert(decoder_);
+  decoder_->RecyclePicture(picture);
+}
+
+void Decoder::DecodeNextFrames() {
+  assert(decoder_);
+  while (encoded_data_next_pos_to_decode_ <= kDataLen) {
+    // If we've just reached the end of the bitstream, flush and wait.
+    if (!flushing_ && encoded_data_next_pos_to_decode_ == kDataLen) {
+      flushing_ = true;
+      decoder_->Flush(callback_factory_.NewCallback(&Decoder::FlushDone));
+      return;
+    }
+
+    // Find the start of the next NALU.
+    size_t start_pos = encoded_data_next_pos_to_decode_;
+    size_t end_pos;
+    GetNextFrame(&start_pos, &end_pos);
+    encoded_data_next_pos_to_decode_ = end_pos;
+    // Decode the NALU.
+    uint32_t size = static_cast<uint32_t>(end_pos - start_pos);
+    int32_t result =
+        decoder_->Decode(next_picture_id_++,
+                         size,
+                         kData + start_pos,
+                         callback_factory_.NewCallback(&Decoder::DecodeDone));
+    if (result != PP_OK) {
+      assert(result == PP_OK_COMPLETIONPENDING);
+      break;
+    }
+  }
+}
+
+void Decoder::DecodeDone(int32_t result) {
+  assert(decoder_);
+  assert(result == PP_OK || result == PP_ERROR_ABORTED);
+  if (result == PP_OK && ready())
+    DecodeNextFrames();
+}
+
+void Decoder::PictureReady(int32_t result, PP_VideoPicture picture) {
+  assert(decoder_);
+  assert(result == PP_OK || result == PP_ERROR_ABORTED);
+  if (result == PP_OK) {
+    decoder_->GetPicture(
+        callback_factory_.NewCallbackWithOutput(&Decoder::PictureReady));
+    instance_->PaintPicture(this, picture);
+  }
+}
+
+void Decoder::FlushDone(int32_t result) {
+  assert(decoder_);
+  assert(result == PP_OK || result == PP_ERROR_ABORTED);
+  flushing_ = false;
+}
+
+void Decoder::ResetDone(int32_t result) {
+  assert(decoder_);
+  assert(result == PP_OK);
+  resetting_ = false;
+}
+
+MyInstance::MyInstance(PP_Instance instance, pp::Module* module)
+    : pp::Instance(instance),
+      pp::Graphics3DClient(this),
+      is_painting_(false),
+      num_frames_rendered_(0),
+      first_frame_delivered_ticks_(-1),

[yzshen1, 2014/05/14 18:08:23]

please also init last_swap_request_ticks_

[bbudge, 2014/05/14 19:35:04]

Done.

+      swap_ticks_(0),
+      callback_factory_(this),
+      context_(NULL) {
+  assert((console_if_ = static_cast<const PPB_Console*>(
+              module->GetBrowserInterface(PPB_CONSOLE_INTERFACE))));
+  assert((core_if_ = static_cast<const PPB_Core*>(
+              module->GetBrowserInterface(PPB_CORE_INTERFACE))));
+  assert((gles2_if_ = static_cast<const PPB_OpenGLES2*>(
+              module->GetBrowserInterface(PPB_OPENGLES2_INTERFACE))));
+  RequestInputEvents(PP_INPUTEVENT_CLASS_MOUSE);
+}
+
+MyInstance::~MyInstance() {
+  if (!context_)
+    return;
+
+  PP_Resource graphics_3d = context_->pp_resource();
+  if (shader_2d_.program)
+    gles2_if_->DeleteProgram(graphics_3d, shader_2d_.program);
+  if (shader_rectangle_arb_.program)
+    gles2_if_->DeleteProgram(graphics_3d, shader_rectangle_arb_.program);
+
+  for (DecoderList::iterator it = video_decoders_.begin();
+       it != video_decoders_.end();
+       ++it)
+    delete *it;
+
+  delete context_;
+}
+
+void MyInstance::DidChangeView(const pp::Rect& position,
+                               const pp::Rect& clip_ignored) {
+  if (position.width() == 0 || position.height() == 0)
+    return;
+  if (plugin_size_.width()) {
+    assert(position.size() == plugin_size_);
+    return;
+  }
+  plugin_size_ = position.size();
+
+  // Initialize graphics.
+  InitGL();
+  InitializeDecoders();
+}
+
+void MyInstance::InitializeDecoders() {
+  assert(video_decoders_.empty());
+  // Create two decoders with ids 0 and 1.
+  video_decoders_.push_back(new Decoder(this, 0, *context_));
+  video_decoders_.push_back(new Decoder(this, 1, *context_));
+}
+
+void MyInstance::PaintPicture(Decoder* decoder,
+                              const PP_VideoPicture& picture) {
+  if (first_frame_delivered_ticks_ == -1)
+    assert((first_frame_delivered_ticks_ = core_if_->GetTimeTicks()) != -1);
+  if (is_painting_) {
+    pictures_pending_paint_.push(std::make_pair(decoder, picture));
+    return;
+  }
+
+  assert(!is_painting_);
+  is_painting_ = true;
+  int x = 0;
+  int y = 0;
+  int half_width = plugin_size_.width() / 2;
+  int half_height = plugin_size_.height() / 2;
+  if (decoder->id() != 0) {
+    x = half_width;
+    y = half_height;
+  }
+
+  PP_Resource graphics_3d = context_->pp_resource();
+  if (picture.texture_target == GL_TEXTURE_2D) {
+    Create2DProgramOnce();
+    gles2_if_->UseProgram(graphics_3d, shader_2d_.program);
+    gles2_if_->Uniform2f(
+        graphics_3d, shader_2d_.texcoord_scale_location, 1.0, 1.0);
+  } else {
+    assert(picture.texture_target == GL_TEXTURE_RECTANGLE_ARB);
+    CreateRectangleARBProgramOnce();
+    gles2_if_->UseProgram(graphics_3d, shader_rectangle_arb_.program);
+    gles2_if_->Uniform2f(graphics_3d,
+                         shader_rectangle_arb_.texcoord_scale_location,
+                         picture.texture_size.width,
+                         picture.texture_size.height);
+  }
+
+  gles2_if_->Viewport(graphics_3d, x, y, half_width, half_height);
+  gles2_if_->ActiveTexture(graphics_3d, GL_TEXTURE0);
+  gles2_if_->BindTexture(
+      graphics_3d, picture.texture_target, picture.texture_id);
+  gles2_if_->DrawArrays(graphics_3d, GL_TRIANGLE_STRIP, 0, 4);
+
+  gles2_if_->UseProgram(graphics_3d, 0);
+
+  last_swap_request_ticks_ = core_if_->GetTimeTicks();
+  assert(PP_OK_COMPLETIONPENDING ==
+         context_->SwapBuffers(callback_factory_.NewCallback(
+             &MyInstance::PaintFinished, decoder, picture)));
+}
+
+void MyInstance::PaintFinished(int32_t result,
+                               Decoder* decoder,
+                               PP_VideoPicture picture) {
+  assert(result == PP_OK);
+  swap_ticks_ += core_if_->GetTimeTicks() - last_swap_request_ticks_;
+  is_painting_ = false;
+  ++num_frames_rendered_;
+  if (num_frames_rendered_ % 50 == 0) {
+    double elapsed = core_if_->GetTimeTicks() - first_frame_delivered_ticks_;
+    double fps = (elapsed > 0) ? num_frames_rendered_ / elapsed : 1000;
+    double ms_per_swap = (swap_ticks_ * 1e3) / num_frames_rendered_;
+    LogError(this).s() << "Rendered frames: " << num_frames_rendered_
+                       << ", fps: " << fps
+                       << ", with average ms/swap of: " << ms_per_swap;
+  }
+  decoder->RecyclePicture(picture);
+  // Keep painting as long as we have pictures.
+  if (!pictures_pending_paint_.empty()) {
+    std::pair<Decoder*, PP_VideoPicture> pending =
+        pictures_pending_paint_.front();
+    pictures_pending_paint_.pop();
+    PaintPicture(pending.first, pending.second);
+  }
+}
+
+void MyInstance::InitGL() {
+  assert(plugin_size_.width() && plugin_size_.height());
+  is_painting_ = false;
+
+  assert(!context_);
+  int32_t context_attributes[] = {
+      PP_GRAPHICS3DATTRIB_ALPHA_SIZE,     8,
+      PP_GRAPHICS3DATTRIB_BLUE_SIZE,      8,
+      PP_GRAPHICS3DATTRIB_GREEN_SIZE,     8,
+      PP_GRAPHICS3DATTRIB_RED_SIZE,       8,
+      PP_GRAPHICS3DATTRIB_DEPTH_SIZE,     0,
+      PP_GRAPHICS3DATTRIB_STENCIL_SIZE,   0,
+      PP_GRAPHICS3DATTRIB_SAMPLES,        0,
+      PP_GRAPHICS3DATTRIB_SAMPLE_BUFFERS, 0,
+      PP_GRAPHICS3DATTRIB_WIDTH,          plugin_size_.width(),
+      PP_GRAPHICS3DATTRIB_HEIGHT,         plugin_size_.height(),
+      PP_GRAPHICS3DATTRIB_NONE,
+  };
+  context_ = new pp::Graphics3D(this, context_attributes);
+  assert(!context_->is_null());
+  assert(BindGraphics(*context_));
+
+  // Clear color bit.
+  gles2_if_->ClearColor(context_->pp_resource(), 1, 0, 0, 1);
+  gles2_if_->Clear(context_->pp_resource(), GL_COLOR_BUFFER_BIT);
+
+  assertNoGLError();
+
+  CreateGLObjects();
+}
+
+void MyInstance::CreateGLObjects() {
+  // Assign vertex positions and texture coordinates to buffers for use in
+  // shader program.
+  static const float kVertices[] = {
+      -1, 1, -1, -1, 1, 1, 1, -1,  // Position coordinates.
+      0,  1, 0,  0,  1, 1, 1, 0,   // Texture coordinates.
+  };
+
+  GLuint buffer;
+  gles2_if_->GenBuffers(context_->pp_resource(), 1, &buffer);
+  gles2_if_->BindBuffer(context_->pp_resource(), GL_ARRAY_BUFFER, buffer);
+
+  gles2_if_->BufferData(context_->pp_resource(),
+                        GL_ARRAY_BUFFER,
+                        sizeof(kVertices),
+                        kVertices,
+                        GL_STATIC_DRAW);
+  assertNoGLError();
+}
+
+static const char kVertexShader[] =
+    "varying vec2 v_texCoord;            \n"
+    "attribute vec4 a_position;          \n"
+    "attribute vec2 a_texCoord;          \n"
+    "uniform vec2 v_scale;               \n"
+    "void main()                         \n"
+    "{                                   \n"
+    "    v_texCoord = v_scale * a_texCoord; \n"
+    "    gl_Position = a_position;       \n"
+    "}";
+
+void MyInstance::Create2DProgramOnce() {
+  if (shader_2d_.program)
+    return;
+  static const char kFragmentShader2D[] =
+      "precision mediump float;            \n"
+      "varying vec2 v_texCoord;            \n"
+      "uniform sampler2D s_texture;        \n"
+      "void main()                         \n"
+      "{"
+      "    gl_FragColor = texture2D(s_texture, v_texCoord); \n"
+      "}";
+  shader_2d_ = CreateProgram(kVertexShader, kFragmentShader2D);
+  assertNoGLError();
+}
+
+void MyInstance::CreateRectangleARBProgramOnce() {
+  if (shader_rectangle_arb_.program)
+    return;
+  static const char kFragmentShaderRectangle[] =
+      "#extension GL_ARB_texture_rectangle : require\n"
+      "precision mediump float;            \n"
+      "varying vec2 v_texCoord;            \n"
+      "uniform sampler2DRect s_texture;    \n"
+      "void main()                         \n"
+      "{"
+      "    gl_FragColor = texture2DRect(s_texture, v_texCoord).rgba; \n"
+      "}";
+  shader_rectangle_arb_ =
+      CreateProgram(kVertexShader, kFragmentShaderRectangle);
+}
+
+Shader MyInstance::CreateProgram(const char* vertex_shader,
+                                 const char* fragment_shader) {
+  Shader shader;
+
+  // Create shader program.
+  shader.program = gles2_if_->CreateProgram(context_->pp_resource());
+  CreateShader(
+      shader.program, GL_VERTEX_SHADER, vertex_shader, strlen(vertex_shader));
+  CreateShader(shader.program,
+               GL_FRAGMENT_SHADER,
+               fragment_shader,
+               strlen(fragment_shader));
+  gles2_if_->LinkProgram(context_->pp_resource(), shader.program);
+  gles2_if_->UseProgram(context_->pp_resource(), shader.program);
+  gles2_if_->Uniform1i(
+      context_->pp_resource(),
+      gles2_if_->GetUniformLocation(
+          context_->pp_resource(), shader.program, "s_texture"),
+      0);
+  assertNoGLError();
+
+  shader.texcoord_scale_location = gles2_if_->GetUniformLocation(
+      context_->pp_resource(), shader.program, "v_scale");
+
+  GLint pos_location = gles2_if_->GetAttribLocation(
+      context_->pp_resource(), shader.program, "a_position");
+  GLint tc_location = gles2_if_->GetAttribLocation(
+      context_->pp_resource(), shader.program, "a_texCoord");
+  assertNoGLError();
+
+  gles2_if_->EnableVertexAttribArray(context_->pp_resource(), pos_location);
+  gles2_if_->VertexAttribPointer(
+      context_->pp_resource(), pos_location, 2, GL_FLOAT, GL_FALSE, 0, 0);
+  gles2_if_->EnableVertexAttribArray(context_->pp_resource(), tc_location);
+  gles2_if_->VertexAttribPointer(
+      context_->pp_resource(),
+      tc_location,
+      2,
+      GL_FLOAT,
+      GL_FALSE,
+      0,
+      static_cast<float*>(0) + 8);  // Skip position coordinates.
+
+  gles2_if_->UseProgram(context_->pp_resource(), 0);
+  assertNoGLError();
+  return shader;
+}
+
+void MyInstance::CreateShader(GLuint program,
+                              GLenum type,
+                              const char* source,
+                              int size) {
+  GLuint shader = gles2_if_->CreateShader(context_->pp_resource(), type);
+  gles2_if_->ShaderSource(context_->pp_resource(), shader, 1, &source, &size);
+  gles2_if_->CompileShader(context_->pp_resource(), shader);
+  gles2_if_->AttachShader(context_->pp_resource(), program, shader);
+  gles2_if_->DeleteShader(context_->pp_resource(), shader);
+}
+
+// This object is the global object representing this plugin library as long
+// as it is loaded.
+class MyModule : public pp::Module {
+ public:
+  MyModule() : pp::Module() {}
+  virtual ~MyModule() {}
+
+  virtual pp::Instance* CreateInstance(PP_Instance instance) {
+    return new MyInstance(instance, this);
+  }
+};
+
+}  // anonymous namespace
+
+namespace pp {
+// Factory function for your specialization of the Module object.
+Module* CreateModule() {
+  return new MyModule();
+}
+}  // namespace pp