Implement Pepper PPB_VideoDecoder interface.

ppapi/examples/video_decode/video_decode.cc

+// Copyright (c) 2014 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include <stdio.h>
+#include <string.h>
+
+#include <iostream>
+#include <queue>
+#include <sstream>
+
+#include "ppapi/c/pp_errors.h"
+#include "ppapi/c/ppb_console.h"
+#include "ppapi/c/ppb_opengles2.h"
+#include "ppapi/cpp/graphics_3d.h"
+#include "ppapi/cpp/graphics_3d_client.h"
+#include "ppapi/cpp/input_event.h"
+#include "ppapi/cpp/instance.h"
+#include "ppapi/cpp/module.h"
+#include "ppapi/cpp/rect.h"
+#include "ppapi/cpp/var.h"
+#include "ppapi/cpp/video_decoder.h"
+#include "ppapi/examples/video_decode/testdata.h"
+#include "ppapi/lib/gl/include/GLES2/gl2.h"
+#include "ppapi/lib/gl/include/GLES2/gl2ext.h"
+#include "ppapi/utility/completion_callback_factory.h"
+
+// Use assert as a poor-man's CHECK, even in non-debug mode.
+// Since <assert.h> redefines assert on every inclusion (it doesn't use
+// include-guards), make sure this is the last file #include'd in this file.
+#undef NDEBUG
+#include <assert.h>
+
+// Assert |context_| isn't holding any GL Errors.  Done as a macro instead of a
+// function to preserve line number information in the failure message.
+#define assertNoGLError() assert(!gles2_if_->GetError(context_->pp_resource()));
+
+namespace {
+
+struct Shader {
+  Shader() : program(0), texcoord_scale_location(0) {}
+  ~Shader() {}
+
+  GLuint program;
+  GLint texcoord_scale_location;
+};
+
+class Decoder;
+class MyInstance;
+
+class MyInstance : public pp::Instance, public pp::Graphics3DClient {
+ public:
+  MyInstance(PP_Instance instance, pp::Module* module);
+  virtual ~MyInstance();
+
+  // pp::Instance implementation.
+  virtual void DidChangeView(const pp::Rect& position,
+                             const pp::Rect& clip_ignored);
+
+  // pp::Graphics3DClient implementation.
+  virtual void Graphics3DContextLost() {
+    // TODO(vrk/fischman): Properly reset after a lost graphics context.  In
+    // particular need to delete context_ and re-create textures.
+    // Probably have to recreate the decoder from scratch, because old textures
+    // can still be outstanding in the decoder!
+    assert(false && "Unexpectedly lost graphics context");
+  }
+
+  void PaintPicture(Decoder* decoder, const PP_VideoPicture& picture);
+
+ private:
+  // Log an error to the developer console and stderr by creating a temporary
+  // object of this type and streaming to it.  Example usage:
+  // LogError(this).s() << "Hello world: " << 42;
+  class LogError {
+   public:
+    LogError(MyInstance* instance) : instance_(instance) {}
+    ~LogError() {
+      const std::string& msg = stream_.str();
+      instance_->console_if_->Log(
+          instance_->pp_instance(), PP_LOGLEVEL_ERROR, pp::Var(msg).pp_var());
+      std::cerr << msg << std::endl;
+    }
+    // Impl note: it would have been nicer to have LogError derive from
+    // std::ostringstream so that it can be streamed to directly, but lookup
+    // rules turn streamed string literals to hex pointers on output.
+    std::ostringstream& s() { return stream_; }
+
+   private:
+    MyInstance* instance_;
+    std::ostringstream stream_;
+  };
+
+  void InitializeDecoders();
+
+  // GL-related functions.
+  void InitGL();
+  void CreateGLObjects();
+  void Create2DProgramOnce();
+  void CreateRectangleARBProgramOnce();
+  Shader CreateProgram(const char* vertex_shader, const char* fragment_shader);
+  void CreateShader(GLuint program, GLenum type, const char* source, int size);
+  void PaintFinished(int32_t result, Decoder* decoder, PP_VideoPicture picture);
+
+  pp::Size plugin_size_;
+  bool is_painting_;
+  // When decode outpaces render, we queue up decoded pictures for later
+  // painting.  Elements are <decoder,picture>.
+  typedef std::queue<std::pair<Decoder*, PP_VideoPicture> > PictureQueue;
+  PictureQueue pictures_pending_paint_;
+
+  int num_frames_rendered_;
+  PP_TimeTicks first_frame_delivered_ticks_;
+  PP_TimeTicks last_swap_request_ticks_;
+  PP_TimeTicks swap_ticks_;
+  pp::CompletionCallbackFactory<MyInstance> callback_factory_;
+
+  // Unowned pointers.
+  const PPB_Console* console_if_;
+  const PPB_Core* core_if_;
+  const PPB_OpenGLES2* gles2_if_;
+
+  // Owned data.
+  pp::Graphics3D* context_;
+  typedef std::vector<Decoder*> DecoderList;
+  DecoderList video_decoders_;
+
+  // Shader program to draw GL_TEXTURE_2D target.
+  Shader shader_2d_;
+  // Shader program to draw GL_TEXTURE_RECTANGLE_ARB target.
+  Shader shader_rectangle_arb_;
+};
+
+class Decoder {
+ public:
+  Decoder(MyInstance* instance, int id, const pp::Graphics3D& graphics_3d);
+  ~Decoder();
+
+  int id() const { return id_; }
+  bool decoding() const { return !flushing_ && !resetting_; }
+
+  void Seek(int frame);
+  void RecyclePicture(const PP_VideoPicture& picture);
+
+ private:
+  void InitializeDone(int32_t result);
+  void Start(int frame);
+  void DecodeNextFrame();
+  void DecodeDone(int32_t result);
+  void PictureReady(int32_t result, PP_VideoPicture picture);
+  void FlushDone(int32_t result);
+  void ResetDone(int32_t result);
+
+  MyInstance* instance_;
+  int id_;
+
+  pp::VideoDecoder* decoder_;
+  pp::CompletionCallbackFactory<Decoder> callback_factory_;
+
+  size_t encoded_data_next_pos_to_decode_;
+  int next_picture_id_;
+  int seek_frame_;
+  bool flushing_;
+  bool resetting_;
+};
+
+// Returns true if the current position is at the start of a NAL unit.
+static bool LookingAtNAL(const unsigned char* encoded, size_t pos) {
+  // H264 frames start with 0, 0, 0, 1 in our test data.
+  return pos + 3 < kDataLen && encoded[pos] == 0 && encoded[pos + 1] == 0 &&
+         encoded[pos + 2] == 0 && encoded[pos + 3] == 1;
+}
+
+// Find the start and end of the next frame.
+static void GetNextFrame(size_t* start_pos, size_t* end_pos) {
+  assert(LookingAtNAL(kData, *start_pos));
+  *end_pos = *start_pos;
+  *end_pos += 4;
+  while (*end_pos < kDataLen && !LookingAtNAL(kData, *end_pos)) {
+    ++*end_pos;
+  }
+}
+
+Decoder::Decoder(MyInstance* instance,
+                 int id,
+                 const pp::Graphics3D& graphics_3d)
+    : instance_(instance),
+      id_(id),
+      decoder_(new pp::VideoDecoder(instance)),
+      callback_factory_(this),
+      encoded_data_next_pos_to_decode_(0),
+      next_picture_id_(0),
+      seek_frame_(0),
+      flushing_(false),
+      resetting_(false) {
+  assert(!decoder_->is_null());
+  const PP_VideoProfile profile = PP_VIDEOPROFILE_H264MAIN;
+  decoder_->Initialize(graphics_3d,
+                       profile,
+                       PP_FALSE /* allow_software_fallback */,
+                       callback_factory_.NewCallback(&Decoder::InitializeDone));
+}
+
+Decoder::~Decoder() {
+  delete decoder_;
+}
+
+void Decoder::InitializeDone(int32_t result) {
+  assert(decoder_);
+  assert(result == PP_OK);
+  assert(decoding());
+  Start(0);
+}
+
+void Decoder::Start(int frame) {
+  assert(decoder_);
+
+  // Skip to |frame|.
+  size_t start_pos = 0;
+  size_t end_pos = 0;
+  for (int i = frame; i > 0; i--)

[Ami GONE FROM CHROMIUM, 2014/05/28 20:50:56]

nit: why counting backwards??

[bbudge, 2014/05/29 00:03:34]

Not sure what I was thinking. Done.

+    GetNextFrame(&start_pos, &end_pos);
+  encoded_data_next_pos_to_decode_ = end_pos;
+
+  // Register callback to get the first picture. We call GetPicture again in
+  // PictureReady to continuously receive pictures as they're decoded.
+  decoder_->GetPicture(
+      callback_factory_.NewCallbackWithOutput(&Decoder::PictureReady));
+
+  // Start the decode loop.
+  DecodeNextFrame();
+}
+
+void Decoder::Seek(int frame) {
+  assert(decoder_);
+  seek_frame_ = frame;

[Ami GONE FROM CHROMIUM, 2014/05/28 20:50:56]

write-only var?

(Seek() seems like dead code)

[bbudge, 2014/05/29 00:03:34]

It is. I was working on implementing Seek to test Reset but decided to leave it
for a future CL.

+  resetting_ = true;
+  decoder_->Reset(callback_factory_.NewCallback(&Decoder::ResetDone));
+}
+
+void Decoder::RecyclePicture(const PP_VideoPicture& picture) {
+  assert(decoder_);
+  decoder_->RecyclePicture(picture);
+}
+
+void Decoder::DecodeNextFrame() {
+  assert(decoder_);
+  if (encoded_data_next_pos_to_decode_ <= kDataLen) {
+    // If we've just reached the end of the bitstream, flush and wait.
+    if (!flushing_ && encoded_data_next_pos_to_decode_ == kDataLen) {
+      flushing_ = true;
+      decoder_->Flush(callback_factory_.NewCallback(&Decoder::FlushDone));
+      return;
+    }
+
+    // Find the start of the next frame.
+    size_t start_pos = encoded_data_next_pos_to_decode_;
+    size_t end_pos;
+    GetNextFrame(&start_pos, &end_pos);
+    encoded_data_next_pos_to_decode_ = end_pos;
+    // Decode the frame. On completion, DecodeDone will call DecodeNextFrame
+    // to implement a decode loop.
+    uint32_t size = static_cast<uint32_t>(end_pos - start_pos);
+    decoder_->Decode(next_picture_id_++,
+                     size,
+                     kData + start_pos,
+                     callback_factory_.NewCallback(&Decoder::DecodeDone));
+  }
+}
+
+void Decoder::DecodeDone(int32_t result) {
+  assert(decoder_);
+  // Break out of the decode loop on abort.
+  if (result == PP_ERROR_ABORTED)
+    return;
+  assert(result == PP_OK);
+  if (decoding())
+    DecodeNextFrame();
+}
+
+void Decoder::PictureReady(int32_t result, PP_VideoPicture picture) {
+  assert(decoder_);
+  // Break out of the get picture loop on abort.
+  if (result == PP_ERROR_ABORTED)
+    return;
+  assert(result == PP_OK);
+  decoder_->GetPicture(
+      callback_factory_.NewCallbackWithOutput(&Decoder::PictureReady));
+  instance_->PaintPicture(this, picture);
+}
+
+void Decoder::FlushDone(int32_t result) {
+  assert(decoder_);
+  assert(result == PP_OK || result == PP_ERROR_ABORTED);
+  flushing_ = false;
+}
+
+void Decoder::ResetDone(int32_t result) {
+  assert(decoder_);
+  assert(result == PP_OK);
+  resetting_ = false;
+}
+
+MyInstance::MyInstance(PP_Instance instance, pp::Module* module)
+    : pp::Instance(instance),
+      pp::Graphics3DClient(this),
+      is_painting_(false),
+      num_frames_rendered_(0),
+      first_frame_delivered_ticks_(-1),
+      last_swap_request_ticks_(-1),
+      swap_ticks_(0),
+      callback_factory_(this),
+      context_(NULL) {
+  assert((console_if_ = static_cast<const PPB_Console*>(
+              module->GetBrowserInterface(PPB_CONSOLE_INTERFACE))));
+  assert((core_if_ = static_cast<const PPB_Core*>(
+              module->GetBrowserInterface(PPB_CORE_INTERFACE))));
+  assert((gles2_if_ = static_cast<const PPB_OpenGLES2*>(
+              module->GetBrowserInterface(PPB_OPENGLES2_INTERFACE))));
+  RequestInputEvents(PP_INPUTEVENT_CLASS_MOUSE);

[Ami GONE FROM CHROMIUM, 2014/05/28 20:50:56]

Did you mean to tie mouse-clicks to seeking somehow?

[bbudge, 2014/05/29 00:03:34]

Yes, I was trying to use the plugin area as a crude scrubber. I've had some
difficulty getting this to work. The software decoder doesn't seem to like
restarting in random places. Do you think this is something that could work, or
should I remove it? It's going to be useful for testing Reset for the software
fallback path, where Flush and Reset are more interesting.

+}
+
+MyInstance::~MyInstance() {
+  if (!context_)
+    return;
+
+  PP_Resource graphics_3d = context_->pp_resource();
+  if (shader_2d_.program)
+    gles2_if_->DeleteProgram(graphics_3d, shader_2d_.program);
+  if (shader_rectangle_arb_.program)
+    gles2_if_->DeleteProgram(graphics_3d, shader_rectangle_arb_.program);
+
+  for (DecoderList::iterator it = video_decoders_.begin();
+       it != video_decoders_.end();
+       ++it)
+    delete *it;
+
+  delete context_;
+}
+
+void MyInstance::DidChangeView(const pp::Rect& position,
+                               const pp::Rect& clip_ignored) {
+  if (position.width() == 0 || position.height() == 0)
+    return;
+  if (plugin_size_.width()) {
+    assert(position.size() == plugin_size_);
+    return;
+  }
+  plugin_size_ = position.size();
+
+  // Initialize graphics.
+  InitGL();
+  InitializeDecoders();
+}
+
+void MyInstance::InitializeDecoders() {
+  assert(video_decoders_.empty());
+  // Create two decoders with ids 0 and 1.
+  video_decoders_.push_back(new Decoder(this, 0, *context_));
+  video_decoders_.push_back(new Decoder(this, 1, *context_));
+}
+
+void MyInstance::PaintPicture(Decoder* decoder,
+                              const PP_VideoPicture& picture) {
+  if (first_frame_delivered_ticks_ == -1)
+    assert((first_frame_delivered_ticks_ = core_if_->GetTimeTicks()) != -1);
+  if (is_painting_) {
+    pictures_pending_paint_.push(std::make_pair(decoder, picture));
+    return;
+  }
+
+  assert(!is_painting_);
+  is_painting_ = true;
+  int x = 0;
+  int y = 0;
+  int half_width = plugin_size_.width() / 2;
+  int half_height = plugin_size_.height() / 2;
+  if (decoder->id() != 0) {
+    x = half_width;
+    y = half_height;
+  }
+
+  PP_Resource graphics_3d = context_->pp_resource();
+  if (picture.texture_target == GL_TEXTURE_2D) {
+    Create2DProgramOnce();
+    gles2_if_->UseProgram(graphics_3d, shader_2d_.program);
+    gles2_if_->Uniform2f(
+        graphics_3d, shader_2d_.texcoord_scale_location, 1.0, 1.0);
+  } else {
+    assert(picture.texture_target == GL_TEXTURE_RECTANGLE_ARB);
+    CreateRectangleARBProgramOnce();
+    gles2_if_->UseProgram(graphics_3d, shader_rectangle_arb_.program);
+    gles2_if_->Uniform2f(graphics_3d,
+                         shader_rectangle_arb_.texcoord_scale_location,
+                         picture.texture_size.width,
+                         picture.texture_size.height);
+  }
+
+  gles2_if_->Viewport(graphics_3d, x, y, half_width, half_height);
+  gles2_if_->ActiveTexture(graphics_3d, GL_TEXTURE0);
+  gles2_if_->BindTexture(
+      graphics_3d, picture.texture_target, picture.texture_id);
+  gles2_if_->DrawArrays(graphics_3d, GL_TRIANGLE_STRIP, 0, 4);
+
+  gles2_if_->UseProgram(graphics_3d, 0);
+
+  last_swap_request_ticks_ = core_if_->GetTimeTicks();
+  assert(PP_OK_COMPLETIONPENDING ==
+         context_->SwapBuffers(callback_factory_.NewCallback(
+             &MyInstance::PaintFinished, decoder, picture)));
+}
+
+void MyInstance::PaintFinished(int32_t result,
+                               Decoder* decoder,
+                               PP_VideoPicture picture) {
+  assert(result == PP_OK);
+  swap_ticks_ += core_if_->GetTimeTicks() - last_swap_request_ticks_;
+  is_painting_ = false;
+  ++num_frames_rendered_;
+  if (num_frames_rendered_ % 50 == 0) {
+    double elapsed = core_if_->GetTimeTicks() - first_frame_delivered_ticks_;
+    double fps = (elapsed > 0) ? num_frames_rendered_ / elapsed : 1000;
+    double ms_per_swap = (swap_ticks_ * 1e3) / num_frames_rendered_;
+    LogError(this).s() << "Rendered frames: " << num_frames_rendered_
+                       << ", fps: " << fps
+                       << ", with average ms/swap of: " << ms_per_swap;
+  }
+  decoder->RecyclePicture(picture);
+  // Keep painting as long as we have pictures.
+  if (!pictures_pending_paint_.empty()) {
+    std::pair<Decoder*, PP_VideoPicture> pending =
+        pictures_pending_paint_.front();
+    pictures_pending_paint_.pop();
+    PaintPicture(pending.first, pending.second);
+  }
+}
+
+void MyInstance::InitGL() {
+  assert(plugin_size_.width() && plugin_size_.height());
+  is_painting_ = false;
+
+  assert(!context_);
+  int32_t context_attributes[] = {
+      PP_GRAPHICS3DATTRIB_ALPHA_SIZE,     8,
+      PP_GRAPHICS3DATTRIB_BLUE_SIZE,      8,
+      PP_GRAPHICS3DATTRIB_GREEN_SIZE,     8,
+      PP_GRAPHICS3DATTRIB_RED_SIZE,       8,
+      PP_GRAPHICS3DATTRIB_DEPTH_SIZE,     0,
+      PP_GRAPHICS3DATTRIB_STENCIL_SIZE,   0,
+      PP_GRAPHICS3DATTRIB_SAMPLES,        0,
+      PP_GRAPHICS3DATTRIB_SAMPLE_BUFFERS, 0,
+      PP_GRAPHICS3DATTRIB_WIDTH,          plugin_size_.width(),
+      PP_GRAPHICS3DATTRIB_HEIGHT,         plugin_size_.height(),
+      PP_GRAPHICS3DATTRIB_NONE,
+  };
+  context_ = new pp::Graphics3D(this, context_attributes);
+  assert(!context_->is_null());
+  assert(BindGraphics(*context_));
+
+  // Clear color bit.
+  gles2_if_->ClearColor(context_->pp_resource(), 1, 0, 0, 1);
+  gles2_if_->Clear(context_->pp_resource(), GL_COLOR_BUFFER_BIT);
+
+  assertNoGLError();
+
+  CreateGLObjects();
+}
+
+void MyInstance::CreateGLObjects() {
+  // Assign vertex positions and texture coordinates to buffers for use in
+  // shader program.
+  static const float kVertices[] = {
+      -1, 1, -1, -1, 1, 1, 1, -1,  // Position coordinates.
+      0,  1, 0,  0,  1, 1, 1, 0,   // Texture coordinates.
+  };
+
+  GLuint buffer;
+  gles2_if_->GenBuffers(context_->pp_resource(), 1, &buffer);
+  gles2_if_->BindBuffer(context_->pp_resource(), GL_ARRAY_BUFFER, buffer);
+
+  gles2_if_->BufferData(context_->pp_resource(),
+                        GL_ARRAY_BUFFER,
+                        sizeof(kVertices),
+                        kVertices,
+                        GL_STATIC_DRAW);
+  assertNoGLError();
+}
+
+static const char kVertexShader[] =
+    "varying vec2 v_texCoord;            \n"
+    "attribute vec4 a_position;          \n"
+    "attribute vec2 a_texCoord;          \n"
+    "uniform vec2 v_scale;               \n"
+    "void main()                         \n"
+    "{                                   \n"
+    "    v_texCoord = v_scale * a_texCoord; \n"
+    "    gl_Position = a_position;       \n"
+    "}";
+
+void MyInstance::Create2DProgramOnce() {
+  if (shader_2d_.program)
+    return;
+  static const char kFragmentShader2D[] =
+      "precision mediump float;            \n"
+      "varying vec2 v_texCoord;            \n"
+      "uniform sampler2D s_texture;        \n"
+      "void main()                         \n"
+      "{"
+      "    gl_FragColor = texture2D(s_texture, v_texCoord); \n"
+      "}";
+  shader_2d_ = CreateProgram(kVertexShader, kFragmentShader2D);
+  assertNoGLError();
+}
+
+void MyInstance::CreateRectangleARBProgramOnce() {
+  if (shader_rectangle_arb_.program)
+    return;
+  static const char kFragmentShaderRectangle[] =
+      "#extension GL_ARB_texture_rectangle : require\n"
+      "precision mediump float;            \n"
+      "varying vec2 v_texCoord;            \n"
+      "uniform sampler2DRect s_texture;    \n"
+      "void main()                         \n"
+      "{"
+      "    gl_FragColor = texture2DRect(s_texture, v_texCoord).rgba; \n"
+      "}";
+  shader_rectangle_arb_ =
+      CreateProgram(kVertexShader, kFragmentShaderRectangle);
+}
+
+Shader MyInstance::CreateProgram(const char* vertex_shader,
+                                 const char* fragment_shader) {
+  Shader shader;
+
+  // Create shader program.
+  shader.program = gles2_if_->CreateProgram(context_->pp_resource());
+  CreateShader(
+      shader.program, GL_VERTEX_SHADER, vertex_shader, strlen(vertex_shader));
+  CreateShader(shader.program,
+               GL_FRAGMENT_SHADER,
+               fragment_shader,
+               strlen(fragment_shader));
+  gles2_if_->LinkProgram(context_->pp_resource(), shader.program);
+  gles2_if_->UseProgram(context_->pp_resource(), shader.program);
+  gles2_if_->Uniform1i(
+      context_->pp_resource(),
+      gles2_if_->GetUniformLocation(
+          context_->pp_resource(), shader.program, "s_texture"),
+      0);
+  assertNoGLError();
+
+  shader.texcoord_scale_location = gles2_if_->GetUniformLocation(
+      context_->pp_resource(), shader.program, "v_scale");
+
+  GLint pos_location = gles2_if_->GetAttribLocation(
+      context_->pp_resource(), shader.program, "a_position");
+  GLint tc_location = gles2_if_->GetAttribLocation(
+      context_->pp_resource(), shader.program, "a_texCoord");
+  assertNoGLError();
+
+  gles2_if_->EnableVertexAttribArray(context_->pp_resource(), pos_location);
+  gles2_if_->VertexAttribPointer(
+      context_->pp_resource(), pos_location, 2, GL_FLOAT, GL_FALSE, 0, 0);
+  gles2_if_->EnableVertexAttribArray(context_->pp_resource(), tc_location);
+  gles2_if_->VertexAttribPointer(
+      context_->pp_resource(),
+      tc_location,
+      2,
+      GL_FLOAT,
+      GL_FALSE,
+      0,
+      static_cast<float*>(0) + 8);  // Skip position coordinates.
+
+  gles2_if_->UseProgram(context_->pp_resource(), 0);
+  assertNoGLError();
+  return shader;
+}
+
+void MyInstance::CreateShader(GLuint program,
+                              GLenum type,
+                              const char* source,
+                              int size) {
+  GLuint shader = gles2_if_->CreateShader(context_->pp_resource(), type);
+  gles2_if_->ShaderSource(context_->pp_resource(), shader, 1, &source, &size);
+  gles2_if_->CompileShader(context_->pp_resource(), shader);
+  gles2_if_->AttachShader(context_->pp_resource(), program, shader);
+  gles2_if_->DeleteShader(context_->pp_resource(), shader);
+}
+
+// This object is the global object representing this plugin library as long
+// as it is loaded.
+class MyModule : public pp::Module {
+ public:
+  MyModule() : pp::Module() {}
+  virtual ~MyModule() {}
+
+  virtual pp::Instance* CreateInstance(PP_Instance instance) {
+    return new MyInstance(instance, this);
+  }
+};
+
+}  // anonymous namespace
+
+namespace pp {
+// Factory function for your specialization of the Module object.
+Module* CreateModule() {
+  return new MyModule();
+}
+}  // namespace pp