Pepper: Add External_OES texture target to PPB_VideoDecoder docs and example.

native_client_sdk/src/examples/api/video_decode/video_decode.cc

 // Copyright (c) 2014 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include <GLES2/gl2.h>
 #include <GLES2/gl2ext.h>
 #include <stdio.h>
 #include <string.h>
 
 #include <iostream>
@@ skipping 11 matching lines @@
 #include "ppapi/cpp/rect.h"
 #include "ppapi/cpp/var.h"
 #include "ppapi/cpp/video_decoder.h"
 #include "ppapi/utility/completion_callback_factory.h"
 
 // VP8 is more likely to work on different versions of Chrome. Undefine this
 // to decode H264.
 #define USE_VP8_TESTDATA_INSTEAD_OF_H264
 #include "testdata.h"
 
-
 // Use assert as a poor-man's CHECK, even in non-debug mode.
 // Since <assert.h> redefines assert on every inclusion (it doesn't use
 // include-guards), make sure this is the last file #include'd in this file.
 #undef NDEBUG
 #include <assert.h>
 
 // Assert |context_| isn't holding any GL Errors.  Done as a macro instead of a
 // function to preserve line number information in the failure message.
 #define assertNoGLError() assert(!gles2_if_->GetError(context_->pp_resource()));
 
@@ skipping 63 matching lines @@
     std::ostringstream stream_;
   };
 
   void InitializeDecoders();
 
   // GL-related functions.
   void InitGL();
   void CreateGLObjects();
   void Create2DProgramOnce();
   void CreateRectangleARBProgramOnce();
+  void CreateExternalOESProgramOnce();
   Shader CreateProgram(const char* vertex_shader, const char* fragment_shader);
   void CreateShader(GLuint program, GLenum type, const char* source, int size);
   void PaintNextPicture();
   void PaintFinished(int32_t result);
 
   pp::Size plugin_size_;
   bool is_painting_;
   // When decode outpaces render, we queue up decoded pictures for later
   // painting.
   typedef std::queue<PendingPicture> PendingPictureQueue;
@@ skipping 12 matching lines @@
 
   // Owned data.
   pp::Graphics3D* context_;
   typedef std::vector<Decoder*> DecoderList;
   DecoderList video_decoders_;
 
   // Shader program to draw GL_TEXTURE_2D target.
   Shader shader_2d_;
   // Shader program to draw GL_TEXTURE_RECTANGLE_ARB target.
   Shader shader_rectangle_arb_;
+  // Shader program to draw GL_TEXTURE_EXTERNAL_OES target.
+  Shader shader_external_oes_;
 };
 
 class Decoder {
  public:
   Decoder(MyInstance* instance, int id, const pp::Graphics3D& graphics_3d);
   ~Decoder();
 
   int id() const { return id_; }
   bool flushing() const { return flushing_; }
   bool resetting() const { return resetting_; }
 
   void Reset();
   void RecyclePicture(const PP_VideoPicture& picture);
 
+  PP_TimeTicks GetAverageLatency() {
+    return num_pictures_ ? total_latency_ / num_pictures_ : 0;
+  }
+
  private:
   void InitializeDone(int32_t result);
   void Start();
   void DecodeNextFrame();
   void DecodeDone(int32_t result);
   void PictureReady(int32_t result, PP_VideoPicture picture);
   void FlushDone(int32_t result);
   void ResetDone(int32_t result);
 
   MyInstance* instance_;
   int id_;
 
   pp::VideoDecoder* decoder_;
   pp::CompletionCallbackFactory<Decoder> callback_factory_;
 
   size_t encoded_data_next_pos_to_decode_;
   int next_picture_id_;
   bool flushing_;
   bool resetting_;
+
+  const PPB_Core* core_if_;
+  static const int kMaxDecodeDelay = 128;
+  PP_TimeTicks decode_time_[kMaxDecodeDelay];
+  PP_TimeTicks total_latency_;
+  int num_pictures_;
 };
 
 #if defined USE_VP8_TESTDATA_INSTEAD_OF_H264
 
 // VP8 is stored in an IVF container.
 // Helpful description: http://wiki.multimedia.cx/index.php?title=IVF
 
 static void GetNextFrame(size_t* start_pos, size_t* end_pos) {
   size_t current_pos = *start_pos;
   if (current_pos == 0)
@@ skipping 29 matching lines @@
 Decoder::Decoder(MyInstance* instance,
                  int id,
                  const pp::Graphics3D& graphics_3d)
     : instance_(instance),
       id_(id),
       decoder_(new pp::VideoDecoder(instance)),
       callback_factory_(this),
       encoded_data_next_pos_to_decode_(0),
       next_picture_id_(0),
       flushing_(false),
-      resetting_(false) {
+      resetting_(false),
+      total_latency_(0.0),
+      num_pictures_(0) {
+  core_if_ = static_cast<const PPB_Core*>(
+      pp::Module::Get()->GetBrowserInterface(PPB_CORE_INTERFACE));
+
 #if defined USE_VP8_TESTDATA_INSTEAD_OF_H264
   const PP_VideoProfile kBitstreamProfile = PP_VIDEOPROFILE_VP8MAIN;
 #else
   const PP_VideoProfile kBitstreamProfile = PP_VIDEOPROFILE_H264MAIN;
 #endif
 
   assert(!decoder_->is_null());
   decoder_->Initialize(graphics_3d,
                        kBitstreamProfile,
                        PP_TRUE /* allow_software_fallback */,
@@ skipping 47 matching lines @@
     }
 
     // Find the start of the next frame.
     size_t start_pos = encoded_data_next_pos_to_decode_;
     size_t end_pos;
     GetNextFrame(&start_pos, &end_pos);
     encoded_data_next_pos_to_decode_ = end_pos;
     // Decode the frame. On completion, DecodeDone will call DecodeNextFrame
     // to implement a decode loop.
     uint32_t size = static_cast<uint32_t>(end_pos - start_pos);
+    decode_time_[next_picture_id_ % kMaxDecodeDelay] = core_if_->GetTimeTicks();
     decoder_->Decode(next_picture_id_++,
                      size,
                      kData + start_pos,
                      callback_factory_.NewCallback(&Decoder::DecodeDone));
   }
 }
 
 void Decoder::DecodeDone(int32_t result) {
   assert(decoder_);
   // Break out of the decode loop on abort.
   if (result == PP_ERROR_ABORTED)
     return;
   assert(result == PP_OK);
   if (!flushing_ && !resetting_)
     DecodeNextFrame();
 }
 
 void Decoder::PictureReady(int32_t result, PP_VideoPicture picture) {
   assert(decoder_);
   // Break out of the get picture loop on abort.
   if (result == PP_ERROR_ABORTED)
     return;
   assert(result == PP_OK);
+
+  num_pictures_++;
+  PP_TimeTicks latency = core_if_->GetTimeTicks() -
+                         decode_time_[picture.decode_id % kMaxDecodeDelay];
+  total_latency_ += latency;
+
   decoder_->GetPicture(
       callback_factory_.NewCallbackWithOutput(&Decoder::PictureReady));
   instance_->PaintPicture(this, picture);
 }
 
 void Decoder::FlushDone(int32_t result) {
   assert(decoder_);
   assert(result == PP_OK || result == PP_ERROR_ABORTED);
   assert(flushing_);
   flushing_ = false;
@@ skipping 11 matching lines @@
 MyInstance::MyInstance(PP_Instance instance, pp::Module* module)
     : pp::Instance(instance),
       pp::Graphics3DClient(this),
       is_painting_(false),
       num_frames_rendered_(0),
       first_frame_delivered_ticks_(-1),
       last_swap_request_ticks_(-1),
       swap_ticks_(0),
       callback_factory_(this),
       context_(NULL) {
-  assert((console_if_ = static_cast<const PPB_Console*>(
-              module->GetBrowserInterface(PPB_CONSOLE_INTERFACE))));
-  assert((core_if_ = static_cast<const PPB_Core*>(
-              module->GetBrowserInterface(PPB_CORE_INTERFACE))));
-  assert((gles2_if_ = static_cast<const PPB_OpenGLES2*>(
-              module->GetBrowserInterface(PPB_OPENGLES2_INTERFACE))));
+  console_if_ = static_cast<const PPB_Console*>(
+      pp::Module::Get()->GetBrowserInterface(PPB_CONSOLE_INTERFACE));
+  core_if_ = static_cast<const PPB_Core*>(
+      pp::Module::Get()->GetBrowserInterface(PPB_CORE_INTERFACE));
+  gles2_if_ = static_cast<const PPB_OpenGLES2*>(
+      pp::Module::Get()->GetBrowserInterface(PPB_OPENGLES2_INTERFACE));
+
   RequestInputEvents(PP_INPUTEVENT_CLASS_MOUSE);
 }
 
 MyInstance::~MyInstance() {
   if (!context_)
     return;
 
   PP_Resource graphics_3d = context_->pp_resource();
   if (shader_2d_.program)
     gles2_if_->DeleteProgram(graphics_3d, shader_2d_.program);
   if (shader_rectangle_arb_.program)
     gles2_if_->DeleteProgram(graphics_3d, shader_rectangle_arb_.program);
+  if (shader_external_oes_.program)
+    gles2_if_->DeleteProgram(graphics_3d, shader_external_oes_.program);
 
   for (DecoderList::iterator it = video_decoders_.begin();
        it != video_decoders_.end();
        ++it)
     delete *it;
 
   delete context_;
 }
 
 void MyInstance::DidChangeView(const pp::Rect& position,
@@ skipping 68 matching lines @@
     x = half_width;
     y = half_height;
   }
 
   PP_Resource graphics_3d = context_->pp_resource();
   if (picture.texture_target == GL_TEXTURE_2D) {
     Create2DProgramOnce();
     gles2_if_->UseProgram(graphics_3d, shader_2d_.program);
     gles2_if_->Uniform2f(
         graphics_3d, shader_2d_.texcoord_scale_location, 1.0, 1.0);
-  } else {
-    assert(picture.texture_target == GL_TEXTURE_RECTANGLE_ARB);
+  } else if (picture.texture_target == GL_TEXTURE_RECTANGLE_ARB) {
     CreateRectangleARBProgramOnce();
     gles2_if_->UseProgram(graphics_3d, shader_rectangle_arb_.program);
     gles2_if_->Uniform2f(graphics_3d,
                          shader_rectangle_arb_.texcoord_scale_location,
                          picture.texture_size.width,
                          picture.texture_size.height);
+  } else {
+    assert(picture.texture_target == GL_TEXTURE_EXTERNAL_OES);
+    CreateExternalOESProgramOnce();
+    gles2_if_->UseProgram(graphics_3d, shader_external_oes_.program);
+    gles2_if_->Uniform2f(
+        graphics_3d, shader_external_oes_.texcoord_scale_location, 1.0, 1.0);
   }
 
   gles2_if_->Viewport(graphics_3d, x, y, half_width, half_height);
   gles2_if_->ActiveTexture(graphics_3d, GL_TEXTURE0);
   gles2_if_->BindTexture(
       graphics_3d, picture.texture_target, picture.texture_id);
   gles2_if_->DrawArrays(graphics_3d, GL_TRIANGLE_STRIP, 0, 4);
 
   gles2_if_->UseProgram(graphics_3d, 0);
 
   last_swap_request_ticks_ = core_if_->GetTimeTicks();
   context_->SwapBuffers(
       callback_factory_.NewCallback(&MyInstance::PaintFinished));
 }
 
 void MyInstance::PaintFinished(int32_t result) {
   assert(result == PP_OK);
   swap_ticks_ += core_if_->GetTimeTicks() - last_swap_request_ticks_;
   is_painting_ = false;
   ++num_frames_rendered_;
   if (num_frames_rendered_ % 50 == 0) {
     double elapsed = core_if_->GetTimeTicks() - first_frame_delivered_ticks_;
     double fps = (elapsed > 0) ? num_frames_rendered_ / elapsed : 1000;
     double ms_per_swap = (swap_ticks_ * 1e3) / num_frames_rendered_;
+    double secs_average_latency = 0;
+    for (DecoderList::iterator it = video_decoders_.begin();
+         it != video_decoders_.end();
+         ++it)
+      secs_average_latency += (*it)->GetAverageLatency();
+    secs_average_latency /= video_decoders_.size();
+    double ms_average_latency = 1000 * secs_average_latency;
     LogError(this).s() << "Rendered frames: " << num_frames_rendered_
                        << ", fps: " << fps
-                       << ", with average ms/swap of: " << ms_per_swap;
+                       << ", with average ms/swap of: " << ms_per_swap
+                       << ", with average latency (ms) of: "
+                       << ms_average_latency;
   }
 
   // If the decoders were reset, this will be empty.
   if (pending_pictures_.empty())
     return;
 
   const PendingPicture& next = pending_pictures_.front();
   Decoder* decoder = next.decoder;
   const PP_VideoPicture& picture = next.picture;
   decoder->RecyclePicture(picture);
@@ skipping 32 matching lines @@
 
   assertNoGLError();
 
   CreateGLObjects();
 }
 
 void MyInstance::CreateGLObjects() {
   // Assign vertex positions and texture coordinates to buffers for use in
   // shader program.
   static const float kVertices[] = {
-      -1, 1, -1, -1, 1, 1, 1, -1,  // Position coordinates.
-      0,  1, 0,  0,  1, 1, 1, 0,   // Texture coordinates.
+      -1, -1, -1, 1, 1, -1, 1, 1,  // Position coordinates.
+      0,  1,  0,  0, 1, 1,  1, 0,  // Texture coordinates.
   };
 
   GLuint buffer;
   gles2_if_->GenBuffers(context_->pp_resource(), 1, &buffer);
   gles2_if_->BindBuffer(context_->pp_resource(), GL_ARRAY_BUFFER, buffer);
 
   gles2_if_->BufferData(context_->pp_resource(),
                         GL_ARRAY_BUFFER,
                         sizeof(kVertices),
                         kVertices,
@@ skipping 34 matching lines @@
       "#extension GL_ARB_texture_rectangle : require\n"
       "precision mediump float;            \n"
       "varying vec2 v_texCoord;            \n"
       "uniform sampler2DRect s_texture;    \n"
       "void main()                         \n"
       "{"
       "    gl_FragColor = texture2DRect(s_texture, v_texCoord).rgba; \n"
       "}";
   shader_rectangle_arb_ =
       CreateProgram(kVertexShader, kFragmentShaderRectangle);
+  assertNoGLError();
+}
+
+void MyInstance::CreateExternalOESProgramOnce() {
+  if (shader_external_oes_.program)
+    return;
+  static const char kFragmentShaderExternal[] =
+      "#extension GL_OES_EGL_image_external : require\n"
+      "precision mediump float;            \n"
+      "varying vec2 v_texCoord;            \n"
+      "uniform samplerExternalOES s_texture; \n"
+      "void main()                         \n"
+      "{"
+      "    gl_FragColor = texture2D(s_texture, v_texCoord); \n"
+      "}";
+  shader_external_oes_ = CreateProgram(kVertexShader, kFragmentShaderExternal);
+  assertNoGLError();
 }
 
 Shader MyInstance::CreateProgram(const char* vertex_shader,
                                  const char* fragment_shader) {
   Shader shader;
 
   // Create shader program.
   shader.program = gles2_if_->CreateProgram(context_->pp_resource());
   CreateShader(
       shader.program, GL_VERTEX_SHADER, vertex_shader, strlen(vertex_shader));
@@ skipping 61 matching lines @@
 };
 
 }  // anonymous namespace
 
 namespace pp {
 // Factory function for your specialization of the Module object.
 Module* CreateModule() {
   return new MyModule();
 }
 }  // namespace pp