[Remoting iOS] Basic viewport manipulation support

remoting/client/ios/display/gl_display_handler.mm

 // Copyright 2017 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #if !defined(__has_feature) || !__has_feature(objc_arc)
 #error "This file requires ARC support."
 #endif
 
 #include <array>
 
 #import "remoting/client/ios/display/gl_display_handler.h"
 
 #import "base/mac/bind_objc_block.h"
 #import "remoting/client/display/sys_opengl.h"
 #import "remoting/client/ios/display/gl_demo_screen.h"
 
 #include "base/bind.h"
 #include "base/macros.h"
 #include "base/memory/ptr_util.h"
 #include "base/memory/weak_ptr.h"
 #include "remoting/client/chromoting_client_runtime.h"
 #include "remoting/client/cursor_shape_stub_proxy.h"
 #include "remoting/client/display/gl_canvas.h"
 #include "remoting/client/display/gl_renderer.h"
 #include "remoting/client/display/gl_renderer_delegate.h"
 #include "remoting/client/dual_buffer_frame_consumer.h"
+#include "remoting/client/queued_task_poster.h"
 #include "remoting/client/software_video_renderer.h"
 
 namespace remoting {
 namespace GlDisplayHandler {
 
 // The core that lives on the display thread.
 class Core : public protocol::CursorShapeStub, public GlRendererDelegate {
  public:
   Core();
   ~Core() override;
 
   void Initialize();
 
+  void SetHandlerDelegate(id<GlDisplayHandlerDelegate> delegate);
+
   // CursorShapeStub interface.
   void SetCursorShape(const protocol::CursorShapeInfo& cursor_shape) override;
 
   // GlRendererDelegate interface.
   bool CanRenderFrame() override;
   void OnFrameRendered() override;
   void OnSizeChanged(int width, int height) override;
 
   void OnFrameReceived(std::unique_ptr<webrtc::DesktopFrame> frame,
                        const base::Closure& done);
   void Stop();
   void SurfaceCreated(GLKView* view);
   void SurfaceChanged(int width, int height);
+  void SetTransformation(const remoting::ViewMatrix& matrix);
   std::unique_ptr<protocol::FrameConsumer> GrabFrameConsumer();
   EAGLContext* GetEAGLContext();
   base::WeakPtr<Core> GetWeakPtr();
 
  private:
   remoting::ChromotingClientRuntime* runtime_;
 
   // Will be std::move'd when GrabFrameConsumer() is called.
   std::unique_ptr<DualBufferFrameConsumer> owned_frame_consumer_;
   base::WeakPtr<DualBufferFrameConsumer> frame_consumer_;
 
+  // TODO(yuweih): Release references once the surface is destroyed.
   GLKView* gl_view_;
   EAGLContext* eagl_context_;
   std::unique_ptr<GlRenderer> renderer_;
   //  GlDemoScreen *demo_screen_;
+  id<GlDisplayHandlerDelegate> handler_delegate_;
 
   // Used on display thread.
   base::WeakPtr<Core> weak_ptr_;
   base::WeakPtrFactory<Core> weak_factory_;
 
   DISALLOW_COPY_AND_ASSIGN(Core);
 };
 
 Core::Core() : weak_factory_(this) {
   runtime_ = ChromotingClientRuntime::GetInstance();
@@ skipping 30 matching lines @@
 
   renderer_ = remoting::GlRenderer::CreateGlRendererWithDesktop();
 
   //  renderer_.RequestCanvasSize();
 
   // demo_screen_ = new GlDemoScreen();
   // renderer_->AddDrawable(demo_screen_->GetWeakPtr());
   renderer_->SetDelegate(weak_ptr_);
 }
 
+void Core::SetHandlerDelegate(id<GlDisplayHandlerDelegate> delegate) {
+  DCHECK(runtime_->display_task_runner()->BelongsToCurrentThread());
+  handler_delegate_ = delegate;
+}
+
 void Core::SetCursorShape(const protocol::CursorShapeInfo& cursor_shape) {
   DCHECK(runtime_->display_task_runner()->BelongsToCurrentThread());
   renderer_->OnCursorShapeChanged(cursor_shape);
 }
 
 bool Core::CanRenderFrame() {
   DCHECK(runtime_->display_task_runner()->BelongsToCurrentThread());
   return gl_view_ != NULL && eagl_context_ != NULL;
 }
 
 std::unique_ptr<protocol::FrameConsumer> Core::GrabFrameConsumer() {
   DCHECK(owned_frame_consumer_) << "The frame consumer is already grabbed.";
   return std::move(owned_frame_consumer_);
 }
 
 void Core::OnFrameReceived(std::unique_ptr<webrtc::DesktopFrame> frame,
                            const base::Closure& done) {
   DCHECK(runtime_->display_task_runner()->BelongsToCurrentThread());
   renderer_->OnFrameReceived(std::move(frame), done);
 }
 
 void Core::OnFrameRendered() {
-  [gl_view_ setNeedsDisplay];
+  [gl_view_ display];
 }
 
 void Core::OnSizeChanged(int width, int height) {
-  // Nothing to do.
+  DCHECK(runtime_->display_task_runner()->BelongsToCurrentThread());
+  runtime_->ui_task_runner()->PostTask(
+      FROM_HERE, base::BindBlockArc(^() {
+        [handler_delegate_ canvasSizeChanged:CGSizeMake(width, height)];
+      }));
 }
 
 void Core::Stop() {
   DCHECK(runtime_->display_task_runner()->BelongsToCurrentThread());
 
   eagl_context_ = nil;
   // demo_screen_ = nil;
 }
 
 void Core::SurfaceCreated(GLKView* view) {
   DCHECK(runtime_->display_task_runner()->BelongsToCurrentThread());
   gl_view_ = view;
 
   renderer_->OnSurfaceCreated(
       base::MakeUnique<GlCanvas>(static_cast<int>([eagl_context_ API])));
 
+  renderer_->RequestCanvasSize();
+
   runtime_->network_task_runner()->PostTask(
       FROM_HERE, base::Bind(&DualBufferFrameConsumer::RequestFullDesktopFrame,
                             frame_consumer_));
 }
 
 void Core::SurfaceChanged(int width, int height) {
   DCHECK(runtime_->display_task_runner()->BelongsToCurrentThread());
   renderer_->OnSurfaceChanged(width, height);
+}
 
-  // TODO(nicholss): This are wrong values but it lets us get something on the
-  // screen.
-  std::array<float, 9> matrix = {{1, 0, 0,  // Row 1
-                                  0, 1, 0,  // Row 2
-                                  0, 0, 1}};
-
-  renderer_->OnPixelTransformationChanged(matrix);
+void Core::SetTransformation(const remoting::ViewMatrix& matrix) {
+  DCHECK(runtime_->display_task_runner()->BelongsToCurrentThread());
+  renderer_->OnPixelTransformationChanged(matrix.ToMatrixArray());
 }
 
 EAGLContext* Core::GetEAGLContext() {
   return eagl_context_;
 }
 
 base::WeakPtr<remoting::GlDisplayHandler::Core> Core::GetWeakPtr() {
   return weak_ptr_;
 }
 
 }  // namespace GlDisplayHandler
 }  // namespace remoting
 
 @interface GlDisplayHandler () {
   remoting::GlDisplayHandler::Core* _core;
   remoting::ChromotingClientRuntime* _runtime;
+  std::unique_ptr<remoting::QueuedTaskPoster> _uiTaskPoster;
 }
 @end
 
 @implementation GlDisplayHandler
 
 - (id)init {
   self = [super init];
   if (self) {
     _runtime = remoting::ChromotingClientRuntime::GetInstance();
     _core = new remoting::GlDisplayHandler::Core();
+    _uiTaskPoster.reset(
+        new remoting::QueuedTaskPoster(_runtime->display_task_runner()));
   }
   return self;
 }
 
 #pragma mark - Public
 
 - (void)stop {
   _runtime->display_task_runner()->PostTask(
       FROM_HERE,
       base::Bind(&remoting::GlDisplayHandler::Core::Stop, _core->GetWeakPtr()));
 }
 
+- (void)setDelegate:(id<GlDisplayHandlerDelegate>)delegate {

[nicholss, 2017/05/03 22:47:35]

You can keep this here, this will override the auto-generated.

Mark your property as write only and don't synthesize it. (I think that will
work...)

[Yuwei, 2017/05/04 00:28:07]

Done. Moved below.

+  _runtime->display_task_runner()->PostTask(
+      FROM_HERE,
+      base::Bind(&remoting::GlDisplayHandler::Core::SetHandlerDelegate,
+                 _core->GetWeakPtr(), delegate));
+}
+
 - (std::unique_ptr<remoting::protocol::VideoRenderer>)CreateVideoRenderer {
   return base::MakeUnique<remoting::SoftwareVideoRenderer>(
       _core->GrabFrameConsumer());
 }
 
 - (std::unique_ptr<remoting::protocol::CursorShapeStub>)CreateCursorShapeStub {
   return base::MakeUnique<remoting::CursorShapeStubProxy>(
       _core->GetWeakPtr(), _runtime->display_task_runner());
 }
 
 - (EAGLContext*)GetEAGLContext {
   return _core->GetEAGLContext();
 }
 
 - (void)onSurfaceCreated:(GLKView*)view {
   _runtime->display_task_runner()->PostTask(
       FROM_HERE, base::Bind(&remoting::GlDisplayHandler::Core::SurfaceCreated,
                             _core->GetWeakPtr(), view));
 }
 
 - (void)onSurfaceChanged:(const CGRect&)frame {
   _runtime->display_task_runner()->PostTask(
       FROM_HERE,
       base::Bind(&remoting::GlDisplayHandler::Core::SurfaceChanged,
                  _core->GetWeakPtr(), frame.size.width, frame.size.height));
 }
 
+- (void)onPixelTransformationChanged:(const remoting::ViewMatrix&)matrix {
+  _uiTaskPoster->AddTask(

[nicholss, 2017/05/03 22:47:35]

How do you choose to AddTask on the task poster vs PostTask on the task runner?

[Yuwei, 2017/05/04 00:28:07]

UITaskPoster is used to synchronize UI events that should be rendered in the
same frame.

We once ran into issue with synchronizing cursor and viewport movements. Say if
we want to do this sequence:
1. Move cursor by <1, 1>.
2. Move viewport in the same direction.

Without synchronizing, rendering can happen between step 1 and 2, and cause the
cursor to randomly jiggle.

I think cursor and viewport movements are the only things currently needed to be
synchronized.

+      base::Bind(&remoting::GlDisplayHandler::Core::SetTransformation,
+                 _core->GetWeakPtr(), matrix));
+}
+
 // TODO(nicholss): Remove this function, it is not used in the final impl,
 // or it should call RequestRender.
 - (void)glkView:(GLKView*)view drawInRect:(CGRect)rect {

[nicholss, 2017/05/03 22:47:35]

Can you delete this? It is not used, just cruft I left around.

[Yuwei, 2017/05/04 00:28:07]

Done.

   if (_core) {
     _runtime->display_task_runner()->PostTask(
         FROM_HERE,
         base::Bind(&remoting::GlDisplayHandler::Core::SurfaceChanged,
                    _core->GetWeakPtr(), rect.size.width, rect.size.height));
   }
 }
 
 @end