Improving the decoder pipeline.

remoting/client/plugin/pepper_view.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "remoting/client/plugin/pepper_view.h"
 
 #include "base/message_loop.h"
 #include "base/string_util.h"
 #include "ppapi/cpp/completion_callback.h"
 #include "ppapi/cpp/graphics_2d.h"
@@ skipping 28 matching lines @@
   DLOG(FATAL) << "Unknown error code" << error;
   return  ChromotingScriptableObject::ERROR_NONE;
 }
 
 }  // namespace
 
 PepperView::PepperView(ChromotingInstance* instance, ClientContext* context)
   : instance_(instance),
     context_(context),
     flush_blocked_(false),
+    view_changed_(false),
+    view_size_(SkISize::Make(0, 0)),
+    clip_area_(SkIRect::MakeEmpty()),
+    screen_size_(SkISize::Make(0, 0)),
     is_static_fill_(false),
     static_fill_color_(0),
     ALLOW_THIS_IN_INITIALIZER_LIST(weak_factory_(this)) {
 }
 
 PepperView::~PepperView() {
 }
 
 bool PepperView::Initialize() {
   return true;
 }
 
 void PepperView::TearDown() {
   DCHECK(context_->main_message_loop()->BelongsToCurrentThread());
 
   weak_factory_.InvalidateWeakPtrs();
 }
 
 void PepperView::Paint() {
   DCHECK(context_->main_message_loop()->BelongsToCurrentThread());
 
-  if (is_static_fill_) {
+  if (is_static_fill_ && !clip_area_.isEmpty()) {
     VLOG(1) << "Static filling " << static_fill_color_;
-    pp::ImageData image(instance_, pp::ImageData::GetNativeImageDataFormat(),
-                        pp::Size(graphics2d_.size().width(),
-                                 graphics2d_.size().height()),
+    pp::ImageData image(instance_, PP_IMAGEDATAFORMAT_BGRA_PREMUL,
+                        pp::Size(clip_area_.width(),
+                                 clip_area_.height()),
                         false);
     if (image.is_null()) {
       LOG(ERROR) << "Unable to allocate image of size: "
-                 << graphics2d_.size().width() << " x "
-                 << graphics2d_.size().height();
+                 << clip_area_.width() << " x "
+                 << clip_area_.height();
       return;
     }
 
     for (int y = 0; y < image.size().height(); y++) {
       for (int x = 0; x < image.size().width(); x++) {
         *image.GetAddr32(pp::Point(x, y)) = static_fill_color_;
       }
     }
 
-    // For ReplaceContents, make sure the image size matches the device context
-    // size!  Otherwise, this will just silently do nothing.
-    graphics2d_.ReplaceContents(&image);
+    graphics2d_.PaintImageData(
+        image,
+        pp::Point(clip_area_.left(), clip_area_.top()));
     FlushGraphics(base::Time::Now());
   } else {
     // TODO(ajwong): We need to keep a backing store image of the host screen
     // that has the data here which can be redrawn.
     return;
   }
 }
 
-void PepperView::SetHostSize(const SkISize& host_size) {
+void PepperView::SetScreenSize(const SkISize& screen_size) {
   DCHECK(context_->main_message_loop()->BelongsToCurrentThread());
 
-  if (host_size_ == host_size)
+  if (screen_size_ == screen_size)
     return;
 
-  host_size_ = host_size;
+  screen_size_ = screen_size;
 
   // Submit an update of desktop size to Javascript.
   instance_->GetScriptableObject()->SetDesktopSize(
-      host_size.width(), host_size.height());
-}
-
-void PepperView::PaintFrame(media::VideoFrame* frame, const SkRegion& region) {
-  DCHECK(context_->main_message_loop()->BelongsToCurrentThread());
-
-  SetHostSize(SkISize::Make(frame->width(), frame->height()));
-
-  if (!backing_store_.get() || backing_store_->is_null()) {
-    LOG(ERROR) << "Backing store is not available.";
-    return;
-  }
-
-  base::Time start_time = base::Time::Now();
-
-  // Copy updated regions to the backing store and then paint the regions.
-  bool changes_made = false;
-  for (SkRegion::Iterator i(region); !i.done(); i.next())
-    changes_made |= PaintRect(frame, i.rect());
-
-  if (changes_made)
-    FlushGraphics(start_time);
-}
-
-bool PepperView::PaintRect(media::VideoFrame* frame, const SkIRect& r) {
-  const uint8* frame_data = frame->data(media::VideoFrame::kRGBPlane);
-  const int kFrameStride = frame->stride(media::VideoFrame::kRGBPlane);
-  const int kBytesPerPixel = GetBytesPerPixel(media::VideoFrame::RGB32);
-
-  pp::Size backing_store_size = backing_store_->size();
-  SkIRect rect(r);
-  if (!rect.intersect(SkIRect::MakeWH(backing_store_size.width(),
-                                      backing_store_size.height()))) {
-    return false;
-  }
-
-  const uint8* in =
-      frame_data +
-      kFrameStride * rect.fTop +   // Y offset.
-      kBytesPerPixel * rect.fLeft;  // X offset.
-  uint8* out =
-      reinterpret_cast<uint8*>(backing_store_->data()) +
-      backing_store_->stride() * rect.fTop +  // Y offset.
-      kBytesPerPixel * rect.fLeft;  // X offset.
-
-  // TODO(hclam): We really should eliminate this memory copy.
-  for (int j = 0; j < rect.height(); ++j) {
-    memcpy(out, in, rect.width() * kBytesPerPixel);
-    in += kFrameStride;
-    out += backing_store_->stride();
-  }
-
-  // Pepper Graphics 2D has a strange and badly documented API that the
-  // point here is the offset from the source rect. Why?
-  graphics2d_.PaintImageData(
-      *backing_store_.get(),
-      pp::Point(0, 0),
-      pp::Rect(rect.fLeft, rect.fTop, rect.width(), rect.height()));
-  return true;
-}
-
-void PepperView::BlankRect(pp::ImageData& image_data, const pp::Rect& rect) {
-  const int kBytesPerPixel = GetBytesPerPixel(media::VideoFrame::RGB32);
-  for (int y = rect.y(); y < rect.bottom(); y++) {
-    uint8* to = reinterpret_cast<uint8*>(image_data.data()) +
-        (y * image_data.stride()) + (rect.x() * kBytesPerPixel);
-    memset(to, 0xff, rect.width() * kBytesPerPixel);
-  }
+      screen_size.width(), screen_size.height());
 }
 
 void PepperView::FlushGraphics(base::Time paint_start) {
   scoped_ptr<base::Closure> task(
       new base::Closure(
-          base::Bind(&PepperView::OnPaintDone, weak_factory_.GetWeakPtr(),
+          base::Bind(&PepperView::OnFlushDone, weak_factory_.GetWeakPtr(),
                      paint_start)));
 
   // Flag needs to be set here in order to get a proper error code for Flush().
   // Otherwise Flush() will always return PP_OK_COMPLETIONPENDING and the error
   // would be hidden.
   //
   // Note that we can also handle this by providing an actual callback which
   // takes the result code. Right now everything goes to the task that doesn't
   // result value.
   pp::CompletionCallback pp_callback(&CompletionCallbackClosureAdapter,
@@ skipping 61 matching lines @@
 
     case protocol::ConnectionToHost::FAILED:
       SetSolidFill(kFailedColor);
       scriptable_obj->SetConnectionStatus(
           ChromotingScriptableObject::STATUS_FAILED,
           ConvertConnectionError(error));
       break;
   }
 }
 
-bool PepperView::SetViewSize(const SkISize& view_size) {
-  if (view_size_ == view_size)
-    return false;
-  view_size_ = view_size;
+bool PepperView::SetView(const SkISize& view_size,
+                         const SkIRect& clip_area) {
+  // Make sure that the clip area is entirely contained within
+  // the view area.
+  DCHECK(!view_size.isEmpty() || clip_area.isEmpty());
+  DCHECK(view_size.isEmpty() ||
+         SkIRect::MakeSize(view_size).contains(clip_area));
 
-  pp::Size pp_size = pp::Size(view_size.width(), view_size.height());
+  bool updated = false;
+  if (clip_area_ != clip_area) {
+    updated = true;
 
-  graphics2d_ = pp::Graphics2D(instance_, pp_size, true);
-  if (!instance_->BindGraphics(graphics2d_)) {
-    LOG(ERROR) << "Couldn't bind the device context.";
-    return false;
+    // YUV to RGB conversion may require even X and Y coordinates for
+    // the top left corner of the clipping area.
+    clip_area_ = AlignRect(clip_area);
+    clip_area_.intersect(SkIRect::MakeSize(view_size));
   }
 
-  if (view_size.isEmpty())
-    return false;
+  if (view_size_ != view_size) {
+    updated = true;
+    view_size_ = view_size;
 
-  // Allocate the backing store to save the desktop image.
-  if ((backing_store_.get() == NULL) ||
-      (backing_store_->size() != pp_size)) {
-    VLOG(1) << "Allocate backing store: "
-            << view_size.width() << " x " << view_size.height();
-    backing_store_.reset(
-        new pp::ImageData(instance_, pp::ImageData::GetNativeImageDataFormat(),
-                          pp_size, false));
-    DCHECK(backing_store_.get() && !backing_store_->is_null())
-        << "Not enough memory for backing store.";
+    pp::Size pp_size = pp::Size(view_size.width(), view_size.height());
+    graphics2d_ = pp::Graphics2D(instance_, pp_size, true);
+    bool result = instance_->BindGraphics(graphics2d_);
+
+    // There is no good way to handle this error currently.
+    DCHECK(result) << "Couldn't bind the device context.";
   }
-  return true;
+
+  view_changed_ = view_changed_ || updated;
+  return updated;
 }
 
-void PepperView::AllocateFrame(media::VideoFrame::Format format,
-                               const SkISize& size,
-                               scoped_refptr<media::VideoFrame>* frame_out,
-                               const base::Closure& done) {
+void PepperView::OnFrameReady(const SkISize& screen_size,
+                              SkISize* view_size_out,
+                              SkIRect* clip_area_out,
+                              scoped_ptr<pp::ImageData>* backing_store_out,
+                              const base::Closure& done) {
   DCHECK(context_->main_message_loop()->BelongsToCurrentThread());
 
-  *frame_out = media::VideoFrame::CreateFrame(
-      media::VideoFrame::RGB32, size.width(), size.height(),
-      base::TimeDelta(), base::TimeDelta());
-  (*frame_out)->AddRef();
+  SetScreenSize(screen_size);
+
+  // Adjust the backing buffer size accordingly to the clipping area size:
+  //  - no buffer is needed when the clipping area is empty.
+  //  - previously allocated buffer can be reused if its size matches.
+  if (clip_area_.isEmpty()) {
+    backing_store_.reset(NULL);
+  } else {
+    pp::Size pp_size = pp::Size(clip_area_.width(), clip_area_.height());
+    if ((backing_store_.get() == NULL) ||
+        (backing_store_->size() != pp_size)) {
+      VLOG(1) << "Allocate backing store: "
+              << clip_area_.width() << " x " << clip_area_.height();
+      backing_store_.reset(
+          new pp::ImageData(instance_,
+                            PP_IMAGEDATAFORMAT_BGRA_PREMUL,
+                            pp_size, false));
+      DCHECK(backing_store_.get() && !backing_store_->is_null())
+          << "Not enough memory for backing store.";
+    }
+  }
+
+  // Watch for any changes to the view size or clipping area while drawing.
+  view_changed_ = false;
+
+  // Capture the drawing parameters and pass them to the caller.
+  *view_size_out = view_size_;
+  *clip_area_out = clip_area_;
+  *backing_store_out = backing_store_.Pass();
   done.Run();
 }
 
-void PepperView::ReleaseFrame(media::VideoFrame* frame) {
+void PepperView::OnPaintDone(scoped_ptr<pp::ImageData> backing_store,
+                             scoped_ptr<SkRegion> updated_region) {
   DCHECK(context_->main_message_loop()->BelongsToCurrentThread());
 
-  if (frame)
-    frame->Release();
+  // Discard the updated backing store entirely in case if
+  // the clipping area has been changed while the drawing
+  // operation was pending. Next painting operation has been
+  // scheduled in response to the change and it will update
+  // the screen properly.
+  if (view_changed_) {
+    instance_->RefreshFullFrame();
+    return;
+  }
+
+  backing_store_ = backing_store.Pass();
+
+  // Notify Pepper API about the updated areas and flush pixels to the screen.
+  base::Time start_time = base::Time::Now();
+
+  for (SkRegion::Iterator i(*updated_region); !i.done(); i.next()) {
+    SkIRect rect = i.rect();
+    if (!rect.intersect(clip_area_))
+      continue;
+
+    // Specify the rectangle coordinates relative to the clipping area.
+    rect.offset(- clip_area_.left(), - clip_area_.top());
+
+    // Pepper Graphics 2D has a strange and badly documented API that the
+    // point here is the offset from the source rect. Why?
+    graphics2d_.PaintImageData(
+        *backing_store_.get(),
+        pp::Point(clip_area_.left(), clip_area_.top()),
+        pp::Rect(rect.left(), rect.top(), rect.width(), rect.height()));
+  }
+
+  FlushGraphics(start_time);
+  return;
 }
 
-void PepperView::OnPartialFrameOutput(media::VideoFrame* frame,
-                                      SkRegion* region,
-                                      const base::Closure& done) {
-  DCHECK(context_->main_message_loop()->BelongsToCurrentThread());
-
-  // TODO(ajwong): Clean up this API to be async so we don't need to use a
-  // member variable as a hack.
-  PaintFrame(frame, *region);
-  done.Run();
-}
-
-void PepperView::OnPaintDone(base::Time paint_start) {
+void PepperView::OnFlushDone(base::Time paint_start) {
   DCHECK(context_->main_message_loop()->BelongsToCurrentThread());
   instance_->GetStats()->video_paint_ms()->Record(
       (base::Time::Now() - paint_start).InMilliseconds());
 
   // If the last flush failed because there was already another one in progress
   // then we perform the flush now.
   if (flush_blocked_)
-    FlushGraphics(base::Time::Now());
+    FlushGraphics(base::Time::Now()); // TODO: why is it not |paint_start|?
   return;
 }
 
 }  // namespace remoting