Detect frames from rotated devices in VideoTrackAdapter on Android.

content/renderer/media/video_track_adapter.cc

 // Copyright 2014 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "content/renderer/media/video_track_adapter.h"
 
 #include <algorithm>
 #include <limits>
 #include <memory>
 #include <string>
 #include <utility>
 
 #include "base/bind.h"
 #include "base/command_line.h"
 #include "base/location.h"
 #include "base/macros.h"
 #include "base/metrics/histogram_macros.h"
 #include "base/strings/string_number_conversions.h"
 #include "base/threading/thread_task_runner_handle.h"
 #include "base/trace_event/trace_event.h"
+#include "build/build_config.h"
 #include "content/public/common/content_switches.h"
 #include "media/base/bind_to_current_loop.h"
 #include "media/base/video_util.h"
 
 namespace content {
 
 namespace {
 
 // Amount of frame intervals to wait before considering the source as muted, for
 // the first frame and under normal conditions, respectively. First frame might
@@ skipping 64 matching lines @@
                       float source_frame_rate);
 
   // Bound to the IO-thread.
   base::ThreadChecker io_thread_checker_;
 
   // The task runner where we will release VideoCaptureDeliverFrameCB
   // registered in AddCallback.
   const scoped_refptr<base::SingleThreadTaskRunner> renderer_task_runner_;
 
   const gfx::Size max_frame_size_;
+  base::Optional<gfx::Size> expected_native_size_;
   const double min_aspect_ratio_;
   const double max_aspect_ratio_;
 
   double frame_rate_;
   base::TimeDelta last_time_stamp_;
   double max_frame_rate_;
   double keep_frame_counter_;
 
   typedef std::pair<const MediaStreamVideoTrack*, VideoCaptureDeliverFrameCB>
       VideoIdCallbackPair;
   std::vector<VideoIdCallbackPair> callbacks_;
 
   DISALLOW_COPY_AND_ASSIGN(VideoFrameResolutionAdapter);
 };
 
 VideoTrackAdapter::VideoFrameResolutionAdapter::VideoFrameResolutionAdapter(
     scoped_refptr<base::SingleThreadTaskRunner> render_message_loop,
     const VideoTrackAdapterSettings& settings)
     : renderer_task_runner_(render_message_loop),
       max_frame_size_(settings.max_width, settings.max_height),
+      expected_native_size_(settings.expected_native_size),
       min_aspect_ratio_(settings.min_aspect_ratio),
       max_aspect_ratio_(settings.max_aspect_ratio),
       frame_rate_(MediaStreamVideoSource::kDefaultFrameRate),
       last_time_stamp_(base::TimeDelta::Max()),
       max_frame_rate_(settings.max_frame_rate),
       keep_frame_counter_(0.0) {
   DCHECK(renderer_task_runner_.get());
   DCHECK(io_thread_checker_.CalledOnValidThread());
   DCHECK_GE(max_aspect_ratio_, min_aspect_ratio_);
   CHECK_NE(0, max_aspect_ratio_);
@@ skipping 74 matching lines @@
     return;
 
   // TODO(perkj): Allow cropping / scaling of textures once
   // http://crbug/362521 is fixed.
   if (frame->HasTextures()) {
     DoDeliverFrame(frame, estimated_capture_time);
     return;
   }
   scoped_refptr<media::VideoFrame> video_frame(frame);
 
+  bool is_rotated = false;
+#if defined(OS_ANDROID)

[tommi (sloooow) - chröme, 2017/05/15 19:15:44]

this is the only OS_ check in this file afaict. If there's a bug for tracking
how to do this "the right way" (without relying on heuristic and the same way
across all platforms), it would be good to mention that bug here.

[Guido Urdaneta, 2017/05/16 08:43:21]

Done.

+  if (expected_native_size_ &&
+      frame->natural_size().width() == expected_native_size_->height() &&
+      frame->natural_size().height() == expected_native_size_->width()) {
+    is_rotated = true;
+  }
+#endif
   gfx::Size desired_size;
-  CalculateTargetSize(frame->natural_size(), max_frame_size_, min_aspect_ratio_,
-                      max_aspect_ratio_, &desired_size);
+  CalculateTargetSize(is_rotated, frame->natural_size(), max_frame_size_,
+                      min_aspect_ratio_, max_aspect_ratio_, &desired_size);
   if (desired_size != frame->natural_size()) {
     // Get the largest centered rectangle with the same aspect ratio of
     // |desired_size| that fits entirely inside of |frame->visible_rect()|.
     // This will be the rect we need to crop the original frame to.
     // From this rect, the original frame can be scaled down to |desired_size|.
     const gfx::Rect region_in_frame =
         media::ComputeLetterboxRegion(frame->visible_rect(), desired_size);
 
     video_frame = media::VideoFrame::WrapVideoFrame(
         frame, frame->format(), region_in_frame, desired_size);
@@ skipping 11 matching lines @@
   DoDeliverFrame(video_frame, estimated_capture_time);
 }
 
 bool VideoTrackAdapter::VideoFrameResolutionAdapter::SettingsMatch(
     const VideoTrackAdapterSettings& settings) const {
   DCHECK(io_thread_checker_.CalledOnValidThread());
   return max_frame_size_.width() == settings.max_width &&
          max_frame_size_.height() == settings.max_height &&
          min_aspect_ratio_ == settings.min_aspect_ratio &&
          max_aspect_ratio_ == settings.max_aspect_ratio &&
-         max_frame_rate_ == settings.max_frame_rate;
+         max_frame_rate_ == settings.max_frame_rate &&
+         expected_native_size_ == settings.expected_native_size;
 }
 
 bool VideoTrackAdapter::VideoFrameResolutionAdapter::IsEmpty() const {
   DCHECK(io_thread_checker_.CalledOnValidThread());
   return callbacks_.empty();
 }
 
 void VideoTrackAdapter::VideoFrameResolutionAdapter::DoDeliverFrame(
     const scoped_refptr<media::VideoFrame>& frame,
     const base::TimeTicks& estimated_capture_time) {
@@ skipping 58 matching lines @@
   }
   DVLOG(3) << "Drop frame. Input frame_rate_ " << frame_rate_ << ".";
   return true;
 }
 
 VideoTrackAdapterSettings::VideoTrackAdapterSettings()
     : VideoTrackAdapterSettings(std::numeric_limits<int>::max(),
                                 std::numeric_limits<int>::max(),
                                 0.0,
                                 std::numeric_limits<double>::max(),
-                                0.0) {}
+                                0.0,
+                                base::Optional<gfx::Size>()) {}
 
-VideoTrackAdapterSettings::VideoTrackAdapterSettings(int max_width,
-                                                     int max_height,
-                                                     double min_aspect_ratio,
-                                                     double max_aspect_ratio,
-                                                     double max_frame_rate)
+VideoTrackAdapterSettings::VideoTrackAdapterSettings(
+    int max_width,
+    int max_height,
+    double min_aspect_ratio,
+    double max_aspect_ratio,
+    double max_frame_rate,
+    const base::Optional<gfx::Size>& expected_native_size)
     : max_width(max_width),
       max_height(max_height),
       min_aspect_ratio(min_aspect_ratio),
       max_aspect_ratio(max_aspect_ratio),
-      max_frame_rate(max_frame_rate) {
+      max_frame_rate(max_frame_rate),
+      expected_native_size(expected_native_size) {
   DCHECK_GE(max_width, 1);
   DCHECK_GE(max_height, 1);
   DCHECK_GE(min_aspect_ratio, 0.0);
   DCHECK_GE(max_aspect_ratio, min_aspect_ratio);
   DCHECK_GE(max_frame_rate, 0.0);
 }
 
+VideoTrackAdapterSettings::VideoTrackAdapterSettings(
+    const VideoTrackAdapterSettings& other) = default;
+VideoTrackAdapterSettings& VideoTrackAdapterSettings::operator=(
+    const VideoTrackAdapterSettings& other) = default;
+
 VideoTrackAdapter::VideoTrackAdapter(
     scoped_refptr<base::SingleThreadTaskRunner> io_task_runner)
     : io_task_runner_(io_task_runner),
       renderer_task_runner_(base::ThreadTaskRunnerHandle::Get()),
       monitoring_frame_rate_(false),
       muted_state_(false),
       frame_counter_(0),
       source_frame_rate_(0.0f) {
   DCHECK(io_task_runner);
 }
@@ skipping 51 matching lines @@
                             bound_on_muted_callback, source_frame_rate));
 }
 
 void VideoTrackAdapter::StopFrameMonitoring() {
   DCHECK(thread_checker_.CalledOnValidThread());
   io_task_runner_->PostTask(
       FROM_HERE, base::Bind(&VideoTrackAdapter::StopFrameMonitoringOnIO, this));
 }
 
 // static
-void VideoTrackAdapter::CalculateTargetSize(const gfx::Size& input_size,
-                                            const gfx::Size& max_frame_size,
-                                            double min_aspect_ratio,
-                                            double max_aspect_ratio,
-                                            gfx::Size* desired_size) {
+void VideoTrackAdapter::CalculateTargetSize(
+    bool is_rotated,
+    const gfx::Size& original_input_size,
+    const gfx::Size& max_frame_size,
+    double min_aspect_ratio,
+    double max_aspect_ratio,
+    gfx::Size* desired_size) {
+  const gfx::Size& input_size =
+      is_rotated
+          ? gfx::Size(original_input_size.height(), original_input_size.width())
+          : original_input_size;
+
   // If |frame| has larger width or height than requested, or the aspect ratio
   // does not match the requested, we want to create a wrapped version of this
   // frame with a size that fulfills the constraints.
   const double input_ratio =
       static_cast<double>(input_size.width()) / input_size.height();
 
   if (input_size.width() > max_frame_size.width() ||
       input_size.height() > max_frame_size.height() ||
       input_ratio > max_aspect_ratio || input_ratio < min_aspect_ratio) {
     int desired_width = std::min(max_frame_size.width(), input_size.width());
@@ skipping 10 matching lines @@
                                         requested_ratio);
       // Make sure we scale to an even height to avoid rounding errors
       desired_height = (desired_height + 1) & ~1;
     } else if (resulting_ratio > requested_ratio) {
       desired_width =
           static_cast<int>((desired_width * requested_ratio) / resulting_ratio);
       // Make sure we scale to an even width to avoid rounding errors.
       desired_width = (desired_width + 1) & ~1;
     }
 
-    *desired_size = gfx::Size(desired_width, desired_height);
+    *desired_size = is_rotated ? gfx::Size(desired_height, desired_width)
+                               : gfx::Size(desired_width, desired_height);
   } else {
-    *desired_size = input_size;
+    *desired_size = original_input_size;
   }
 }
 
 void VideoTrackAdapter::StartFrameMonitoringOnIO(
     const OnMutedCallback& on_muted_callback,
     double source_frame_rate) {
   DCHECK(io_task_runner_->BelongsToCurrentThread());
   DCHECK(!monitoring_frame_rate_);
 
   monitoring_frame_rate_ = true;
@@ skipping 56 matching lines @@
   }
 
   io_task_runner_->PostDelayedTask(
       FROM_HERE, base::Bind(&VideoTrackAdapter::CheckFramesReceivedOnIO, this,
                             set_muted_state_callback, frame_counter_),
       base::TimeDelta::FromSecondsD(kNormalFrameTimeoutInFrameIntervals /
                                     source_frame_rate_));
 }
 
 }  // namespace content