Media Capabilities encoding: wire the hardware encoding support

content/renderer/media_recorder/media_recorder_handler.cc

 // Copyright 2015 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "content/renderer/media_recorder/media_recorder_handler.h"
 
 #include <utility>
 
 #include "base/bind.h"
 #include "base/location.h"
@@ skipping 37 matching lines @@
     case VideoTrackRecorder::CodecId::H264:
       return media::kCodecH264;
 #endif
     case VideoTrackRecorder::CodecId::LAST:
       return media::kUnknownVideoCodec;
   }
   NOTREACHED() << "Unsupported codec";
   return media::kUnknownVideoCodec;
 }
 
+// Extracts the first recognised CodecId of |codecs| or CodecId::LAST if none
+// of them is known.
+VideoTrackRecorder::CodecId StringToCodecId(const blink::WebString& codecs) {
+  const std::string& codecs_str = ToLowerASCII(codecs.Utf8());
+
+  if (codecs_str.find("vp8") != std::string::npos)
+    return VideoTrackRecorder::CodecId::VP8;
+  else if (codecs_str.find("vp9") != std::string::npos)
+    return VideoTrackRecorder::CodecId::VP9;
+#if BUILDFLAG(RTC_USE_H264)

[emircan, 2017/04/25 02:04:30]

We should use IS_H264_SUPPORTED as the android cl lands.

+  else if (codecs_str.find("h264") != std::string::npos ||
+           codecs_str.find("avc1") != std::string::npos)
+    return VideoTrackRecorder::CodecId::H264;
+#endif
+  return VideoTrackRecorder::CodecId::LAST;
+}
+
 void OnEncodingInfoError(
     std::unique_ptr<WebMediaCapabilitiesQueryCallbacks> callbacks) {
   callbacks->OnError();
 }
 
 }  // anonymous namespace
 
 MediaRecorderHandler::MediaRecorderHandler()
     : video_bits_per_second_(0),
       audio_bits_per_second_(0),
@@ skipping 62 matching lines @@
   // Save histogram data so we can see how much MediaStream Recorder is used.
   // The histogram counts the number of calls to the JS API.
   UpdateWebRTCMethodCount(WEBKIT_MEDIA_STREAM_RECORDER);
 
   if (!CanSupportMimeType(type, codecs)) {
     DLOG(ERROR) << "Unsupported " << type.Utf8() << ";codecs=" << codecs.Utf8();
     return false;
   }
 
   // Once established that we support the codec(s), hunt then individually.
-  const std::string& codecs_str = ToLowerASCII(codecs.Utf8());
-  if (codecs_str.find("vp8") != std::string::npos)
-    codec_id_ = VideoTrackRecorder::CodecId::VP8;
-  else if (codecs_str.find("vp9") != std::string::npos)
-    codec_id_ = VideoTrackRecorder::CodecId::VP9;
-#if defined(IS_H264_SUPPORTED)
-  else if (codecs_str.find("h264") != std::string::npos)
-    codec_id_ = VideoTrackRecorder::CodecId::H264;
-  else if (codecs_str.find("avc1") != std::string::npos)
-    codec_id_ = VideoTrackRecorder::CodecId::H264;
-#endif
-  else
-    codec_id_ = VideoTrackRecorder::GetPreferredCodecId();
+  const VideoTrackRecorder::CodecId codec_id = StringToCodecId(codecs);
+  codec_id_ = (codec_id != VideoTrackRecorder::CodecId::LAST)
+                  ? codec_id
+                  : VideoTrackRecorder::GetPreferredCodecId();
 
-  DVLOG_IF(1, codecs_str.empty()) << "Falling back to preferred codec id "
-                                  << static_cast<int>(codec_id_);
+  DVLOG_IF(1, codec_id == VideoTrackRecorder::CodecId::LAST)
+      << "Falling back to preferred codec id " << static_cast<int>(codec_id_);
 
   media_stream_ = media_stream;
   DCHECK(client);
   client_ = client;
 
   audio_bits_per_second_ = audio_bits_per_second;
   video_bits_per_second_ = video_bits_per_second;
   return true;
 }
 
@@ skipping 145 matching lines @@
     const std::string parameters = mime_tokenizer.token();
     base::StringTokenizer parameter_tokenizer(parameters, "=");
     if (parameter_tokenizer.GetNext() &&
         base::ToLowerASCII(parameter_tokenizer.token()) == "codecs" &&
         parameter_tokenizer.GetNext()) {
       web_codecs = blink::WebString::FromASCII(parameter_tokenizer.token());
     }
   }
 
   info->supported = CanSupportMimeType(web_type, web_codecs);
+
+  if (configuration.video_configuration && info->supported) {
+    info->smooth = VideoTrackRecorder::IsEncodingLikelyAccelerated(

[chcunningham, 2017/04/24 20:58:17]

Please build this out more to consider system specs and requested resolution
etc. We really do not want "smooth" to simply proxy "accelerated". Software
codecs are often just as performant.

Power efficient (obviously ambiguous) is more likely to proxy "accelerated".
Even still, for decoding, we've seen that power advantages of hw acceleration
tend to disappear under a certain resolution threshold (the specifics matter,
but I think this was h264 vs vp9 at 720p and under). My tentative plan is to
return power_efficient = true whenever sw and hw perform roughly the same,
otherwise returning true just for hw accelerated. 

[emircan, 2017/04/25 02:04:30]

We do not have power metrics currently, so we can leave it a default value for
now, if there is any. 
Smooth is true for all HW encoders available as we try to initialize a session
before reporting support at all. It is not true for some SW encoders as the
algorithms do not scale well, i.e 4K SW VP8 encoding don't go beyond ~4 fps[0].
We definitely should definitely make a decision using
video_configuration.width/height on SW. I am OK leaving it for another CL. 

[0] https://goo.gl/Kb4rP8

[mcasas, 2017/04/25 18:16:07]

Please note that IsEncodingLikelyAccelerated() includes codec
_and_ resolutions, and in the implementation we return false
if the resolution is not larger than a minimum, which is 
platform dependent. IOW we skip hw encoder if the resolution
is too small, which is essentially what you're saying below.
I guess the real question is the meaning of |smooth| and
|power_efficient|.

I'm using |smooth| if VEA encoder is available, but
chcunningham@ says that's not OK.  We coud use 
|power_efficient| and leave |smooth| to convey other
information, e.g. framerate-dependent queries?

[chcunningham, 2017/04/25 18:52:52]

Totally fine for the initial implementation to be missing some things; follow up
here should be P1 and block shipping the feature.

As it stands you will be claiming "not smooth" for a significant chunk of
recording sessions that may in fact be buttery smooth. We made a deliberate
choice not to simply expose "accelerated" as a flag for media capabilities
because it focuses on the wrong thing.
Thats not exactly what I'm looking for. For instance, in such a case you will
mark smooth = false (due to lack of acceleration), but this likely incorrect.
Generally SW should easily manage smooth encoding of small frame sizes. It may
also do fine with larger frame sizes.

+        StringToCodecId(web_codecs), configuration.video_configuration->width,
+        configuration.video_configuration->height);
+    info->power_efficient = info->smooth;
+  }
   DVLOG(1) << "type: " << web_type.Ascii() << ", params:" << web_codecs.Ascii()
-           << " is" << (info->supported ? " supported" : " NOT supported");
+           << " is" << (info->supported ? " supported" : " NOT supported")
+           << " and" << (info->smooth ? " smooth" : " NOT smooth");
 
   scoped_callbacks.PassCallbacks()->OnSuccess(std::move(info));
 }
 
 void MediaRecorderHandler::OnEncodedVideo(
     const media::WebmMuxer::VideoParameters& params,
     std::unique_ptr<std::string> encoded_data,
     std::unique_ptr<std::string> encoded_alpha,
     TimeTicks timestamp,
     bool is_key_frame) {
@@ skipping 99 matching lines @@
     recorder->OnData(audio_bus, timestamp);
 }
 
 void MediaRecorderHandler::SetAudioFormatForTesting(
     const media::AudioParameters& params) {
   for (const auto& recorder : audio_recorders_)
     recorder->OnSetFormat(params);
 }
 
 }  // namespace content