OLD | NEW |
---|---|
1 // Copyright 2014 The Chromium Authors. All rights reserved. | 1 // Copyright 2014 The Chromium Authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
4 | 4 |
5 #include "content/renderer/media/webrtc/webrtc_video_capturer_adapter.h" | 5 #include "content/renderer/media/webrtc/webrtc_video_capturer_adapter.h" |
6 | 6 |
7 #include "base/bind.h" | 7 #include "base/bind.h" |
8 #include "base/memory/aligned_memory.h" | 8 #include "base/memory/aligned_memory.h" |
9 #include "base/synchronization/waitable_event.h" | |
9 #include "base/trace_event/trace_event.h" | 10 #include "base/trace_event/trace_event.h" |
10 #include "content/renderer/media/webrtc/webrtc_video_frame_adapter.h" | 11 #include "content/common/gpu/client/context_provider_command_buffer.h" |
12 #include "content/renderer/render_thread_impl.h" | |
13 #include "media/base/bind_to_current_loop.h" | |
11 #include "media/base/timestamp_constants.h" | 14 #include "media/base/timestamp_constants.h" |
12 #include "media/base/video_util.h" | 15 #include "media/base/video_util.h" |
16 #include "skia/ext/platform_canvas.h" | |
17 #include "third_party/libyuv/include/libyuv/convert.h" | |
13 #include "third_party/libyuv/include/libyuv/convert_from.h" | 18 #include "third_party/libyuv/include/libyuv/convert_from.h" |
14 #include "third_party/libyuv/include/libyuv/scale.h" | 19 #include "third_party/libyuv/include/libyuv/scale.h" |
20 #include "third_party/skia/include/core/SkSurface.h" | |
15 #include "third_party/webrtc/common_video/include/video_frame_buffer.h" | 21 #include "third_party/webrtc/common_video/include/video_frame_buffer.h" |
16 #include "third_party/webrtc/common_video/rotation.h" | 22 #include "third_party/webrtc/common_video/rotation.h" |
17 #include "third_party/webrtc/media/engine/webrtcvideoframe.h" | 23 #include "third_party/webrtc/media/engine/webrtcvideoframe.h" |
18 | 24 |
19 namespace content { | 25 namespace content { |
20 namespace { | 26 namespace { |
21 | 27 |
22 // Empty method used for keeping a reference to the original media::VideoFrame. | 28 // Empty method used for keeping a reference to the original media::VideoFrame. |
23 // The reference to |frame| is kept in the closure that calls this method. | 29 // The reference to |frame| is kept in the closure that calls this method. |
24 void ReleaseOriginalFrame(const scoped_refptr<media::VideoFrame>& frame) { | 30 void ReleaseOriginalFrame(const scoped_refptr<media::VideoFrame>& frame) { |
25 } | 31 } |
26 | 32 |
27 } // anonymous namespace | 33 } // anonymous namespace |
28 | 34 |
29 WebRtcVideoCapturerAdapter::WebRtcVideoCapturerAdapter(bool is_screencast) | 35 WebRtcVideoCapturerAdapter::WebRtcVideoCapturerAdapter(bool is_screencast) |
30 : is_screencast_(is_screencast), | 36 : main_thread_task_runner_(base::ThreadTaskRunnerHandle::Get()), |
31 running_(false) { | 37 provider_(RenderThreadImpl::current()->SharedMainThreadContextProvider()), |
38 is_screencast_(is_screencast), | |
39 running_(false), | |
40 weak_factory_(this) { | |
32 thread_checker_.DetachFromThread(); | 41 thread_checker_.DetachFromThread(); |
42 copy_texture_callback_ = | |
43 base::Bind(&WebRtcVideoCapturerAdapter::CopyTextureFrame, | |
44 weak_factory_.GetWeakPtr()); | |
33 } | 45 } |
34 | 46 |
35 WebRtcVideoCapturerAdapter::~WebRtcVideoCapturerAdapter() { | 47 WebRtcVideoCapturerAdapter::~WebRtcVideoCapturerAdapter() { |
36 DVLOG(3) << " WebRtcVideoCapturerAdapter::dtor"; | 48 DVLOG(3) << " WebRtcVideoCapturerAdapter::dtor"; |
mcasas
2016/10/26 16:06:02
unrelated nit: I'm trying to correct these stateme
emircan
2016/10/27 00:37:56
Done.
| |
37 } | 49 } |
38 | 50 |
39 cricket::CaptureState WebRtcVideoCapturerAdapter::Start( | |
40 const cricket::VideoFormat& capture_format) { | |
41 DCHECK(thread_checker_.CalledOnValidThread()); | |
42 DCHECK(!running_); | |
43 DVLOG(3) << " WebRtcVideoCapturerAdapter::Start w = " << capture_format.width | |
44 << " h = " << capture_format.height; | |
45 | |
46 running_ = true; | |
47 return cricket::CS_RUNNING; | |
48 } | |
49 | |
50 void WebRtcVideoCapturerAdapter::Stop() { | |
51 DCHECK(thread_checker_.CalledOnValidThread()); | |
52 DVLOG(3) << " WebRtcVideoCapturerAdapter::Stop "; | |
53 DCHECK(running_); | |
54 running_ = false; | |
55 SetCaptureFormat(NULL); | |
56 SignalStateChange(this, cricket::CS_STOPPED); | |
57 } | |
58 | |
59 bool WebRtcVideoCapturerAdapter::IsRunning() { | |
60 DCHECK(thread_checker_.CalledOnValidThread()); | |
61 return running_; | |
62 } | |
63 | |
64 bool WebRtcVideoCapturerAdapter::GetPreferredFourccs( | |
65 std::vector<uint32_t>* fourccs) { | |
66 DCHECK(thread_checker_.CalledOnValidThread()); | |
67 DCHECK(!fourccs || fourccs->empty()); | |
68 if (fourccs) | |
69 fourccs->push_back(cricket::FOURCC_I420); | |
70 return fourccs != NULL; | |
71 } | |
72 | |
73 bool WebRtcVideoCapturerAdapter::IsScreencast() const { | |
74 return is_screencast_; | |
75 } | |
76 | |
77 bool WebRtcVideoCapturerAdapter::GetBestCaptureFormat( | |
78 const cricket::VideoFormat& desired, | |
79 cricket::VideoFormat* best_format) { | |
80 DCHECK(thread_checker_.CalledOnValidThread()); | |
81 DVLOG(3) << " GetBestCaptureFormat:: " | |
82 << " w = " << desired.width | |
83 << " h = " << desired.height; | |
84 | |
85 // Capability enumeration is done in MediaStreamVideoSource. The adapter can | |
86 // just use what is provided. | |
87 // Use the desired format as the best format. | |
88 best_format->width = desired.width; | |
89 best_format->height = desired.height; | |
90 best_format->fourcc = cricket::FOURCC_I420; | |
91 best_format->interval = desired.interval; | |
92 return true; | |
93 } | |
94 | |
95 void WebRtcVideoCapturerAdapter::OnFrameCaptured( | 51 void WebRtcVideoCapturerAdapter::OnFrameCaptured( |
96 const scoped_refptr<media::VideoFrame>& input_frame) { | 52 const scoped_refptr<media::VideoFrame>& input_frame) { |
97 DCHECK(thread_checker_.CalledOnValidThread()); | 53 DCHECK(thread_checker_.CalledOnValidThread()); |
98 TRACE_EVENT0("video", "WebRtcVideoCapturerAdapter::OnFrameCaptured"); | 54 TRACE_EVENT0("video", "WebRtcVideoCapturerAdapter::OnFrameCaptured"); |
99 if (!(input_frame->IsMappable() && | 55 if (!(input_frame->IsMappable() && |
100 (input_frame->format() == media::PIXEL_FORMAT_I420 || | 56 (input_frame->format() == media::PIXEL_FORMAT_I420 || |
101 input_frame->format() == media::PIXEL_FORMAT_YV12 || | 57 input_frame->format() == media::PIXEL_FORMAT_YV12 || |
102 input_frame->format() == media::PIXEL_FORMAT_YV12A))) { | 58 input_frame->format() == media::PIXEL_FORMAT_YV12A)) && |
59 !input_frame->HasTextures()) { | |
103 // Since connecting sources and sinks do not check the format, we need to | 60 // Since connecting sources and sinks do not check the format, we need to |
104 // just ignore formats that we can not handle. | 61 // just ignore formats that we can not handle. |
62 LOG(ERROR) << "We cannot send frame with storage type: " | |
63 << input_frame->storage_type() | |
64 << " format: " << input_frame->format(); | |
mcasas
2016/10/26 16:06:02
These are going to print numbers, right?
Consider
emircan
2016/10/27 00:37:56
Done.
| |
105 NOTREACHED(); | 65 NOTREACHED(); |
106 return; | 66 return; |
107 } | 67 } |
108 scoped_refptr<media::VideoFrame> frame = input_frame; | 68 scoped_refptr<media::VideoFrame> frame = input_frame; |
109 // Drop alpha channel since we do not support it yet. | 69 // Drop alpha channel since we do not support it yet. |
110 if (frame->format() == media::PIXEL_FORMAT_YV12A) | 70 if (frame->format() == media::PIXEL_FORMAT_YV12A) |
111 frame = media::WrapAsI420VideoFrame(input_frame); | 71 frame = media::WrapAsI420VideoFrame(input_frame); |
112 | 72 |
113 const int orig_width = frame->natural_size().width(); | 73 const int orig_width = frame->natural_size().width(); |
114 const int orig_height = frame->natural_size().height(); | 74 const int orig_height = frame->natural_size().height(); |
(...skipping 11 matching lines...) Expand all Loading... | |
126 frame->timestamp().InMicroseconds(), | 86 frame->timestamp().InMicroseconds(), |
127 rtc::TimeMicros(), | 87 rtc::TimeMicros(), |
128 &adapted_width, &adapted_height, | 88 &adapted_width, &adapted_height, |
129 &crop_width, &crop_height, &crop_x, &crop_y, | 89 &crop_width, &crop_height, &crop_x, &crop_y, |
130 &translated_camera_time_us)) { | 90 &translated_camera_time_us)) { |
131 return; | 91 return; |
132 } | 92 } |
133 | 93 |
134 // Return |frame| directly if it is texture backed, because there is no | 94 // Return |frame| directly if it is texture backed, because there is no |
135 // cropping support for texture yet. See http://crbug/503653. | 95 // cropping support for texture yet. See http://crbug/503653. |
136 // Return |frame| directly if it is GpuMemoryBuffer backed, as we want to | |
137 // keep the frame on native buffers. | |
138 if (frame->HasTextures()) { | 96 if (frame->HasTextures()) { |
139 OnFrame(cricket::WebRtcVideoFrame( | 97 OnFrame(cricket::WebRtcVideoFrame( |
140 new rtc::RefCountedObject<WebRtcVideoFrameAdapter>(frame), | 98 new rtc::RefCountedObject<WebRtcVideoFrameAdapter>( |
99 frame, copy_texture_callback_), | |
141 webrtc::kVideoRotation_0, translated_camera_time_us), | 100 webrtc::kVideoRotation_0, translated_camera_time_us), |
142 orig_width, orig_height); | 101 orig_width, orig_height); |
143 return; | 102 return; |
144 } | 103 } |
145 | 104 |
146 // Translate crop rectangle from natural size to visible size. | 105 // Translate crop rectangle from natural size to visible size. |
147 gfx::Rect cropped_visible_rect( | 106 gfx::Rect cropped_visible_rect( |
148 frame->visible_rect().x() + | 107 frame->visible_rect().x() + |
149 crop_x * frame->visible_rect().width() / orig_width, | 108 crop_x * frame->visible_rect().width() / orig_width, |
150 frame->visible_rect().y() + | 109 frame->visible_rect().y() + |
151 crop_y * frame->visible_rect().height() / orig_height, | 110 crop_y * frame->visible_rect().height() / orig_height, |
152 crop_width * frame->visible_rect().width() / orig_width, | 111 crop_width * frame->visible_rect().width() / orig_width, |
153 crop_height * frame->visible_rect().height() / orig_height); | 112 crop_height * frame->visible_rect().height() / orig_height); |
154 | 113 |
155 const gfx::Size adapted_size(adapted_width, adapted_height); | 114 const gfx::Size adapted_size(adapted_width, adapted_height); |
156 scoped_refptr<media::VideoFrame> video_frame = | 115 scoped_refptr<media::VideoFrame> video_frame = |
157 media::VideoFrame::WrapVideoFrame(frame, frame->format(), | 116 media::VideoFrame::WrapVideoFrame(frame, frame->format(), |
158 cropped_visible_rect, adapted_size); | 117 cropped_visible_rect, adapted_size); |
159 if (!video_frame) | 118 if (!video_frame) |
160 return; | 119 return; |
161 | 120 |
162 video_frame->AddDestructionObserver(base::Bind(&ReleaseOriginalFrame, frame)); | 121 video_frame->AddDestructionObserver(base::Bind(&ReleaseOriginalFrame, frame)); |
163 | 122 |
164 // If no scaling is needed, return a wrapped version of |frame| directly. | 123 // If no scaling is needed, return a wrapped version of |frame| directly. |
165 if (video_frame->natural_size() == video_frame->visible_rect().size()) { | 124 if (video_frame->natural_size() == video_frame->visible_rect().size()) { |
166 OnFrame(cricket::WebRtcVideoFrame( | 125 OnFrame(cricket::WebRtcVideoFrame( |
167 new rtc::RefCountedObject<WebRtcVideoFrameAdapter>(video_frame), | 126 new rtc::RefCountedObject<WebRtcVideoFrameAdapter>( |
127 video_frame, copy_texture_callback_), | |
168 webrtc::kVideoRotation_0, translated_camera_time_us), | 128 webrtc::kVideoRotation_0, translated_camera_time_us), |
169 orig_width, orig_height); | 129 orig_width, orig_height); |
170 return; | 130 return; |
171 } | 131 } |
172 | 132 |
173 // We need to scale the frame before we hand it over to webrtc. | 133 // We need to scale the frame before we hand it over to webrtc. |
174 scoped_refptr<media::VideoFrame> scaled_frame = | 134 scoped_refptr<media::VideoFrame> scaled_frame = |
175 scaled_frame_pool_.CreateFrame(media::PIXEL_FORMAT_I420, adapted_size, | 135 scaled_frame_pool_.CreateFrame(media::PIXEL_FORMAT_I420, adapted_size, |
176 gfx::Rect(adapted_size), adapted_size, | 136 gfx::Rect(adapted_size), adapted_size, |
177 frame->timestamp()); | 137 frame->timestamp()); |
178 libyuv::I420Scale(video_frame->visible_data(media::VideoFrame::kYPlane), | 138 libyuv::I420Scale(video_frame->visible_data(media::VideoFrame::kYPlane), |
179 video_frame->stride(media::VideoFrame::kYPlane), | 139 video_frame->stride(media::VideoFrame::kYPlane), |
180 video_frame->visible_data(media::VideoFrame::kUPlane), | 140 video_frame->visible_data(media::VideoFrame::kUPlane), |
181 video_frame->stride(media::VideoFrame::kUPlane), | 141 video_frame->stride(media::VideoFrame::kUPlane), |
182 video_frame->visible_data(media::VideoFrame::kVPlane), | 142 video_frame->visible_data(media::VideoFrame::kVPlane), |
183 video_frame->stride(media::VideoFrame::kVPlane), | 143 video_frame->stride(media::VideoFrame::kVPlane), |
184 video_frame->visible_rect().width(), | 144 video_frame->visible_rect().width(), |
185 video_frame->visible_rect().height(), | 145 video_frame->visible_rect().height(), |
186 scaled_frame->data(media::VideoFrame::kYPlane), | 146 scaled_frame->data(media::VideoFrame::kYPlane), |
187 scaled_frame->stride(media::VideoFrame::kYPlane), | 147 scaled_frame->stride(media::VideoFrame::kYPlane), |
188 scaled_frame->data(media::VideoFrame::kUPlane), | 148 scaled_frame->data(media::VideoFrame::kUPlane), |
189 scaled_frame->stride(media::VideoFrame::kUPlane), | 149 scaled_frame->stride(media::VideoFrame::kUPlane), |
190 scaled_frame->data(media::VideoFrame::kVPlane), | 150 scaled_frame->data(media::VideoFrame::kVPlane), |
191 scaled_frame->stride(media::VideoFrame::kVPlane), | 151 scaled_frame->stride(media::VideoFrame::kVPlane), |
192 adapted_width, adapted_height, libyuv::kFilterBilinear); | 152 adapted_width, adapted_height, libyuv::kFilterBilinear); |
193 | 153 |
194 OnFrame(cricket::WebRtcVideoFrame( | 154 OnFrame(cricket::WebRtcVideoFrame( |
195 new rtc::RefCountedObject<WebRtcVideoFrameAdapter>(scaled_frame), | 155 new rtc::RefCountedObject<WebRtcVideoFrameAdapter>( |
156 scaled_frame, copy_texture_callback_), | |
196 webrtc::kVideoRotation_0, translated_camera_time_us), | 157 webrtc::kVideoRotation_0, translated_camera_time_us), |
197 orig_width, orig_height); | 158 orig_width, orig_height); |
198 } | 159 } |
199 | 160 |
161 cricket::CaptureState WebRtcVideoCapturerAdapter::Start( | |
162 const cricket::VideoFormat& capture_format) { | |
163 DCHECK(thread_checker_.CalledOnValidThread()); | |
164 DCHECK(!running_); | |
165 DVLOG(3) << " WebRtcVideoCapturerAdapter::Start w = " << capture_format.width | |
166 << " h = " << capture_format.height; | |
167 | |
168 running_ = true; | |
169 return cricket::CS_RUNNING; | |
170 } | |
171 | |
172 void WebRtcVideoCapturerAdapter::Stop() { | |
173 DCHECK(thread_checker_.CalledOnValidThread()); | |
174 DVLOG(3) << " WebRtcVideoCapturerAdapter::Stop "; | |
175 DCHECK(running_); | |
176 running_ = false; | |
177 SetCaptureFormat(NULL); | |
178 SignalStateChange(this, cricket::CS_STOPPED); | |
179 } | |
180 | |
181 bool WebRtcVideoCapturerAdapter::IsRunning() { | |
182 DCHECK(thread_checker_.CalledOnValidThread()); | |
183 return running_; | |
184 } | |
185 | |
186 bool WebRtcVideoCapturerAdapter::GetPreferredFourccs( | |
187 std::vector<uint32_t>* fourccs) { | |
188 DCHECK(thread_checker_.CalledOnValidThread()); | |
189 DCHECK(!fourccs || fourccs->empty()); | |
mcasas
2016/10/26 16:06:02
nit: not your code, but here we should not DCHECK(
emircan
2016/10/27 00:37:56
Done.
| |
190 if (fourccs) | |
191 fourccs->push_back(cricket::FOURCC_I420); | |
192 return fourccs != NULL; | |
193 } | |
194 | |
195 bool WebRtcVideoCapturerAdapter::IsScreencast() const { | |
196 return is_screencast_; | |
197 } | |
198 | |
199 bool WebRtcVideoCapturerAdapter::GetBestCaptureFormat( | |
200 const cricket::VideoFormat& desired, | |
201 cricket::VideoFormat* best_format) { | |
202 DCHECK(thread_checker_.CalledOnValidThread()); | |
203 DVLOG(3) << " GetBestCaptureFormat:: " | |
204 << " w = " << desired.width | |
205 << " h = " << desired.height; | |
mcasas
2016/10/26 16:06:02
nit:
DVLOG(3) << __func__ << " desired: " << desir
emircan
2016/10/27 00:37:56
Done.
| |
206 | |
207 // Capability enumeration is done in MediaStreamVideoSource. The adapter can | |
208 // just use what is provided. | |
209 // Use the desired format as the best format. | |
210 best_format->width = desired.width; | |
211 best_format->height = desired.height; | |
212 best_format->fourcc = cricket::FOURCC_I420; | |
213 best_format->interval = desired.interval; | |
214 return true; | |
215 } | |
216 | |
217 void WebRtcVideoCapturerAdapter::CopyTextureFrame( | |
218 const scoped_refptr<media::VideoFrame>& frame, | |
219 scoped_refptr<media::VideoFrame>* new_frame) { | |
220 base::WaitableEvent waiter(base::WaitableEvent::ResetPolicy::MANUAL, | |
221 base::WaitableEvent::InitialState::NOT_SIGNALED); | |
222 main_thread_task_runner_->PostTask( | |
223 FROM_HERE, | |
224 base::Bind(&WebRtcVideoCapturerAdapter::CopyTextureFrameOnMainThread, | |
225 base::Unretained(this), frame, new_frame, &waiter)); | |
mcasas
2016/10/26 16:06:02
weak_factory_.GetWeakPtr() ?
emircan
2016/10/27 00:37:56
I can add a WeakPtr here to be used on main thread
| |
226 waiter.Wait(); | |
227 } | |
228 | |
229 void WebRtcVideoCapturerAdapter::CopyTextureFrameOnMainThread( | |
230 const scoped_refptr<media::VideoFrame>& frame, | |
231 scoped_refptr<media::VideoFrame>* new_frame, | |
232 base::WaitableEvent* waiter) { | |
233 DCHECK(main_thread_task_runner_->BelongsToCurrentThread()); | |
234 DCHECK(frame->format() == media::PIXEL_FORMAT_ARGB || | |
235 frame->format() == media::PIXEL_FORMAT_XRGB || | |
236 frame->format() == media::PIXEL_FORMAT_I420 || | |
237 frame->format() == media::PIXEL_FORMAT_UYVY || | |
238 frame->format() == media::PIXEL_FORMAT_NV12); | |
239 *new_frame = media::VideoFrame::CreateFrame( | |
240 media::PIXEL_FORMAT_I420, frame->coded_size(), frame->visible_rect(), | |
241 frame->natural_size(), frame->timestamp()); | |
242 | |
243 sk_sp<SkSurface> surface = SkSurface::MakeRasterN32Premul( | |
244 frame->visible_rect().width(), frame->visible_rect().height()); | |
245 | |
246 if (surface && provider_) { | |
247 DCHECK(provider_->ContextGL()); | |
248 canvas_video_renderer_.Copy( | |
249 frame.get(), surface->getCanvas(), | |
250 media::Context3D(provider_->ContextGL(), provider_->GrContext())); | |
251 } else { | |
252 // Return a black frame (yuv = {0, 0x80, 0x80}). | |
253 *new_frame = media::VideoFrame::CreateColorFrame( | |
254 frame->visible_rect().size(), 0u, 0x80, 0x80, frame->timestamp()); | |
mcasas
2016/10/26 16:06:02
Shouldn't you return here (and in l.259)? Ah but t
emircan
2016/10/27 00:37:56
Thanks. I added it to this file and restructured t
| |
255 } | |
256 | |
257 SkPixmap pixmap; | |
258 const bool result = surface->getCanvas()->peekPixels(&pixmap); | |
259 DCHECK(result) << "Error trying to access SkSurface's pixels"; | |
260 | |
261 const uint32 source_pixel_format = | |
262 (kN32_SkColorType == kRGBA_8888_SkColorType) ? cricket::FOURCC_ABGR | |
263 : cricket::FOURCC_ARGB; | |
264 libyuv::ConvertToI420(static_cast<const uint8*>(pixmap.addr(0, 0)), | |
265 pixmap.getSafeSize64(), | |
266 (*new_frame)->visible_data(media::VideoFrame::kYPlane), | |
267 (*new_frame)->stride(media::VideoFrame::kYPlane), | |
268 (*new_frame)->visible_data(media::VideoFrame::kUPlane), | |
269 (*new_frame)->stride(media::VideoFrame::kUPlane), | |
270 (*new_frame)->visible_data(media::VideoFrame::kVPlane), | |
271 (*new_frame)->stride(media::VideoFrame::kVPlane), | |
272 0 /* crop_x */, 0 /* crop_y */, pixmap.width(), | |
273 pixmap.height(), (*new_frame)->visible_rect().width(), | |
274 (*new_frame)->visible_rect().height(), libyuv::kRotate0, | |
275 source_pixel_format); | |
276 | |
277 if (waiter) | |
278 waiter->Signal(); | |
mcasas
2016/10/26 16:06:02
Is there a chance that |waiter| might be nullptr?
emircan
2016/10/27 00:37:56
Done.
| |
279 } | |
280 | |
200 } // namespace content | 281 } // namespace content |
OLD | NEW |