OLD | NEW |
1 /* | 1 /* |
2 * Copyright (c) 2014 The WebM project authors. All Rights Reserved. | 2 * Copyright (c) 2014 The WebM project authors. All Rights Reserved. |
3 * | 3 * |
4 * Use of this source code is governed by a BSD-style license | 4 * Use of this source code is governed by a BSD-style license |
5 * that can be found in the LICENSE file in the root of the source | 5 * that can be found in the LICENSE file in the root of the source |
6 * tree. An additional intellectual property rights grant can be found | 6 * tree. An additional intellectual property rights grant can be found |
7 * in the file PATENTS. All contributing project authors may | 7 * in the file PATENTS. All contributing project authors may |
8 * be found in the AUTHORS file in the root of the source tree. | 8 * be found in the AUTHORS file in the root of the source tree. |
9 */ | 9 */ |
10 | 10 |
11 #include <arm_neon.h> | 11 #include <arm_neon.h> |
12 #include "./vp9_rtcd.h" | 12 #include "./vp9_rtcd.h" |
13 #include "./vpx_config.h" | 13 #include "./vpx_config.h" |
14 | 14 |
15 #include "vpx_ports/mem.h" | 15 #include "vpx_ports/mem.h" |
16 #include "vpx/vpx_integer.h" | 16 #include "vpx/vpx_integer.h" |
17 | 17 |
18 #include "vp9/common/vp9_common.h" | 18 #include "vp9/common/vp9_common.h" |
19 #include "vp9/common/vp9_filter.h" | 19 #include "vp9/common/vp9_filter.h" |
20 | 20 |
21 #include "vp9/encoder/vp9_variance.h" | 21 #include "vp9/encoder/vp9_variance.h" |
22 | 22 |
23 enum { kWidth8 = 8 }; | |
24 enum { kHeight8 = 8 }; | |
25 enum { kHeight8PlusOne = 9 }; | |
26 enum { kWidth16 = 16 }; | |
27 enum { kHeight16 = 16 }; | |
28 enum { kHeight16PlusOne = 17 }; | |
29 enum { kWidth32 = 32 }; | |
30 enum { kHeight32 = 32 }; | |
31 enum { kHeight32PlusOne = 33 }; | |
32 enum { kWidth64 = 64 }; | |
33 enum { kHeight64 = 64 }; | |
34 enum { kHeight64PlusOne = 65 }; | |
35 enum { kPixelStepOne = 1 }; | |
36 enum { kAlign16 = 16 }; | |
37 | |
38 static INLINE int horizontal_add_s16x8(const int16x8_t v_16x8) { | 23 static INLINE int horizontal_add_s16x8(const int16x8_t v_16x8) { |
39 const int32x4_t a = vpaddlq_s16(v_16x8); | 24 const int32x4_t a = vpaddlq_s16(v_16x8); |
40 const int64x2_t b = vpaddlq_s32(a); | 25 const int64x2_t b = vpaddlq_s32(a); |
41 const int32x2_t c = vadd_s32(vreinterpret_s32_s64(vget_low_s64(b)), | 26 const int32x2_t c = vadd_s32(vreinterpret_s32_s64(vget_low_s64(b)), |
42 vreinterpret_s32_s64(vget_high_s64(b))); | 27 vreinterpret_s32_s64(vget_high_s64(b))); |
43 return vget_lane_s32(c, 0); | 28 return vget_lane_s32(c, 0); |
44 } | 29 } |
45 | 30 |
46 static INLINE int horizontal_add_s32x4(const int32x4_t v_32x4) { | 31 static INLINE int horizontal_add_s32x4(const int32x4_t v_32x4) { |
47 const int64x2_t b = vpaddlq_s32(v_32x4); | 32 const int64x2_t b = vpaddlq_s32(v_32x4); |
(...skipping 29 matching lines...) Expand all Loading... |
77 b += b_stride; | 62 b += b_stride; |
78 } | 63 } |
79 | 64 |
80 *sum = horizontal_add_s16x8(v_sum); | 65 *sum = horizontal_add_s16x8(v_sum); |
81 *sse = (unsigned int)horizontal_add_s32x4(vaddq_s32(v_sse_lo, v_sse_hi)); | 66 *sse = (unsigned int)horizontal_add_s32x4(vaddq_s32(v_sse_lo, v_sse_hi)); |
82 } | 67 } |
83 | 68 |
84 void vp9_get8x8var_neon(const uint8_t *src_ptr, int source_stride, | 69 void vp9_get8x8var_neon(const uint8_t *src_ptr, int source_stride, |
85 const uint8_t *ref_ptr, int ref_stride, | 70 const uint8_t *ref_ptr, int ref_stride, |
86 unsigned int *sse, int *sum) { | 71 unsigned int *sse, int *sum) { |
87 variance_neon_w8(src_ptr, source_stride, ref_ptr, ref_stride, kWidth8, | 72 variance_neon_w8(src_ptr, source_stride, ref_ptr, ref_stride, 8, |
88 kHeight8, sse, sum); | 73 8, sse, sum); |
89 } | 74 } |
90 | 75 |
91 unsigned int vp9_variance8x8_neon(const uint8_t *a, int a_stride, | 76 unsigned int vp9_variance8x8_neon(const uint8_t *a, int a_stride, |
92 const uint8_t *b, int b_stride, | 77 const uint8_t *b, int b_stride, |
93 unsigned int *sse) { | 78 unsigned int *sse) { |
94 int sum; | 79 int sum; |
95 variance_neon_w8(a, a_stride, b, b_stride, kWidth8, kHeight8, sse, &sum); | 80 variance_neon_w8(a, a_stride, b, b_stride, 8, 8, sse, &sum); |
96 return *sse - (((int64_t)sum * sum) >> 6); // >> 6 = / 8 * 8 | 81 return *sse - (((int64_t)sum * sum) >> 6); // >> 6 = / 8 * 8 |
97 } | 82 } |
98 | 83 |
99 void vp9_get16x16var_neon(const uint8_t *src_ptr, int source_stride, | 84 void vp9_get16x16var_neon(const uint8_t *src_ptr, int source_stride, |
100 const uint8_t *ref_ptr, int ref_stride, | 85 const uint8_t *ref_ptr, int ref_stride, |
101 unsigned int *sse, int *sum) { | 86 unsigned int *sse, int *sum) { |
102 variance_neon_w8(src_ptr, source_stride, ref_ptr, ref_stride, kWidth16, | 87 variance_neon_w8(src_ptr, source_stride, ref_ptr, ref_stride, 16, |
103 kHeight16, sse, sum); | 88 16, sse, sum); |
104 } | 89 } |
105 | 90 |
106 unsigned int vp9_variance16x16_neon(const uint8_t *a, int a_stride, | 91 unsigned int vp9_variance16x16_neon(const uint8_t *a, int a_stride, |
107 const uint8_t *b, int b_stride, | 92 const uint8_t *b, int b_stride, |
108 unsigned int *sse) { | 93 unsigned int *sse) { |
109 int sum; | 94 int sum; |
110 variance_neon_w8(a, a_stride, b, b_stride, kWidth16, kHeight16, sse, &sum); | 95 variance_neon_w8(a, a_stride, b, b_stride, 16, 16, sse, &sum); |
111 return *sse - (((int64_t)sum * sum) >> 8); // >> 8 = / 16 * 16 | 96 return *sse - (((int64_t)sum * sum) >> 8); // >> 8 = / 16 * 16 |
112 } | 97 } |
113 | 98 |
114 static void var_filter_block2d_bil_w8(const uint8_t *src_ptr, | 99 static void var_filter_block2d_bil_w8(const uint8_t *src_ptr, |
115 uint8_t *output_ptr, | 100 uint8_t *output_ptr, |
116 unsigned int src_pixels_per_line, | 101 unsigned int src_pixels_per_line, |
117 int pixel_step, | 102 int pixel_step, |
118 unsigned int output_height, | 103 unsigned int output_height, |
119 unsigned int output_width, | 104 unsigned int output_width, |
120 const int16_t *vp9_filter) { | 105 const int16_t *vp9_filter) { |
(...skipping 41 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
162 } | 147 } |
163 } | 148 } |
164 | 149 |
165 unsigned int vp9_sub_pixel_variance8x8_neon(const uint8_t *src, | 150 unsigned int vp9_sub_pixel_variance8x8_neon(const uint8_t *src, |
166 int src_stride, | 151 int src_stride, |
167 int xoffset, | 152 int xoffset, |
168 int yoffset, | 153 int yoffset, |
169 const uint8_t *dst, | 154 const uint8_t *dst, |
170 int dst_stride, | 155 int dst_stride, |
171 unsigned int *sse) { | 156 unsigned int *sse) { |
172 DECLARE_ALIGNED_ARRAY(kAlign16, uint8_t, temp2, kHeight8 * kWidth8); | 157 DECLARE_ALIGNED(16, uint8_t, temp2[8 * 8]); |
173 DECLARE_ALIGNED_ARRAY(kAlign16, uint8_t, fdata3, kHeight8PlusOne * kWidth8); | 158 DECLARE_ALIGNED(16, uint8_t, fdata3[9 * 8]); |
174 | 159 |
175 var_filter_block2d_bil_w8(src, fdata3, src_stride, kPixelStepOne, | 160 var_filter_block2d_bil_w8(src, fdata3, src_stride, 1, |
176 kHeight8PlusOne, kWidth8, | 161 9, 8, |
177 BILINEAR_FILTERS_2TAP(xoffset)); | 162 BILINEAR_FILTERS_2TAP(xoffset)); |
178 var_filter_block2d_bil_w8(fdata3, temp2, kWidth8, kWidth8, kHeight8, | 163 var_filter_block2d_bil_w8(fdata3, temp2, 8, 8, 8, |
179 kWidth8, BILINEAR_FILTERS_2TAP(yoffset)); | 164 8, BILINEAR_FILTERS_2TAP(yoffset)); |
180 return vp9_variance8x8_neon(temp2, kWidth8, dst, dst_stride, sse); | 165 return vp9_variance8x8_neon(temp2, 8, dst, dst_stride, sse); |
181 } | 166 } |
182 | 167 |
183 unsigned int vp9_sub_pixel_variance16x16_neon(const uint8_t *src, | 168 unsigned int vp9_sub_pixel_variance16x16_neon(const uint8_t *src, |
184 int src_stride, | 169 int src_stride, |
185 int xoffset, | 170 int xoffset, |
186 int yoffset, | 171 int yoffset, |
187 const uint8_t *dst, | 172 const uint8_t *dst, |
188 int dst_stride, | 173 int dst_stride, |
189 unsigned int *sse) { | 174 unsigned int *sse) { |
190 DECLARE_ALIGNED_ARRAY(kAlign16, uint8_t, temp2, kHeight16 * kWidth16); | 175 DECLARE_ALIGNED(16, uint8_t, temp2[16 * 16]); |
191 DECLARE_ALIGNED_ARRAY(kAlign16, uint8_t, fdata3, kHeight16PlusOne * kWidth16); | 176 DECLARE_ALIGNED(16, uint8_t, fdata3[17 * 16]); |
192 | 177 |
193 var_filter_block2d_bil_w16(src, fdata3, src_stride, kPixelStepOne, | 178 var_filter_block2d_bil_w16(src, fdata3, src_stride, 1, |
194 kHeight16PlusOne, kWidth16, | 179 17, 16, |
195 BILINEAR_FILTERS_2TAP(xoffset)); | 180 BILINEAR_FILTERS_2TAP(xoffset)); |
196 var_filter_block2d_bil_w16(fdata3, temp2, kWidth16, kWidth16, kHeight16, | 181 var_filter_block2d_bil_w16(fdata3, temp2, 16, 16, 16, |
197 kWidth16, BILINEAR_FILTERS_2TAP(yoffset)); | 182 16, BILINEAR_FILTERS_2TAP(yoffset)); |
198 return vp9_variance16x16_neon(temp2, kWidth16, dst, dst_stride, sse); | 183 return vp9_variance16x16_neon(temp2, 16, dst, dst_stride, sse); |
199 } | 184 } |
200 | 185 |
201 void vp9_get32x32var_neon(const uint8_t *src_ptr, int source_stride, | 186 void vp9_get32x32var_neon(const uint8_t *src_ptr, int source_stride, |
202 const uint8_t *ref_ptr, int ref_stride, | 187 const uint8_t *ref_ptr, int ref_stride, |
203 unsigned int *sse, int *sum) { | 188 unsigned int *sse, int *sum) { |
204 variance_neon_w8(src_ptr, source_stride, ref_ptr, ref_stride, kWidth32, | 189 variance_neon_w8(src_ptr, source_stride, ref_ptr, ref_stride, 32, |
205 kHeight32, sse, sum); | 190 32, sse, sum); |
206 } | 191 } |
207 | 192 |
208 unsigned int vp9_variance32x32_neon(const uint8_t *a, int a_stride, | 193 unsigned int vp9_variance32x32_neon(const uint8_t *a, int a_stride, |
209 const uint8_t *b, int b_stride, | 194 const uint8_t *b, int b_stride, |
210 unsigned int *sse) { | 195 unsigned int *sse) { |
211 int sum; | 196 int sum; |
212 variance_neon_w8(a, a_stride, b, b_stride, kWidth32, kHeight32, sse, &sum); | 197 variance_neon_w8(a, a_stride, b, b_stride, 32, 32, sse, &sum); |
213 return *sse - (((int64_t)sum * sum) >> 10); // >> 10 = / 32 * 32 | 198 return *sse - (((int64_t)sum * sum) >> 10); // >> 10 = / 32 * 32 |
214 } | 199 } |
215 | 200 |
216 unsigned int vp9_variance32x64_neon(const uint8_t *a, int a_stride, | 201 unsigned int vp9_variance32x64_neon(const uint8_t *a, int a_stride, |
217 const uint8_t *b, int b_stride, | 202 const uint8_t *b, int b_stride, |
218 unsigned int *sse) { | 203 unsigned int *sse) { |
219 int sum1, sum2; | 204 int sum1, sum2; |
220 uint32_t sse1, sse2; | 205 uint32_t sse1, sse2; |
221 variance_neon_w8(a, a_stride, b, b_stride, kWidth32, kHeight32, &sse1, &sum1); | 206 variance_neon_w8(a, a_stride, b, b_stride, 32, 32, &sse1, &sum1); |
222 variance_neon_w8(a + (kHeight32 * a_stride), a_stride, | 207 variance_neon_w8(a + (32 * a_stride), a_stride, |
223 b + (kHeight32 * b_stride), b_stride, kWidth32, kHeight32, | 208 b + (32 * b_stride), b_stride, 32, 32, |
224 &sse2, &sum2); | 209 &sse2, &sum2); |
225 *sse = sse1 + sse2; | 210 *sse = sse1 + sse2; |
226 sum1 += sum2; | 211 sum1 += sum2; |
227 return *sse - (((int64_t)sum1 * sum1) >> 11); // >> 11 = / 32 * 64 | 212 return *sse - (((int64_t)sum1 * sum1) >> 11); // >> 11 = / 32 * 64 |
228 } | 213 } |
229 | 214 |
230 unsigned int vp9_variance64x32_neon(const uint8_t *a, int a_stride, | 215 unsigned int vp9_variance64x32_neon(const uint8_t *a, int a_stride, |
231 const uint8_t *b, int b_stride, | 216 const uint8_t *b, int b_stride, |
232 unsigned int *sse) { | 217 unsigned int *sse) { |
233 int sum1, sum2; | 218 int sum1, sum2; |
234 uint32_t sse1, sse2; | 219 uint32_t sse1, sse2; |
235 variance_neon_w8(a, a_stride, b, b_stride, kWidth64, kHeight16, &sse1, &sum1); | 220 variance_neon_w8(a, a_stride, b, b_stride, 64, 16, &sse1, &sum1); |
236 variance_neon_w8(a + (kHeight16 * a_stride), a_stride, | 221 variance_neon_w8(a + (16 * a_stride), a_stride, |
237 b + (kHeight16 * b_stride), b_stride, kWidth64, kHeight16, | 222 b + (16 * b_stride), b_stride, 64, 16, |
238 &sse2, &sum2); | 223 &sse2, &sum2); |
239 *sse = sse1 + sse2; | 224 *sse = sse1 + sse2; |
240 sum1 += sum2; | 225 sum1 += sum2; |
241 return *sse - (((int64_t)sum1 * sum1) >> 11); // >> 11 = / 32 * 64 | 226 return *sse - (((int64_t)sum1 * sum1) >> 11); // >> 11 = / 32 * 64 |
242 } | 227 } |
243 | 228 |
244 unsigned int vp9_variance64x64_neon(const uint8_t *a, int a_stride, | 229 unsigned int vp9_variance64x64_neon(const uint8_t *a, int a_stride, |
245 const uint8_t *b, int b_stride, | 230 const uint8_t *b, int b_stride, |
246 unsigned int *sse) { | 231 unsigned int *sse) { |
247 int sum1, sum2; | 232 int sum1, sum2; |
248 uint32_t sse1, sse2; | 233 uint32_t sse1, sse2; |
249 | 234 |
250 variance_neon_w8(a, a_stride, b, b_stride, kWidth64, kHeight16, &sse1, &sum1); | 235 variance_neon_w8(a, a_stride, b, b_stride, 64, 16, &sse1, &sum1); |
251 variance_neon_w8(a + (kHeight16 * a_stride), a_stride, | 236 variance_neon_w8(a + (16 * a_stride), a_stride, |
252 b + (kHeight16 * b_stride), b_stride, kWidth64, kHeight16, | 237 b + (16 * b_stride), b_stride, 64, 16, |
253 &sse2, &sum2); | 238 &sse2, &sum2); |
254 sse1 += sse2; | 239 sse1 += sse2; |
255 sum1 += sum2; | 240 sum1 += sum2; |
256 | 241 |
257 variance_neon_w8(a + (kHeight16 * 2 * a_stride), a_stride, | 242 variance_neon_w8(a + (16 * 2 * a_stride), a_stride, |
258 b + (kHeight16 * 2 * b_stride), b_stride, | 243 b + (16 * 2 * b_stride), b_stride, |
259 kWidth64, kHeight16, &sse2, &sum2); | 244 64, 16, &sse2, &sum2); |
260 sse1 += sse2; | 245 sse1 += sse2; |
261 sum1 += sum2; | 246 sum1 += sum2; |
262 | 247 |
263 variance_neon_w8(a + (kHeight16 * 3 * a_stride), a_stride, | 248 variance_neon_w8(a + (16 * 3 * a_stride), a_stride, |
264 b + (kHeight16 * 3 * b_stride), b_stride, | 249 b + (16 * 3 * b_stride), b_stride, |
265 kWidth64, kHeight16, &sse2, &sum2); | 250 64, 16, &sse2, &sum2); |
266 *sse = sse1 + sse2; | 251 *sse = sse1 + sse2; |
267 sum1 += sum2; | 252 sum1 += sum2; |
268 return *sse - (((int64_t)sum1 * sum1) >> 12); // >> 12 = / 64 * 64 | 253 return *sse - (((int64_t)sum1 * sum1) >> 12); // >> 12 = / 64 * 64 |
269 } | 254 } |
270 | 255 |
271 unsigned int vp9_sub_pixel_variance32x32_neon(const uint8_t *src, | 256 unsigned int vp9_sub_pixel_variance32x32_neon(const uint8_t *src, |
272 int src_stride, | 257 int src_stride, |
273 int xoffset, | 258 int xoffset, |
274 int yoffset, | 259 int yoffset, |
275 const uint8_t *dst, | 260 const uint8_t *dst, |
276 int dst_stride, | 261 int dst_stride, |
277 unsigned int *sse) { | 262 unsigned int *sse) { |
278 DECLARE_ALIGNED_ARRAY(kAlign16, uint8_t, temp2, kHeight32 * kWidth32); | 263 DECLARE_ALIGNED(16, uint8_t, temp2[32 * 32]); |
279 DECLARE_ALIGNED_ARRAY(kAlign16, uint8_t, fdata3, kHeight32PlusOne * kWidth32); | 264 DECLARE_ALIGNED(16, uint8_t, fdata3[33 * 32]); |
280 | 265 |
281 var_filter_block2d_bil_w16(src, fdata3, src_stride, kPixelStepOne, | 266 var_filter_block2d_bil_w16(src, fdata3, src_stride, 1, |
282 kHeight32PlusOne, kWidth32, | 267 33, 32, |
283 BILINEAR_FILTERS_2TAP(xoffset)); | 268 BILINEAR_FILTERS_2TAP(xoffset)); |
284 var_filter_block2d_bil_w16(fdata3, temp2, kWidth32, kWidth32, kHeight32, | 269 var_filter_block2d_bil_w16(fdata3, temp2, 32, 32, 32, |
285 kWidth32, BILINEAR_FILTERS_2TAP(yoffset)); | 270 32, BILINEAR_FILTERS_2TAP(yoffset)); |
286 return vp9_variance32x32_neon(temp2, kWidth32, dst, dst_stride, sse); | 271 return vp9_variance32x32_neon(temp2, 32, dst, dst_stride, sse); |
287 } | 272 } |
288 | 273 |
289 unsigned int vp9_sub_pixel_variance64x64_neon(const uint8_t *src, | 274 unsigned int vp9_sub_pixel_variance64x64_neon(const uint8_t *src, |
290 int src_stride, | 275 int src_stride, |
291 int xoffset, | 276 int xoffset, |
292 int yoffset, | 277 int yoffset, |
293 const uint8_t *dst, | 278 const uint8_t *dst, |
294 int dst_stride, | 279 int dst_stride, |
295 unsigned int *sse) { | 280 unsigned int *sse) { |
296 DECLARE_ALIGNED_ARRAY(kAlign16, uint8_t, temp2, kHeight64 * kWidth64); | 281 DECLARE_ALIGNED(16, uint8_t, temp2[64 * 64]); |
297 DECLARE_ALIGNED_ARRAY(kAlign16, uint8_t, fdata3, kHeight64PlusOne * kWidth64); | 282 DECLARE_ALIGNED(16, uint8_t, fdata3[65 * 64]); |
298 | 283 |
299 var_filter_block2d_bil_w16(src, fdata3, src_stride, kPixelStepOne, | 284 var_filter_block2d_bil_w16(src, fdata3, src_stride, 1, |
300 kHeight64PlusOne, kWidth64, | 285 65, 64, |
301 BILINEAR_FILTERS_2TAP(xoffset)); | 286 BILINEAR_FILTERS_2TAP(xoffset)); |
302 var_filter_block2d_bil_w16(fdata3, temp2, kWidth64, kWidth64, kHeight64, | 287 var_filter_block2d_bil_w16(fdata3, temp2, 64, 64, 64, |
303 kWidth64, BILINEAR_FILTERS_2TAP(yoffset)); | 288 64, BILINEAR_FILTERS_2TAP(yoffset)); |
304 return vp9_variance64x64_neon(temp2, kWidth64, dst, dst_stride, sse); | 289 return vp9_variance64x64_neon(temp2, 64, dst, dst_stride, sse); |
305 } | 290 } |
OLD | NEW |