| OLD | NEW |
| 1 /* | 1 /* |
| 2 * Copyright (c) 2012 The WebM project authors. All Rights Reserved. | 2 * Copyright (c) 2012 The WebM project authors. All Rights Reserved. |
| 3 * | 3 * |
| 4 * Use of this source code is governed by a BSD-style license | 4 * Use of this source code is governed by a BSD-style license |
| 5 * that can be found in the LICENSE file in the root of the source | 5 * that can be found in the LICENSE file in the root of the source |
| 6 * tree. An additional intellectual property rights grant can be found | 6 * tree. An additional intellectual property rights grant can be found |
| 7 * in the file PATENTS. All contributing project authors may | 7 * in the file PATENTS. All contributing project authors may |
| 8 * be found in the AUTHORS file in the root of the source tree. | 8 * be found in the AUTHORS file in the root of the source tree. |
| 9 */ | 9 */ |
| 10 #include <immintrin.h> | 10 #include <immintrin.h> |
| 11 #include "vpx_ports/mem.h" | 11 #include "vpx_ports/mem.h" |
| 12 | 12 |
| 13 #define FSAD64_H(h) \ | 13 #define FSAD64_H(h) \ |
| 14 unsigned int vp9_sad64x##h##_avx2(const uint8_t *src_ptr, \ | 14 unsigned int vpx_sad64x##h##_avx2(const uint8_t *src_ptr, \ |
| 15 int src_stride, \ | 15 int src_stride, \ |
| 16 const uint8_t *ref_ptr, \ | 16 const uint8_t *ref_ptr, \ |
| 17 int ref_stride) { \ | 17 int ref_stride) { \ |
| 18 int i, res; \ | 18 int i, res; \ |
| 19 __m256i sad1_reg, sad2_reg, ref1_reg, ref2_reg; \ | 19 __m256i sad1_reg, sad2_reg, ref1_reg, ref2_reg; \ |
| 20 __m256i sum_sad = _mm256_setzero_si256(); \ | 20 __m256i sum_sad = _mm256_setzero_si256(); \ |
| 21 __m256i sum_sad_h; \ | 21 __m256i sum_sad_h; \ |
| 22 __m128i sum_sad128; \ | 22 __m128i sum_sad128; \ |
| 23 for (i = 0 ; i < h ; i++) { \ | 23 for (i = 0 ; i < h ; i++) { \ |
| 24 ref1_reg = _mm256_loadu_si256((__m256i const *)ref_ptr); \ | 24 ref1_reg = _mm256_loadu_si256((__m256i const *)ref_ptr); \ |
| 25 ref2_reg = _mm256_loadu_si256((__m256i const *)(ref_ptr + 32)); \ | 25 ref2_reg = _mm256_loadu_si256((__m256i const *)(ref_ptr + 32)); \ |
| 26 sad1_reg = _mm256_sad_epu8(ref1_reg, \ | 26 sad1_reg = _mm256_sad_epu8(ref1_reg, \ |
| 27 _mm256_loadu_si256((__m256i const *)src_ptr)); \ | 27 _mm256_loadu_si256((__m256i const *)src_ptr)); \ |
| 28 sad2_reg = _mm256_sad_epu8(ref2_reg, \ | 28 sad2_reg = _mm256_sad_epu8(ref2_reg, \ |
| 29 _mm256_loadu_si256((__m256i const *)(src_ptr + 32))); \ | 29 _mm256_loadu_si256((__m256i const *)(src_ptr + 32))); \ |
| 30 sum_sad = _mm256_add_epi32(sum_sad, _mm256_add_epi32(sad1_reg, sad2_reg)); \ | 30 sum_sad = _mm256_add_epi32(sum_sad, _mm256_add_epi32(sad1_reg, sad2_reg)); \ |
| 31 ref_ptr+= ref_stride; \ | 31 ref_ptr+= ref_stride; \ |
| 32 src_ptr+= src_stride; \ | 32 src_ptr+= src_stride; \ |
| 33 } \ | 33 } \ |
| 34 sum_sad_h = _mm256_srli_si256(sum_sad, 8); \ | 34 sum_sad_h = _mm256_srli_si256(sum_sad, 8); \ |
| 35 sum_sad = _mm256_add_epi32(sum_sad, sum_sad_h); \ | 35 sum_sad = _mm256_add_epi32(sum_sad, sum_sad_h); \ |
| 36 sum_sad128 = _mm256_extracti128_si256(sum_sad, 1); \ | 36 sum_sad128 = _mm256_extracti128_si256(sum_sad, 1); \ |
| 37 sum_sad128 = _mm_add_epi32(_mm256_castsi256_si128(sum_sad), sum_sad128); \ | 37 sum_sad128 = _mm_add_epi32(_mm256_castsi256_si128(sum_sad), sum_sad128); \ |
| 38 res = _mm_cvtsi128_si32(sum_sad128); \ | 38 res = _mm_cvtsi128_si32(sum_sad128); \ |
| 39 return res; \ | 39 return res; \ |
| 40 } | 40 } |
| 41 | 41 |
| 42 #define FSAD32_H(h) \ | 42 #define FSAD32_H(h) \ |
| 43 unsigned int vp9_sad32x##h##_avx2(const uint8_t *src_ptr, \ | 43 unsigned int vpx_sad32x##h##_avx2(const uint8_t *src_ptr, \ |
| 44 int src_stride, \ | 44 int src_stride, \ |
| 45 const uint8_t *ref_ptr, \ | 45 const uint8_t *ref_ptr, \ |
| 46 int ref_stride) { \ | 46 int ref_stride) { \ |
| 47 int i, res; \ | 47 int i, res; \ |
| 48 __m256i sad1_reg, sad2_reg, ref1_reg, ref2_reg; \ | 48 __m256i sad1_reg, sad2_reg, ref1_reg, ref2_reg; \ |
| 49 __m256i sum_sad = _mm256_setzero_si256(); \ | 49 __m256i sum_sad = _mm256_setzero_si256(); \ |
| 50 __m256i sum_sad_h; \ | 50 __m256i sum_sad_h; \ |
| 51 __m128i sum_sad128; \ | 51 __m128i sum_sad128; \ |
| 52 int ref2_stride = ref_stride << 1; \ | 52 int ref2_stride = ref_stride << 1; \ |
| 53 int src2_stride = src_stride << 1; \ | 53 int src2_stride = src_stride << 1; \ |
| (...skipping 28 matching lines...) Expand all Loading... |
| 82 | 82 |
| 83 FSAD64; | 83 FSAD64; |
| 84 FSAD32; | 84 FSAD32; |
| 85 | 85 |
| 86 #undef FSAD64 | 86 #undef FSAD64 |
| 87 #undef FSAD32 | 87 #undef FSAD32 |
| 88 #undef FSAD64_H | 88 #undef FSAD64_H |
| 89 #undef FSAD32_H | 89 #undef FSAD32_H |
| 90 | 90 |
| 91 #define FSADAVG64_H(h) \ | 91 #define FSADAVG64_H(h) \ |
| 92 unsigned int vp9_sad64x##h##_avg_avx2(const uint8_t *src_ptr, \ | 92 unsigned int vpx_sad64x##h##_avg_avx2(const uint8_t *src_ptr, \ |
| 93 int src_stride, \ | 93 int src_stride, \ |
| 94 const uint8_t *ref_ptr, \ | 94 const uint8_t *ref_ptr, \ |
| 95 int ref_stride, \ | 95 int ref_stride, \ |
| 96 const uint8_t *second_pred) { \ | 96 const uint8_t *second_pred) { \ |
| 97 int i, res; \ | 97 int i, res; \ |
| 98 __m256i sad1_reg, sad2_reg, ref1_reg, ref2_reg; \ | 98 __m256i sad1_reg, sad2_reg, ref1_reg, ref2_reg; \ |
| 99 __m256i sum_sad = _mm256_setzero_si256(); \ | 99 __m256i sum_sad = _mm256_setzero_si256(); \ |
| 100 __m256i sum_sad_h; \ | 100 __m256i sum_sad_h; \ |
| 101 __m128i sum_sad128; \ | 101 __m128i sum_sad128; \ |
| 102 for (i = 0 ; i < h ; i++) { \ | 102 for (i = 0 ; i < h ; i++) { \ |
| (...skipping 14 matching lines...) Expand all Loading... |
| 117 } \ | 117 } \ |
| 118 sum_sad_h = _mm256_srli_si256(sum_sad, 8); \ | 118 sum_sad_h = _mm256_srli_si256(sum_sad, 8); \ |
| 119 sum_sad = _mm256_add_epi32(sum_sad, sum_sad_h); \ | 119 sum_sad = _mm256_add_epi32(sum_sad, sum_sad_h); \ |
| 120 sum_sad128 = _mm256_extracti128_si256(sum_sad, 1); \ | 120 sum_sad128 = _mm256_extracti128_si256(sum_sad, 1); \ |
| 121 sum_sad128 = _mm_add_epi32(_mm256_castsi256_si128(sum_sad), sum_sad128); \ | 121 sum_sad128 = _mm_add_epi32(_mm256_castsi256_si128(sum_sad), sum_sad128); \ |
| 122 res = _mm_cvtsi128_si32(sum_sad128); \ | 122 res = _mm_cvtsi128_si32(sum_sad128); \ |
| 123 return res; \ | 123 return res; \ |
| 124 } | 124 } |
| 125 | 125 |
| 126 #define FSADAVG32_H(h) \ | 126 #define FSADAVG32_H(h) \ |
| 127 unsigned int vp9_sad32x##h##_avg_avx2(const uint8_t *src_ptr, \ | 127 unsigned int vpx_sad32x##h##_avg_avx2(const uint8_t *src_ptr, \ |
| 128 int src_stride, \ | 128 int src_stride, \ |
| 129 const uint8_t *ref_ptr, \ | 129 const uint8_t *ref_ptr, \ |
| 130 int ref_stride, \ | 130 int ref_stride, \ |
| 131 const uint8_t *second_pred) { \ | 131 const uint8_t *second_pred) { \ |
| 132 int i, res; \ | 132 int i, res; \ |
| 133 __m256i sad1_reg, sad2_reg, ref1_reg, ref2_reg; \ | 133 __m256i sad1_reg, sad2_reg, ref1_reg, ref2_reg; \ |
| 134 __m256i sum_sad = _mm256_setzero_si256(); \ | 134 __m256i sum_sad = _mm256_setzero_si256(); \ |
| 135 __m256i sum_sad_h; \ | 135 __m256i sum_sad_h; \ |
| 136 __m128i sum_sad128; \ | 136 __m128i sum_sad128; \ |
| 137 int ref2_stride = ref_stride << 1; \ | 137 int ref2_stride = ref_stride << 1; \ |
| (...skipping 33 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 171 FSADAVG32_H(32); \ | 171 FSADAVG32_H(32); \ |
| 172 FSADAVG32_H(16); | 172 FSADAVG32_H(16); |
| 173 | 173 |
| 174 FSADAVG64; | 174 FSADAVG64; |
| 175 FSADAVG32; | 175 FSADAVG32; |
| 176 | 176 |
| 177 #undef FSADAVG64 | 177 #undef FSADAVG64 |
| 178 #undef FSADAVG32 | 178 #undef FSADAVG32 |
| 179 #undef FSADAVG64_H | 179 #undef FSADAVG64_H |
| 180 #undef FSADAVG32_H | 180 #undef FSADAVG32_H |
| OLD | NEW |