OLD | NEW |
1 // Copyright (c) 2012 The Chromium Authors. All rights reserved. | 1 // Copyright (c) 2012 The Chromium Authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
4 | 4 |
5 #include "media/base/vector_math.h" | 5 #include "media/base/vector_math.h" |
6 #include "media/base/vector_math_testing.h" | 6 #include "media/base/vector_math_testing.h" |
7 | 7 |
8 #include <algorithm> | 8 #include <algorithm> |
9 | 9 |
10 #include "base/cpu.h" | |
11 #include "base/logging.h" | 10 #include "base/logging.h" |
12 #include "build/build_config.h" | 11 #include "build/build_config.h" |
13 | 12 |
14 #if defined(ARCH_CPU_ARM_FAMILY) && defined(USE_NEON) | 13 // NaCl does not allow intrinsics. |
| 14 #if defined(ARCH_CPU_X86_FAMILY) && !defined(OS_NACL) |
| 15 #include <xmmintrin.h> |
| 16 #define FMAC_FUNC FMAC_SSE |
| 17 #define FMUL_FUNC FMUL_SSE |
| 18 #define EWMAAndMaxPower_FUNC EWMAAndMaxPower_SSE |
| 19 #elif defined(ARCH_CPU_ARM_FAMILY) && defined(USE_NEON) |
15 #include <arm_neon.h> | 20 #include <arm_neon.h> |
| 21 #define FMAC_FUNC FMAC_NEON |
| 22 #define FMUL_FUNC FMUL_NEON |
| 23 #define EWMAAndMaxPower_FUNC EWMAAndMaxPower_NEON |
| 24 #else |
| 25 #define FMAC_FUNC FMAC_C |
| 26 #define FMUL_FUNC FMUL_C |
| 27 #define EWMAAndMaxPower_FUNC EWMAAndMaxPower_C |
16 #endif | 28 #endif |
17 | 29 |
18 namespace media { | 30 namespace media { |
19 namespace vector_math { | 31 namespace vector_math { |
20 | 32 |
21 // If we know the minimum architecture at compile time, avoid CPU detection. | |
22 // Force NaCl code to use C routines since (at present) nothing there uses these | |
23 // methods and plumbing the -msse built library is non-trivial. | |
24 #if defined(ARCH_CPU_X86_FAMILY) && !defined(OS_NACL) | |
25 #if defined(__SSE__) | |
26 #define FMAC_FUNC FMAC_SSE | |
27 #define FMUL_FUNC FMUL_SSE | |
28 #define EWMAAndMaxPower_FUNC EWMAAndMaxPower_SSE | |
29 void Initialize() {} | |
30 #else | |
31 // X86 CPU detection required. Functions will be set by Initialize(). | |
32 // TODO(dalecurtis): Once Chrome moves to an SSE baseline this can be removed. | |
33 #define FMAC_FUNC g_fmac_proc_ | |
34 #define FMUL_FUNC g_fmul_proc_ | |
35 #define EWMAAndMaxPower_FUNC g_ewma_power_proc_ | |
36 | |
37 typedef void (*MathProc)(const float src[], float scale, int len, float dest[]); | |
38 static MathProc g_fmac_proc_ = NULL; | |
39 static MathProc g_fmul_proc_ = NULL; | |
40 typedef std::pair<float, float> (*EWMAAndMaxPowerProc)( | |
41 float initial_value, const float src[], int len, float smoothing_factor); | |
42 static EWMAAndMaxPowerProc g_ewma_power_proc_ = NULL; | |
43 | |
44 void Initialize() { | |
45 CHECK(!g_fmac_proc_); | |
46 CHECK(!g_fmul_proc_); | |
47 CHECK(!g_ewma_power_proc_); | |
48 const bool kUseSSE = base::CPU().has_sse(); | |
49 g_fmac_proc_ = kUseSSE ? FMAC_SSE : FMAC_C; | |
50 g_fmul_proc_ = kUseSSE ? FMUL_SSE : FMUL_C; | |
51 g_ewma_power_proc_ = kUseSSE ? EWMAAndMaxPower_SSE : EWMAAndMaxPower_C; | |
52 } | |
53 #endif | |
54 #elif defined(ARCH_CPU_ARM_FAMILY) && defined(USE_NEON) | |
55 #define FMAC_FUNC FMAC_NEON | |
56 #define FMUL_FUNC FMUL_NEON | |
57 #define EWMAAndMaxPower_FUNC EWMAAndMaxPower_NEON | |
58 void Initialize() {} | |
59 #else | |
60 // Unknown architecture. | |
61 #define FMAC_FUNC FMAC_C | |
62 #define FMUL_FUNC FMUL_C | |
63 #define EWMAAndMaxPower_FUNC EWMAAndMaxPower_C | |
64 void Initialize() {} | |
65 #endif | |
66 | |
67 void FMAC(const float src[], float scale, int len, float dest[]) { | 33 void FMAC(const float src[], float scale, int len, float dest[]) { |
68 // Ensure |src| and |dest| are 16-byte aligned. | 34 // Ensure |src| and |dest| are 16-byte aligned. |
69 DCHECK_EQ(0u, reinterpret_cast<uintptr_t>(src) & (kRequiredAlignment - 1)); | 35 DCHECK_EQ(0u, reinterpret_cast<uintptr_t>(src) & (kRequiredAlignment - 1)); |
70 DCHECK_EQ(0u, reinterpret_cast<uintptr_t>(dest) & (kRequiredAlignment - 1)); | 36 DCHECK_EQ(0u, reinterpret_cast<uintptr_t>(dest) & (kRequiredAlignment - 1)); |
71 return FMAC_FUNC(src, scale, len, dest); | 37 return FMAC_FUNC(src, scale, len, dest); |
72 } | 38 } |
73 | 39 |
74 void FMAC_C(const float src[], float scale, int len, float dest[]) { | 40 void FMAC_C(const float src[], float scale, int len, float dest[]) { |
75 for (int i = 0; i < len; ++i) | 41 for (int i = 0; i < len; ++i) |
76 dest[i] += src[i] * scale; | 42 dest[i] += src[i] * scale; |
(...skipping 32 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
109 for (int i = 0; i < len; ++i) { | 75 for (int i = 0; i < len; ++i) { |
110 result.first *= weight_prev; | 76 result.first *= weight_prev; |
111 const float sample = src[i]; | 77 const float sample = src[i]; |
112 const float sample_squared = sample * sample; | 78 const float sample_squared = sample * sample; |
113 result.first += sample_squared * smoothing_factor; | 79 result.first += sample_squared * smoothing_factor; |
114 result.second = std::max(result.second, sample_squared); | 80 result.second = std::max(result.second, sample_squared); |
115 } | 81 } |
116 return result; | 82 return result; |
117 } | 83 } |
118 | 84 |
| 85 #if defined(ARCH_CPU_X86_FAMILY) && !defined(OS_NACL) |
| 86 void FMUL_SSE(const float src[], float scale, int len, float dest[]) { |
| 87 const int rem = len % 4; |
| 88 const int last_index = len - rem; |
| 89 __m128 m_scale = _mm_set_ps1(scale); |
| 90 for (int i = 0; i < last_index; i += 4) |
| 91 _mm_store_ps(dest + i, _mm_mul_ps(_mm_load_ps(src + i), m_scale)); |
| 92 |
| 93 // Handle any remaining values that wouldn't fit in an SSE pass. |
| 94 for (int i = last_index; i < len; ++i) |
| 95 dest[i] = src[i] * scale; |
| 96 } |
| 97 |
| 98 void FMAC_SSE(const float src[], float scale, int len, float dest[]) { |
| 99 const int rem = len % 4; |
| 100 const int last_index = len - rem; |
| 101 __m128 m_scale = _mm_set_ps1(scale); |
| 102 for (int i = 0; i < last_index; i += 4) { |
| 103 _mm_store_ps(dest + i, _mm_add_ps(_mm_load_ps(dest + i), |
| 104 _mm_mul_ps(_mm_load_ps(src + i), m_scale))); |
| 105 } |
| 106 |
| 107 // Handle any remaining values that wouldn't fit in an SSE pass. |
| 108 for (int i = last_index; i < len; ++i) |
| 109 dest[i] += src[i] * scale; |
| 110 } |
| 111 |
| 112 // Convenience macro to extract float 0 through 3 from the vector |a|. This is |
| 113 // needed because compilers other than clang don't support access via |
| 114 // operator[](). |
| 115 #define EXTRACT_FLOAT(a, i) \ |
| 116 (i == 0 ? \ |
| 117 _mm_cvtss_f32(a) : \ |
| 118 _mm_cvtss_f32(_mm_shuffle_ps(a, a, i))) |
| 119 |
| 120 std::pair<float, float> EWMAAndMaxPower_SSE( |
| 121 float initial_value, const float src[], int len, float smoothing_factor) { |
| 122 // When the recurrence is unrolled, we see that we can split it into 4 |
| 123 // separate lanes of evaluation: |
| 124 // |
| 125 // y[n] = a(S[n]^2) + (1-a)(y[n-1]) |
| 126 // = a(S[n]^2) + (1-a)^1(aS[n-1]^2) + (1-a)^2(aS[n-2]^2) + ... |
| 127 // = z[n] + (1-a)^1(z[n-1]) + (1-a)^2(z[n-2]) + (1-a)^3(z[n-3]) |
| 128 // |
| 129 // where z[n] = a(S[n]^2) + (1-a)^4(z[n-4]) + (1-a)^8(z[n-8]) + ... |
| 130 // |
| 131 // Thus, the strategy here is to compute z[n], z[n-1], z[n-2], and z[n-3] in |
| 132 // each of the 4 lanes, and then combine them to give y[n]. |
| 133 |
| 134 const int rem = len % 4; |
| 135 const int last_index = len - rem; |
| 136 |
| 137 const __m128 smoothing_factor_x4 = _mm_set_ps1(smoothing_factor); |
| 138 const float weight_prev = 1.0f - smoothing_factor; |
| 139 const __m128 weight_prev_x4 = _mm_set_ps1(weight_prev); |
| 140 const __m128 weight_prev_squared_x4 = |
| 141 _mm_mul_ps(weight_prev_x4, weight_prev_x4); |
| 142 const __m128 weight_prev_4th_x4 = |
| 143 _mm_mul_ps(weight_prev_squared_x4, weight_prev_squared_x4); |
| 144 |
| 145 // Compute z[n], z[n-1], z[n-2], and z[n-3] in parallel in lanes 3, 2, 1 and |
| 146 // 0, respectively. |
| 147 __m128 max_x4 = _mm_setzero_ps(); |
| 148 __m128 ewma_x4 = _mm_setr_ps(0.0f, 0.0f, 0.0f, initial_value); |
| 149 int i; |
| 150 for (i = 0; i < last_index; i += 4) { |
| 151 ewma_x4 = _mm_mul_ps(ewma_x4, weight_prev_4th_x4); |
| 152 const __m128 sample_x4 = _mm_load_ps(src + i); |
| 153 const __m128 sample_squared_x4 = _mm_mul_ps(sample_x4, sample_x4); |
| 154 max_x4 = _mm_max_ps(max_x4, sample_squared_x4); |
| 155 // Note: The compiler optimizes this to a single multiply-and-accumulate |
| 156 // instruction: |
| 157 ewma_x4 = _mm_add_ps(ewma_x4, |
| 158 _mm_mul_ps(sample_squared_x4, smoothing_factor_x4)); |
| 159 } |
| 160 |
| 161 // y[n] = z[n] + (1-a)^1(z[n-1]) + (1-a)^2(z[n-2]) + (1-a)^3(z[n-3]) |
| 162 float ewma = EXTRACT_FLOAT(ewma_x4, 3); |
| 163 ewma_x4 = _mm_mul_ps(ewma_x4, weight_prev_x4); |
| 164 ewma += EXTRACT_FLOAT(ewma_x4, 2); |
| 165 ewma_x4 = _mm_mul_ps(ewma_x4, weight_prev_x4); |
| 166 ewma += EXTRACT_FLOAT(ewma_x4, 1); |
| 167 ewma_x4 = _mm_mul_ss(ewma_x4, weight_prev_x4); |
| 168 ewma += EXTRACT_FLOAT(ewma_x4, 0); |
| 169 |
| 170 // Fold the maximums together to get the overall maximum. |
| 171 max_x4 = _mm_max_ps(max_x4, |
| 172 _mm_shuffle_ps(max_x4, max_x4, _MM_SHUFFLE(3, 3, 1, 1))); |
| 173 max_x4 = _mm_max_ss(max_x4, _mm_shuffle_ps(max_x4, max_x4, 2)); |
| 174 |
| 175 std::pair<float, float> result(ewma, EXTRACT_FLOAT(max_x4, 0)); |
| 176 |
| 177 // Handle remaining values at the end of |src|. |
| 178 for (; i < len; ++i) { |
| 179 result.first *= weight_prev; |
| 180 const float sample = src[i]; |
| 181 const float sample_squared = sample * sample; |
| 182 result.first += sample_squared * smoothing_factor; |
| 183 result.second = std::max(result.second, sample_squared); |
| 184 } |
| 185 |
| 186 return result; |
| 187 } |
| 188 #endif |
| 189 |
119 #if defined(ARCH_CPU_ARM_FAMILY) && defined(USE_NEON) | 190 #if defined(ARCH_CPU_ARM_FAMILY) && defined(USE_NEON) |
120 void FMAC_NEON(const float src[], float scale, int len, float dest[]) { | 191 void FMAC_NEON(const float src[], float scale, int len, float dest[]) { |
121 const int rem = len % 4; | 192 const int rem = len % 4; |
122 const int last_index = len - rem; | 193 const int last_index = len - rem; |
123 float32x4_t m_scale = vmovq_n_f32(scale); | 194 float32x4_t m_scale = vmovq_n_f32(scale); |
124 for (int i = 0; i < last_index; i += 4) { | 195 for (int i = 0; i < last_index; i += 4) { |
125 vst1q_f32(dest + i, vmlaq_f32( | 196 vst1q_f32(dest + i, vmlaq_f32( |
126 vld1q_f32(dest + i), vld1q_f32(src + i), m_scale)); | 197 vld1q_f32(dest + i), vld1q_f32(src + i), m_scale)); |
127 } | 198 } |
128 | 199 |
(...skipping 75 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
204 result.first += sample_squared * smoothing_factor; | 275 result.first += sample_squared * smoothing_factor; |
205 result.second = std::max(result.second, sample_squared); | 276 result.second = std::max(result.second, sample_squared); |
206 } | 277 } |
207 | 278 |
208 return result; | 279 return result; |
209 } | 280 } |
210 #endif | 281 #endif |
211 | 282 |
212 } // namespace vector_math | 283 } // namespace vector_math |
213 } // namespace media | 284 } // namespace media |
OLD | NEW |