Index: skia/ext/convolver.cc |
diff --git a/skia/ext/convolver.cc b/skia/ext/convolver.cc |
index a42a9daf32a17a3a77602f89d841de997f2787de..dd1e5068a383091fd1fcee307e97b7a5fae9fdb7 100644 |
--- a/skia/ext/convolver.cc |
+++ b/skia/ext/convolver.cc |
@@ -7,6 +7,10 @@ |
#include "skia/ext/convolver.h" |
#include "third_party/skia/include/core/SkTypes.h" |
+#if defined(ARCH_CPU_X86_FAMILY) |
+#include <emmintrin.h> // ARCH_CPU_X86_FAMILY was defined in build/config.h |
+#endif |
+ |
namespace skia { |
namespace { |
@@ -199,7 +203,7 @@ void ConvolveVertically(const ConvolutionFilter1D::Fixed* filter_values, |
if (has_alpha) { |
unsigned char alpha = ClampTo8(accum[3]); |
- // Make sure the alpha channel doesn't come out larger than any of the |
+ // Make sure the alpha channel doesn't come out smaller than any of the |
// color channels. We use premultipled alpha channels, so this should |
// never happen, but rounding errors will cause this from time to time. |
// These "impossible" colors will cause overflows (and hence random pixel |
@@ -219,6 +223,367 @@ void ConvolveVertically(const ConvolutionFilter1D::Fixed* filter_values, |
} |
} |
+ |
+// Convolves horizontally along a single row. The row data is given in |
+// |src_data| and continues for the num_values() of the filter. |
+void ConvolveHorizontally_SSE2(const unsigned char* src_data, |
+ const ConvolutionFilter1D& filter, |
+ unsigned char* out_row) { |
+#ifdef ARCH_CPU_X86_FAMILY |
+ int num_values = filter.num_values(); |
+ |
+ int filter_offset, filter_length; |
+ __m128i zero = _mm_setzero_si128(); |
+ __m128i mask[4]; |
brettw
2011/02/21 04:45:45
What's mask[0] for? Can you provide a comment for
jiesun
2011/02/22 21:37:03
yes, mask[0] is not used.
|
+ mask[1] = _mm_set_epi16(0, 0, 0, 0, 0, 0, 0, -1); |
+ mask[2] = _mm_set_epi16(0, 0, 0, 0, 0, 0, -1, -1); |
+ mask[3] = _mm_set_epi16(0, 0, 0, 0, 0, -1, -1, -1); |
+ |
+ for (int out_x = 0; out_x < num_values; out_x += 1) { |
+ const ConvolutionFilter1D::Fixed* filter_values = |
+ filter.FilterForValue(out_x, &filter_offset, &filter_length); |
+ |
+ __m128i accum = _mm_setzero_si128(); |
+ |
+ const __m128i* row_to_filter = |
brettw
2011/02/21 04:45:45
Can you comment what this means?
jiesun
2011/02/22 21:37:03
Done.
|
+ reinterpret_cast<const __m128i*>(&src_data[filter_offset << 2]); |
+ // Four filter taps per iteration. |
+ for (int j = 0; j < filter_length >> 2; ++j) { |
+ __m128i coeff, coeff16; |
brettw
2011/02/21 04:45:45
For each of the "blocks" of SSE code you're writte
jiesun
2011/02/22 21:37:03
Done.
|
+ // [16] xx xx xx xx c3 c2 c1 c0 |
+ coeff = _mm_loadl_epi64(reinterpret_cast<const __m128i*>(filter_values)); |
+ // [16] xx xx xx xx c1 c1 c0 c0 |
+ coeff16 = _mm_shufflelo_epi16(coeff, _MM_SHUFFLE(1, 1, 0, 0)); |
+ // [16] c1 c1 c1 c1 c0 c0 c0 c0 |
+ coeff16 = _mm_unpacklo_epi16(coeff16, coeff16); |
+ |
+ // [8] a3 b3 g3 r3 a2 b2 g2 r2 a1 b1 g1 r1 a0 b0 g0 r0 |
+ __m128i src8 = _mm_loadu_si128(row_to_filter); |
+ // [16] a1 b1 g1 r1 a0 b0 g0 r0 |
+ __m128i src16 = _mm_unpacklo_epi8(src8, zero); |
+ __m128i mul_hi = _mm_mulhi_epi16(src16, coeff16); |
+ __m128i mul_lo = _mm_mullo_epi16(src16, coeff16); |
+ // [32] a0*c0 b0*c0 g0*c0 r0*c0 |
+ __m128i t = _mm_unpacklo_epi16(mul_lo, mul_hi); |
+ accum = _mm_add_epi32(accum, t); |
+ // [32] a1*c1 b1*c1 g1*c1 r1*c1 |
+ t = _mm_unpackhi_epi16(mul_lo, mul_hi); |
+ accum = _mm_add_epi32(accum, t); |
+ |
+ // [16] xx xx xx xx c3 c3 c2 c2 |
+ coeff16 = _mm_shufflelo_epi16(coeff, _MM_SHUFFLE(3, 3, 2, 2)); |
+ // [16] c3 c3 c3 c3 c2 c2 c2 c2 |
+ coeff16 = _mm_unpacklo_epi16(coeff16, coeff16); |
+ // [16] a3 g3 b3 r3 a2 g2 b2 r2 |
+ src16 = _mm_unpackhi_epi8(src8, zero); |
+ mul_hi = _mm_mulhi_epi16(src16, coeff16); |
+ mul_lo = _mm_mullo_epi16(src16, coeff16); |
+ // [32] a2*c2 b2*c2 g2*c2 r2*c2 |
+ t = _mm_unpacklo_epi16(mul_lo, mul_hi); |
+ accum = _mm_add_epi32(accum, t); |
+ // [32] a3*c3 b3*c3 g3*c3 r3*c3 |
+ t = _mm_unpackhi_epi16(mul_lo, mul_hi); |
+ accum = _mm_add_epi32(accum, t); |
+ |
+ row_to_filter += 1; |
+ filter_values += 4; |
+ } |
+ |
+ // remaining |
brettw
2011/02/21 04:45:45
Can you provide a better comment here?
jiesun
2011/02/22 21:37:03
Done.
|
+ int r = filter_length&3; |
+ if (r) { |
+ // Note: filter_values must be padded to align_up(filter_offset, 8). |
+ __m128i coeff, coeff16; |
+ coeff = _mm_loadl_epi64(reinterpret_cast<const __m128i*>(filter_values)); |
+ // Mask out extra filter taps. |
+ coeff = _mm_and_si128(coeff, mask[r]); |
+ coeff16 = _mm_shufflelo_epi16(coeff, _MM_SHUFFLE(1, 1, 0, 0)); |
+ coeff16 = _mm_unpacklo_epi16(coeff16, coeff16); |
+ |
+ // Note: line buffer must be padded to align_up(filter_offset, 16). |
+ // We resolve this by use C-version for the last horizontal line. |
+ __m128i src8 = _mm_loadu_si128(row_to_filter); |
+ __m128i src16 = _mm_unpacklo_epi8(src8, zero); |
+ __m128i mul_hi = _mm_mulhi_epi16(src16, coeff16); |
+ __m128i mul_lo = _mm_mullo_epi16(src16, coeff16); |
+ __m128i t = _mm_unpacklo_epi16(mul_lo, mul_hi); |
+ accum = _mm_add_epi32(accum, t); |
+ t = _mm_unpackhi_epi16(mul_lo, mul_hi); |
+ accum = _mm_add_epi32(accum, t); |
+ |
+ src16 = _mm_unpackhi_epi8(src8, zero); |
+ coeff16 = _mm_shufflelo_epi16(coeff, _MM_SHUFFLE(3, 3, 2, 2)); |
+ coeff16 = _mm_unpacklo_epi16(coeff16, coeff16); |
+ mul_hi = _mm_mulhi_epi16(src16, coeff16); |
+ mul_lo = _mm_mullo_epi16(src16, coeff16); |
+ t = _mm_unpacklo_epi16(mul_lo, mul_hi); |
+ accum = _mm_add_epi32(accum, t); |
+ } |
+ |
+ // shift right for fix point implementation before saturation. |
+ accum = _mm_srai_epi32(accum, ConvolutionFilter1D::kShiftBits); |
+ accum = _mm_packs_epi32(accum, zero); |
+ accum = _mm_packus_epi16(accum, zero); |
+ |
+ *(reinterpret_cast<int*>(out_row)) = _mm_cvtsi128_si32(accum); |
+ out_row += 4; |
+ } |
+#endif |
+} |
+ |
+// Convolves horizontally along four rows. The row data is given in |
+// |src_data| and continues for the num_values() of the filter. |
+void ConvolveHorizontally4_SSE2(const unsigned char* src_data[4], |
+ const ConvolutionFilter1D& filter, |
+ unsigned char* out_row[4]) { |
+#ifdef ARCH_CPU_X86_FAMILY |
+ int width = filter.num_values(); |
+ |
+ int filter_offset, filter_length; |
+ __m128i zero = _mm_setzero_si128(); |
+ __m128i mask[4]; |
+ mask[1] = _mm_set_epi16(0, 0, 0, 0, 0, 0, 0, -1); |
+ mask[2] = _mm_set_epi16(0, 0, 0, 0, 0, 0, -1, -1); |
+ mask[3] = _mm_set_epi16(0, 0, 0, 0, 0, -1, -1, -1); |
+ |
+ for (int i = 0; i < width; ++i) { |
+ const ConvolutionFilter1D::Fixed* filter_values = |
+ filter.FilterForValue(i, &filter_offset, &filter_length); |
+ |
+ // four pixels in a column per iteration. |
+ __m128i accum0 = _mm_setzero_si128(); |
+ __m128i accum1 = _mm_setzero_si128(); |
+ __m128i accum2 = _mm_setzero_si128(); |
+ __m128i accum3 = _mm_setzero_si128(); |
+ int start = (filter_offset<<2); |
+ for (int j = 0; j < (filter_length >> 2); ++j) { |
+ __m128i coeff, coeff16lo, coeff16hi; |
+ // [16] xx xx xx xx c3 c2 c1 c0 |
+ coeff = _mm_loadl_epi64(reinterpret_cast<const __m128i*>(filter_values)); |
+ // [16] xx xx xx xx c1 c1 c0 c0 |
+ coeff16lo = _mm_shufflelo_epi16(coeff, _MM_SHUFFLE(1, 1, 0, 0)); |
+ // [16] c1 c1 c1 c1 c0 c0 c0 c0 |
+ coeff16lo = _mm_unpacklo_epi16(coeff16lo, coeff16lo); |
+ // [16] xx xx xx xx c3 c3 c2 c2 |
+ coeff16hi = _mm_shufflelo_epi16(coeff, _MM_SHUFFLE(3, 3, 2, 2)); |
+ // [16] c3 c3 c3 c3 c2 c2 c2 c2 |
+ coeff16hi = _mm_unpacklo_epi16(coeff16hi, coeff16hi); |
+ |
+ __m128i src8, src16, mul_hi, mul_lo, t; |
+ |
+#define ITERATION(src, accum) \ |
+ src8 = _mm_loadu_si128(reinterpret_cast<const __m128i*>(src)); \ |
+ src16 = _mm_unpacklo_epi8(src8, zero); \ |
+ mul_hi = _mm_mulhi_epi16(src16, coeff16lo); \ |
+ mul_lo = _mm_mullo_epi16(src16, coeff16lo); \ |
+ t = _mm_unpacklo_epi16(mul_lo, mul_hi); \ |
+ accum = _mm_add_epi32(accum, t); \ |
+ t = _mm_unpackhi_epi16(mul_lo, mul_hi); \ |
+ accum = _mm_add_epi32(accum, t); \ |
+ src16 = _mm_unpackhi_epi8(src8, zero); \ |
+ mul_hi = _mm_mulhi_epi16(src16, coeff16hi); \ |
+ mul_lo = _mm_mullo_epi16(src16, coeff16hi); \ |
+ t = _mm_unpacklo_epi16(mul_lo, mul_hi); \ |
+ accum = _mm_add_epi32(accum, t); \ |
+ t = _mm_unpackhi_epi16(mul_lo, mul_hi); \ |
+ accum = _mm_add_epi32(accum, t) |
+ |
+ ITERATION(src_data[0]+start, accum0); |
+ ITERATION(src_data[1]+start, accum1); |
+ ITERATION(src_data[2]+start, accum2); |
+ ITERATION(src_data[3]+start, accum3); |
+ |
+ start += 16; |
+ filter_values += 4; |
+ } |
+ |
+ int r = filter_length&3; |
+ if (r) { |
+ // Note: filter_values must be padded to align_up(filter_offset, 8); |
+ __m128i coeff; |
+ coeff = _mm_loadl_epi64(reinterpret_cast<const __m128i*>(filter_values)); |
+ // Mask out extra filter taps. |
+ coeff = _mm_and_si128(coeff, mask[r]); |
+ |
+ __m128i coeff16lo = _mm_shufflelo_epi16(coeff, _MM_SHUFFLE(1, 1, 0, 0)); |
+ /* c1 c1 c1 c1 c0 c0 c0 c0 */ |
+ coeff16lo = _mm_unpacklo_epi16(coeff16lo, coeff16lo); |
+ __m128i coeff16hi = _mm_shufflelo_epi16(coeff, _MM_SHUFFLE(3, 3, 2, 2)); |
+ coeff16hi = _mm_unpacklo_epi16(coeff16hi, coeff16hi); |
+ |
+ __m128i src8, src16, mul_hi, mul_lo, t; |
+ |
+ ITERATION(src_data[0]+start, accum0); |
+ ITERATION(src_data[1]+start, accum1); |
+ ITERATION(src_data[2]+start, accum2); |
+ ITERATION(src_data[3]+start, accum3); |
+ } |
+ |
+ accum0 = _mm_srai_epi32(accum0, ConvolutionFilter1D::kShiftBits); |
+ accum0 = _mm_packs_epi32(accum0, zero); |
+ accum0 = _mm_packus_epi16(accum0, zero); |
+ accum1 = _mm_srai_epi32(accum1, ConvolutionFilter1D::kShiftBits); |
+ accum1 = _mm_packs_epi32(accum1, zero); |
+ accum1 = _mm_packus_epi16(accum1, zero); |
+ accum2 = _mm_srai_epi32(accum2, ConvolutionFilter1D::kShiftBits); |
+ accum2 = _mm_packs_epi32(accum2, zero); |
+ accum2 = _mm_packus_epi16(accum2, zero); |
+ accum3 = _mm_srai_epi32(accum3, ConvolutionFilter1D::kShiftBits); |
+ accum3 = _mm_packs_epi32(accum3, zero); |
+ accum3 = _mm_packus_epi16(accum3, zero); |
+ |
+ *(reinterpret_cast<int*>(out_row[0])) = _mm_cvtsi128_si32(accum0); |
+ *(reinterpret_cast<int*>(out_row[1])) = _mm_cvtsi128_si32(accum1); |
+ *(reinterpret_cast<int*>(out_row[2])) = _mm_cvtsi128_si32(accum2); |
+ *(reinterpret_cast<int*>(out_row[3])) = _mm_cvtsi128_si32(accum3); |
+ |
+ out_row[0] += 4; |
+ out_row[1] += 4; |
+ out_row[2] += 4; |
+ out_row[3] += 4; |
+ } |
+#endif |
+} |
+ |
+template<bool has_alpha> |
+void ConvolveVertically_SSE2(const ConvolutionFilter1D::Fixed* filter_values, |
+ int filter_length, |
brettw
2011/02/21 04:45:45
Check indentation
jiesun
2011/02/22 21:37:03
Done.
|
+ unsigned char* const* source_data_rows, |
+ int pixel_width, |
+ unsigned char* out_row) { |
+#ifdef ARCH_CPU_X86_FAMILY |
+ int width = pixel_width & ~3; |
+ |
+ __m128i zero = _mm_setzero_si128(); |
+ __m128i accum0, accum1, accum2, accum3, coeff16; |
+ const __m128i* src; |
+ for (int i = 0; i < width; i += 4) { // Four pixels per iteration. |
+ accum0 = _mm_setzero_si128(); |
+ accum1 = _mm_setzero_si128(); |
+ accum2 = _mm_setzero_si128(); |
+ accum3 = _mm_setzero_si128(); |
+ for (int j = 0; j < filter_length; ++j) { |
+ coeff16 = _mm_set1_epi16(filter_values[j]); |
+ |
+ // aligned load due to row_buffer is 16 byte aligned. |
+ // [8] a3 b3 g3 r3 a2 b2 g2 r2 a1 b1 g1 r1 a0 b0 g0 r0 |
+ src = reinterpret_cast<const __m128i*>(&source_data_rows[j][i<<2]); |
+ __m128i src8 = _mm_loadu_si128(src); |
+ // [16] a1 b1 g1 r1 a0 b0 g0 r0 |
+ __m128i src16 = _mm_unpacklo_epi8(src8, zero); |
+ __m128i mul_hi = _mm_mulhi_epi16(src16, coeff16); |
+ __m128i mul_lo = _mm_mullo_epi16(src16, coeff16); |
+ // [32] a0 b0 g0 r0 |
+ __m128i t = _mm_unpacklo_epi16(mul_lo, mul_hi); |
+ accum0 = _mm_add_epi32(accum0, t); |
+ // [32] a1 b1 g1 r1 |
+ t = _mm_unpackhi_epi16(mul_lo, mul_hi); |
+ accum1 = _mm_add_epi32(accum1, t); |
+ // [16] a3 b3 g3 r3 a2 b2 g2 r2 |
+ src16 = _mm_unpackhi_epi8(src8, zero); |
+ mul_hi = _mm_mulhi_epi16(src16, coeff16); |
+ mul_lo = _mm_mullo_epi16(src16, coeff16); |
+ // [32] a2 b2 g2 r2 |
+ t = _mm_unpacklo_epi16(mul_lo, mul_hi); |
+ accum2 = _mm_add_epi32(accum2, t); |
+ // [32] a3 b3 g3 r3 |
+ t = _mm_unpackhi_epi16(mul_lo, mul_hi); |
+ accum3 = _mm_add_epi32(accum3, t); |
+ } |
+ |
+ accum0 = _mm_srai_epi32(accum0, ConvolutionFilter1D::kShiftBits); |
+ accum1 = _mm_srai_epi32(accum1, ConvolutionFilter1D::kShiftBits); |
+ accum2 = _mm_srai_epi32(accum2, ConvolutionFilter1D::kShiftBits); |
+ accum3 = _mm_srai_epi32(accum3, ConvolutionFilter1D::kShiftBits); |
+ // [16] a1 b1 g1 r1 a0 b0 g0 r0 |
+ accum0 = _mm_packs_epi32(accum0, accum1); |
+ // [16] a3 b3 g3 r3 a2 b2 g2 r2 |
+ accum2 = _mm_packs_epi32(accum2, accum3); |
+ // [8] a3 b3 g3 r3 a2 b2 g2 r2 a1 b1 g1 r1 a0 b0 g0 r0 |
+ accum0 = _mm_packus_epi16(accum0, accum2); |
+ if (has_alpha) { |
+ // [8] xx a3 b3 g3 xx a2 b2 g2 xx a1 b1 g1 xx a0 b0 g0 |
+ __m128i a = _mm_srli_epi32(accum0, 8); |
+ // [8] xx xx xx max3 xx xx xx max2 xx xx xx max1 xx xx xx max0 |
+ __m128i b = _mm_max_epu8(a, accum0); // Max of r and g. |
+ // [8] xx xx a3 b3 xx xx a2 b2 xx xx a1 b1 xx xx a0 b0 |
+ a = _mm_srli_epi32(accum0, 16); |
+ // [8] xx xx xx max3 xx xx xx max2 xx xx xx max1 xx xx xx max0 |
+ b = _mm_max_epu8(a, b); // Max of r and g and b. |
+ // [8] max3 00 00 00 max2 00 00 00 max1 00 00 00 max0 00 00 00 |
+ b = _mm_slli_epi32(b, 24); |
+ accum0 = _mm_max_epu8(b, accum0); |
+ } else { |
+ __m128i mask = _mm_set1_epi32(0xff000000); |
+ accum0 = _mm_or_si128(accum0, mask); |
+ } |
+ _mm_storeu_si128(reinterpret_cast<__m128i*>(out_row), accum0); |
+ out_row += 16; |
+ } |
+ |
+ if (pixel_width & 3) { |
+ accum0 = _mm_setzero_si128(); |
+ accum1 = _mm_setzero_si128(); |
+ accum2 = _mm_setzero_si128(); |
+ for (int j = 0; j < filter_length; ++j) { |
+ coeff16 = _mm_set1_epi16(filter_values[j]); |
+ // [8] a3 b3 g3 r3 a2 b2 g2 r2 a1 b1 g1 r1 a0 b0 g0 r0 |
+ src = reinterpret_cast<const __m128i*>(&source_data_rows[j][width<<2]); |
+ __m128i src8 = _mm_loadu_si128(src); |
+ // [16] a1 b1 g1 r1 a0 b0 g0 r0 |
+ __m128i src16 = _mm_unpacklo_epi8(src8, zero); |
+ __m128i mul_hi = _mm_mulhi_epi16(src16, coeff16); |
+ __m128i mul_lo = _mm_mullo_epi16(src16, coeff16); |
+ // [32] a0 b0 g0 r0 |
+ __m128i t = _mm_unpacklo_epi16(mul_lo, mul_hi); |
+ accum0 = _mm_add_epi32(accum0, t); |
+ // [32] a1 b1 g1 r1 |
+ t = _mm_unpackhi_epi16(mul_lo, mul_hi); |
+ accum1 = _mm_add_epi32(accum1, t); |
+ // [16] a3 b3 g3 r3 a2 b2 g2 r2 |
+ src16 = _mm_unpackhi_epi8(src8, zero); |
+ mul_hi = _mm_mulhi_epi16(src16, coeff16); |
+ mul_lo = _mm_mullo_epi16(src16, coeff16); |
+ // [32] a2 b2 g2 r2 |
+ t = _mm_unpacklo_epi16(mul_lo, mul_hi); |
+ accum2 = _mm_add_epi32(accum2, t); |
+ } |
+ |
+ accum0 = _mm_srai_epi32(accum0, ConvolutionFilter1D::kShiftBits); |
+ accum1 = _mm_srai_epi32(accum1, ConvolutionFilter1D::kShiftBits); |
+ accum2 = _mm_srai_epi32(accum2, ConvolutionFilter1D::kShiftBits); |
+ // [16] a1 b1 g1 r1 a0 b0 g0 r0 |
+ accum0 = _mm_packs_epi32(accum0, accum1); |
+ // [16] a3 b3 g3 r3 a2 b2 g2 r2 |
+ accum2 = _mm_packs_epi32(accum2, zero); |
+ // [8] a3 b3 g3 r3 a2 b2 g2 r2 a1 b1 g1 r1 a0 b0 g0 r0 |
+ accum0 = _mm_packus_epi16(accum0, accum2); |
+ if (has_alpha) { |
+ // [8] xx a3 b3 g3 xx a2 b2 g2 xx a1 b1 g1 xx a0 b0 g0 |
+ __m128i a = _mm_srli_epi32(accum0, 8); |
+ // [8] xx xx xx max3 xx xx xx max2 xx xx xx max1 xx xx xx max0 |
+ __m128i b = _mm_max_epu8(a, accum0); // Max of r and g. |
+ // [8] xx xx a3 b3 xx xx a2 b2 xx xx a1 b1 xx xx a0 b0 |
+ a = _mm_srli_epi32(accum0, 16); |
+ // [8] xx xx xx max3 xx xx xx max2 xx xx xx max1 xx xx xx max0 |
+ b = _mm_max_epu8(a, b); // Max of r and g and b. |
+ // [8] max3 00 00 00 max2 00 00 00 max1 00 00 00 max0 00 00 00 |
+ b = _mm_slli_epi32(b, 24); |
+ accum0 = _mm_max_epu8(b, accum0); |
+ } else { |
+ __m128i mask = _mm_set1_epi32(0xff000000); |
+ accum0 = _mm_or_si128(accum0, mask); |
+ } |
+ |
+ for (int i = width; i < pixel_width; ++i) { |
+ *(reinterpret_cast<int*>(out_row)) = _mm_cvtsi128_si32(accum0); |
+ accum0 = _mm_srli_si128(accum0, 4); |
+ out_row += 4; |
+ } |
+ } |
+#endif |
+} |
+ |
} // namespace |
// ConvolutionFilter1D --------------------------------------------------------- |
@@ -284,15 +649,13 @@ void ConvolutionFilter1D::AddFilter(int filter_offset, |
max_filter_ = std::max(max_filter_, filter_length); |
} |
-// BGRAConvolve2D ------------------------------------------------------------- |
- |
-void BGRAConvolve2D(const unsigned char* source_data, |
- int source_byte_row_stride, |
- bool source_has_alpha, |
- const ConvolutionFilter1D& filter_x, |
- const ConvolutionFilter1D& filter_y, |
- int output_byte_row_stride, |
- unsigned char* output) { |
+void BGRAConvolve2D_C(const unsigned char* source_data, |
+ int source_byte_row_stride, |
+ bool source_has_alpha, |
+ const ConvolutionFilter1D& filter_x, |
+ const ConvolutionFilter1D& filter_y, |
+ int output_byte_row_stride, |
+ unsigned char* output) { |
int max_y_filter_size = filter_y.max_filter(); |
// The next row in the input that we will generate a horizontally |
@@ -317,6 +680,11 @@ void BGRAConvolve2D(const unsigned char* source_data, |
// convolutions to run each subsequent vertical convolution. |
SkASSERT(output_byte_row_stride >= filter_x.num_values() * 4); |
int num_output_rows = filter_y.num_values(); |
+ |
+ int last_filter_offset, last_filter_length; |
+ filter_y.FilterForValue(num_output_rows-1, &last_filter_offset, |
+ &last_filter_length); |
+ |
for (int out_y = 0; out_y < num_output_rows; out_y++) { |
filter_values = filter_y.FilterForValue(out_y, |
&filter_offset, &filter_length); |
@@ -354,10 +722,132 @@ void BGRAConvolve2D(const unsigned char* source_data, |
filter_x.num_values(), cur_output_row); |
} else { |
ConvolveVertically<false>(filter_values, filter_length, |
- first_row_for_filter, |
- filter_x.num_values(), cur_output_row); |
+ first_row_for_filter, |
+ filter_x.num_values(), cur_output_row); |
} |
} |
} |
+// BGRAConvolve2D ------------------------------------------------------------- |
+ |
+void BGRAConvolve2D_SSE2(const unsigned char* source_data, |
+ int source_byte_row_stride, |
+ bool source_has_alpha, |
+ const ConvolutionFilter1D& filter_x, |
+ const ConvolutionFilter1D& filter_y, |
+ int output_byte_row_stride, |
+ unsigned char* output) { |
+ int max_y_filter_size = filter_y.max_filter(); |
+ |
+ // The next row in the input that we will generate a horizontally |
+ // convolved row for. If the filter doesn't start at the beginning of the |
+ // image (this is the case when we are only resizing a subset), then we |
+ // don't want to generate any output rows before that. Compute the starting |
+ // row for convolution as the first pixel for the first vertical filter. |
+ int filter_offset, filter_length; |
+ const ConvolutionFilter1D::Fixed* filter_values = |
+ filter_y.FilterForValue(0, &filter_offset, &filter_length); |
+ int next_x_row = filter_offset; |
+ |
+ // We loop over each row in the input doing a horizontal convolution. This |
+ // will result in a horizontally convolved image. We write the results into |
+ // a circular buffer of convolved rows and do vertical convolution as rows |
+ // are available. This prevents us from having to store the entire |
+ // intermediate image and helps cache coherency. |
+ // We will need four extra rows to allow horizontal convolution could be done |
+ // simultaneously. |
+ int row_buffer_width = (filter_x.num_values() + 15) & ~0xF; |
+ int row_buffer_height = max_y_filter_size + 4; |
+ CircularRowBuffer row_buffer(row_buffer_width, |
+ row_buffer_height, |
+ filter_offset); |
+ |
+ // Loop over every possible output row, processing just enough horizontal |
+ // convolutions to run each subsequent vertical convolution. |
+ SkASSERT(output_byte_row_stride >= filter_x.num_values() * 4); |
+ int num_output_rows = filter_y.num_values(); |
+ |
+ int last_filter_offset, last_filter_length; |
+ filter_y.FilterForValue(num_output_rows-1, &last_filter_offset, |
+ &last_filter_length); |
+ |
+ for (int out_y = 0; out_y < num_output_rows; out_y++) { |
+ filter_values = filter_y.FilterForValue(out_y, |
+ &filter_offset, &filter_length); |
+ |
+ // Generate output rows until we have enough to run the current filter. |
+ while (next_x_row < filter_offset + filter_length) { |
+ if (next_x_row + 3 < last_filter_offset + last_filter_length - 1) { |
+ const unsigned char* src[4]; |
+ unsigned char* out_row[4]; |
+ for (int i = 0; i < 4; ++i) { |
+ src[i] = &source_data[(next_x_row+i) * source_byte_row_stride]; |
+ out_row[i] = row_buffer.AdvanceRow(); |
+ } |
+ ConvolveHorizontally4_SSE2(src, filter_x, out_row); |
+ next_x_row+=4; |
+ } else { |
+ // For the last row, SSE2 load possibly to access data beyond the |
+ // image area. therefore we use C version here. Hacking into skia |
+ // to add line paddings is not something in my mind. |
+ if (next_x_row == last_filter_offset + last_filter_length - 1) { |
+ if (source_has_alpha) |
+ ConvolveHorizontally<true>( |
+ &source_data[next_x_row * source_byte_row_stride], |
+ filter_x, row_buffer.AdvanceRow()); |
+ else |
+ ConvolveHorizontally<false>( |
+ &source_data[next_x_row * source_byte_row_stride], |
+ filter_x, row_buffer.AdvanceRow()); |
+ } else { |
+ ConvolveHorizontally_SSE2( |
+ &source_data[next_x_row * source_byte_row_stride], |
+ filter_x, row_buffer.AdvanceRow()); |
+ } |
+ next_x_row++; |
+ } |
+ } |
+ |
+ // Compute where in the output image this row of final data will go. |
+ unsigned char* cur_output_row = &output[out_y * output_byte_row_stride]; |
+ |
+ // Get the list of rows that the circular buffer has, in order. |
+ int first_row_in_circular_buffer; |
+ unsigned char* const* rows_to_convolve = |
+ row_buffer.GetRowAddresses(&first_row_in_circular_buffer); |
+ |
+ // Now compute the start of the subset of those rows that the filter |
+ // needs. |
+ unsigned char* const* first_row_for_filter = |
+ &rows_to_convolve[filter_offset - first_row_in_circular_buffer]; |
+ |
+ if (source_has_alpha) { |
+ ConvolveVertically_SSE2<true>(filter_values, filter_length, |
+ first_row_for_filter, |
+ filter_x.num_values(), cur_output_row); |
+ } else { |
+ ConvolveVertically_SSE2<false>(filter_values, filter_length, |
+ first_row_for_filter, |
+ filter_x.num_values(), cur_output_row); |
+ } |
+ } |
+} |
+ |
+void BGRAConvolve2D(const unsigned char* source_data, |
+ int source_byte_row_stride, |
+ bool source_has_alpha, |
+ const ConvolutionFilter1D& filter_x, |
+ const ConvolutionFilter1D& filter_y, |
+ int output_byte_row_stride, |
+ unsigned char* output) { |
+ base::CPU cpu; |
+ if (cpu.has_sse2()) { |
+ BGRAConvolve2D_SSE2(source_data, source_byte_row_stride, source_has_alpha, |
+ filter_x, filter_y, output_byte_row_stride, output); |
+ } else { |
+ BGRAConvolve2D_C(source_data, source_byte_row_stride, source_has_alpha, |
+ filter_x, filter_y, output_byte_row_stride, output); |
+ } |
+} |
+ |
} // namespace skia |