SIMD implementation of Convolver for Lanczos filter etc.

skia/ext/convolver.cc

Index: skia/ext/convolver.cc
diff --git a/skia/ext/convolver.cc b/skia/ext/convolver.cc
index a42a9daf32a17a3a77602f89d841de997f2787de..dd1e5068a383091fd1fcee307e97b7a5fae9fdb7 100644
--- a/skia/ext/convolver.cc
+++ b/skia/ext/convolver.cc
@@ -7,6 +7,10 @@
 #include "skia/ext/convolver.h"
 #include "third_party/skia/include/core/SkTypes.h"
 
+#if defined(ARCH_CPU_X86_FAMILY)
+#include <emmintrin.h>  // ARCH_CPU_X86_FAMILY was defined in build/config.h
+#endif
+
 namespace skia {
 
 namespace {
@@ -199,7 +203,7 @@ void ConvolveVertically(const ConvolutionFilter1D::Fixed* filter_values,
     if (has_alpha) {
       unsigned char alpha = ClampTo8(accum[3]);
 
-      // Make sure the alpha channel doesn't come out larger than any of the
+      // Make sure the alpha channel doesn't come out smaller than any of the
       // color channels. We use premultipled alpha channels, so this should
       // never happen, but rounding errors will cause this from time to time.
       // These "impossible" colors will cause overflows (and hence random pixel
@@ -219,6 +223,367 @@ void ConvolveVertically(const ConvolutionFilter1D::Fixed* filter_values,
   }
 }
 
+
+// Convolves horizontally along a single row. The row data is given in
+// |src_data| and continues for the num_values() of the filter.
+void ConvolveHorizontally_SSE2(const unsigned char* src_data,
+                               const ConvolutionFilter1D& filter,
+                               unsigned char* out_row) {
+#ifdef ARCH_CPU_X86_FAMILY
+  int num_values = filter.num_values();
+
+  int filter_offset, filter_length;
+  __m128i zero = _mm_setzero_si128();
+  __m128i mask[4];

[brettw, 2011/02/21 04:45:45]

What's mask[0] for? Can you provide a comment for what this is for and note you
you don't need mask[0]?

[jiesun, 2011/02/22 21:37:03]

yes, mask[0] is not used.

+  mask[1] = _mm_set_epi16(0, 0, 0, 0, 0, 0, 0, -1);
+  mask[2] = _mm_set_epi16(0, 0, 0, 0, 0, 0, -1, -1);
+  mask[3] = _mm_set_epi16(0, 0, 0, 0, 0, -1, -1, -1);
+
+  for (int out_x = 0; out_x < num_values; out_x += 1) {
+    const ConvolutionFilter1D::Fixed* filter_values =
+        filter.FilterForValue(out_x, &filter_offset, &filter_length);
+
+    __m128i accum = _mm_setzero_si128();
+
+    const __m128i* row_to_filter =

[brettw, 2011/02/21 04:45:45]

Can you comment what this means?

[jiesun, 2011/02/22 21:37:03]

Done.

+        reinterpret_cast<const __m128i*>(&src_data[filter_offset << 2]);
+    // Four filter taps per iteration.
+    for (int j = 0; j < filter_length >> 2; ++j) {
+      __m128i coeff, coeff16;

[brettw, 2011/02/21 04:45:45]

For each of the "blocks" of SSE code you're written, can you provide comments
explaining in English what you're doing and what the contents of the different
variables will be at the end?

[jiesun, 2011/02/22 21:37:03]

Done.

+      // [16] xx xx xx xx c3 c2 c1 c0
+      coeff = _mm_loadl_epi64(reinterpret_cast<const __m128i*>(filter_values));
+      // [16] xx xx xx xx c1 c1 c0 c0
+      coeff16 = _mm_shufflelo_epi16(coeff, _MM_SHUFFLE(1, 1, 0, 0));
+      // [16] c1 c1 c1 c1 c0 c0 c0 c0
+      coeff16 = _mm_unpacklo_epi16(coeff16, coeff16);
+
+      // [8] a3 b3 g3 r3 a2 b2 g2 r2 a1 b1 g1 r1 a0 b0 g0 r0
+      __m128i src8 = _mm_loadu_si128(row_to_filter);
+      // [16] a1 b1 g1 r1 a0 b0 g0 r0
+      __m128i src16 = _mm_unpacklo_epi8(src8, zero);
+      __m128i mul_hi = _mm_mulhi_epi16(src16, coeff16);
+      __m128i mul_lo = _mm_mullo_epi16(src16, coeff16);
+      // [32]  a0*c0 b0*c0 g0*c0 r0*c0
+      __m128i t = _mm_unpacklo_epi16(mul_lo, mul_hi);
+      accum = _mm_add_epi32(accum, t);
+      // [32]  a1*c1 b1*c1 g1*c1 r1*c1
+      t = _mm_unpackhi_epi16(mul_lo, mul_hi);
+      accum = _mm_add_epi32(accum, t);
+
+      // [16] xx xx xx xx c3 c3 c2 c2
+      coeff16 = _mm_shufflelo_epi16(coeff, _MM_SHUFFLE(3, 3, 2, 2));
+      // [16] c3 c3 c3 c3 c2 c2 c2 c2
+      coeff16 = _mm_unpacklo_epi16(coeff16, coeff16);
+      // [16] a3 g3 b3 r3 a2 g2 b2 r2
+      src16 = _mm_unpackhi_epi8(src8, zero);
+      mul_hi = _mm_mulhi_epi16(src16, coeff16);
+      mul_lo = _mm_mullo_epi16(src16, coeff16);
+      // [32]  a2*c2 b2*c2 g2*c2 r2*c2
+      t = _mm_unpacklo_epi16(mul_lo, mul_hi);
+      accum = _mm_add_epi32(accum, t);
+      // [32]  a3*c3 b3*c3 g3*c3 r3*c3
+      t = _mm_unpackhi_epi16(mul_lo, mul_hi);
+      accum = _mm_add_epi32(accum, t);
+
+      row_to_filter += 1;
+      filter_values += 4;
+    }
+
+    // remaining

[brettw, 2011/02/21 04:45:45]

Can you provide a better comment here?

[jiesun, 2011/02/22 21:37:03]

Done.

+    int r = filter_length&3;
+    if (r) {
+      // Note: filter_values must be padded to align_up(filter_offset, 8).
+      __m128i coeff, coeff16;
+      coeff = _mm_loadl_epi64(reinterpret_cast<const __m128i*>(filter_values));
+      // Mask out extra filter taps.
+      coeff = _mm_and_si128(coeff, mask[r]);
+      coeff16 = _mm_shufflelo_epi16(coeff, _MM_SHUFFLE(1, 1, 0, 0));
+      coeff16 = _mm_unpacklo_epi16(coeff16, coeff16);
+
+      // Note: line buffer must be padded to align_up(filter_offset, 16).
+      // We resolve this by use C-version for the last horizontal line.
+      __m128i src8 = _mm_loadu_si128(row_to_filter);
+      __m128i src16 = _mm_unpacklo_epi8(src8, zero);
+      __m128i mul_hi = _mm_mulhi_epi16(src16, coeff16);
+      __m128i mul_lo = _mm_mullo_epi16(src16, coeff16);
+      __m128i t = _mm_unpacklo_epi16(mul_lo, mul_hi);
+      accum = _mm_add_epi32(accum, t);
+      t = _mm_unpackhi_epi16(mul_lo, mul_hi);
+      accum = _mm_add_epi32(accum, t);
+
+      src16 = _mm_unpackhi_epi8(src8, zero);
+      coeff16 = _mm_shufflelo_epi16(coeff, _MM_SHUFFLE(3, 3, 2, 2));
+      coeff16 = _mm_unpacklo_epi16(coeff16, coeff16);
+      mul_hi = _mm_mulhi_epi16(src16, coeff16);
+      mul_lo = _mm_mullo_epi16(src16, coeff16);
+      t = _mm_unpacklo_epi16(mul_lo, mul_hi);
+      accum = _mm_add_epi32(accum, t);
+    }
+
+    // shift right for fix point implementation before saturation.
+    accum = _mm_srai_epi32(accum, ConvolutionFilter1D::kShiftBits);
+    accum = _mm_packs_epi32(accum, zero);
+    accum = _mm_packus_epi16(accum, zero);
+
+    *(reinterpret_cast<int*>(out_row)) = _mm_cvtsi128_si32(accum);
+    out_row += 4;
+  }
+#endif
+}
+
+// Convolves horizontally along four rows. The row data is given in
+// |src_data| and continues for the num_values() of the filter.
+void ConvolveHorizontally4_SSE2(const unsigned char* src_data[4],
+                                const ConvolutionFilter1D& filter,
+                                unsigned char* out_row[4]) {
+#ifdef ARCH_CPU_X86_FAMILY
+  int width = filter.num_values();
+
+  int filter_offset, filter_length;
+  __m128i zero = _mm_setzero_si128();
+  __m128i mask[4];
+  mask[1] = _mm_set_epi16(0, 0, 0, 0, 0, 0, 0, -1);
+  mask[2] = _mm_set_epi16(0, 0, 0, 0, 0, 0, -1, -1);
+  mask[3] = _mm_set_epi16(0, 0, 0, 0, 0, -1, -1, -1);
+
+  for (int i = 0; i < width; ++i) {
+    const ConvolutionFilter1D::Fixed* filter_values =
+        filter.FilterForValue(i, &filter_offset, &filter_length);
+
+    // four pixels in a column per iteration.
+    __m128i accum0 = _mm_setzero_si128();
+    __m128i accum1 = _mm_setzero_si128();
+    __m128i accum2 = _mm_setzero_si128();
+    __m128i accum3 = _mm_setzero_si128();
+    int start = (filter_offset<<2);
+    for (int j = 0; j < (filter_length >> 2); ++j) {
+      __m128i coeff, coeff16lo, coeff16hi;
+      // [16] xx xx xx xx c3 c2 c1 c0
+      coeff = _mm_loadl_epi64(reinterpret_cast<const __m128i*>(filter_values));
+      // [16] xx xx xx xx c1 c1 c0 c0
+      coeff16lo = _mm_shufflelo_epi16(coeff, _MM_SHUFFLE(1, 1, 0, 0));
+      // [16] c1 c1 c1 c1 c0 c0 c0 c0
+      coeff16lo = _mm_unpacklo_epi16(coeff16lo, coeff16lo);
+      // [16] xx xx xx xx c3 c3 c2 c2
+      coeff16hi = _mm_shufflelo_epi16(coeff, _MM_SHUFFLE(3, 3, 2, 2));
+      // [16] c3 c3 c3 c3 c2 c2 c2 c2
+      coeff16hi = _mm_unpacklo_epi16(coeff16hi, coeff16hi);
+
+      __m128i src8, src16, mul_hi, mul_lo, t;
+
+#define ITERATION(src, accum)                                          \
+      src8 = _mm_loadu_si128(reinterpret_cast<const __m128i*>(src));   \
+      src16 = _mm_unpacklo_epi8(src8, zero);                           \
+      mul_hi = _mm_mulhi_epi16(src16, coeff16lo);                      \
+      mul_lo = _mm_mullo_epi16(src16, coeff16lo);                      \
+      t = _mm_unpacklo_epi16(mul_lo, mul_hi);                          \
+      accum = _mm_add_epi32(accum, t);                                 \
+      t = _mm_unpackhi_epi16(mul_lo, mul_hi);                          \
+      accum = _mm_add_epi32(accum, t);                                 \
+      src16 = _mm_unpackhi_epi8(src8, zero);                           \
+      mul_hi = _mm_mulhi_epi16(src16, coeff16hi);                      \
+      mul_lo = _mm_mullo_epi16(src16, coeff16hi);                      \
+      t = _mm_unpacklo_epi16(mul_lo, mul_hi);                          \
+      accum = _mm_add_epi32(accum, t);                                 \
+      t = _mm_unpackhi_epi16(mul_lo, mul_hi);                          \
+      accum = _mm_add_epi32(accum, t)
+
+      ITERATION(src_data[0]+start, accum0);
+      ITERATION(src_data[1]+start, accum1);
+      ITERATION(src_data[2]+start, accum2);
+      ITERATION(src_data[3]+start, accum3);
+
+      start += 16;
+      filter_values += 4;
+    }
+
+    int r = filter_length&3;
+    if (r) {
+      // Note: filter_values must be padded to align_up(filter_offset, 8);
+      __m128i coeff;
+      coeff = _mm_loadl_epi64(reinterpret_cast<const __m128i*>(filter_values));
+      // Mask out extra filter taps.
+      coeff = _mm_and_si128(coeff, mask[r]);
+
+      __m128i coeff16lo = _mm_shufflelo_epi16(coeff, _MM_SHUFFLE(1, 1, 0, 0));
+      /* c1 c1 c1 c1 c0 c0 c0 c0 */
+      coeff16lo = _mm_unpacklo_epi16(coeff16lo, coeff16lo);
+      __m128i coeff16hi = _mm_shufflelo_epi16(coeff, _MM_SHUFFLE(3, 3, 2, 2));
+      coeff16hi = _mm_unpacklo_epi16(coeff16hi, coeff16hi);
+
+      __m128i src8, src16, mul_hi, mul_lo, t;
+
+      ITERATION(src_data[0]+start, accum0);
+      ITERATION(src_data[1]+start, accum1);
+      ITERATION(src_data[2]+start, accum2);
+      ITERATION(src_data[3]+start, accum3);
+    }
+
+    accum0 = _mm_srai_epi32(accum0, ConvolutionFilter1D::kShiftBits);
+    accum0 = _mm_packs_epi32(accum0, zero);
+    accum0 = _mm_packus_epi16(accum0, zero);
+    accum1 = _mm_srai_epi32(accum1, ConvolutionFilter1D::kShiftBits);
+    accum1 = _mm_packs_epi32(accum1, zero);
+    accum1 = _mm_packus_epi16(accum1, zero);
+    accum2 = _mm_srai_epi32(accum2, ConvolutionFilter1D::kShiftBits);
+    accum2 = _mm_packs_epi32(accum2, zero);
+    accum2 = _mm_packus_epi16(accum2, zero);
+    accum3 = _mm_srai_epi32(accum3, ConvolutionFilter1D::kShiftBits);
+    accum3 = _mm_packs_epi32(accum3, zero);
+    accum3 = _mm_packus_epi16(accum3, zero);
+
+    *(reinterpret_cast<int*>(out_row[0])) = _mm_cvtsi128_si32(accum0);
+    *(reinterpret_cast<int*>(out_row[1])) = _mm_cvtsi128_si32(accum1);
+    *(reinterpret_cast<int*>(out_row[2])) = _mm_cvtsi128_si32(accum2);
+    *(reinterpret_cast<int*>(out_row[3])) = _mm_cvtsi128_si32(accum3);
+
+    out_row[0] += 4;
+    out_row[1] += 4;
+    out_row[2] += 4;
+    out_row[3] += 4;
+  }
+#endif
+}
+
+template<bool has_alpha>
+void ConvolveVertically_SSE2(const ConvolutionFilter1D::Fixed* filter_values,
+                        int filter_length,

[brettw, 2011/02/21 04:45:45]

Check indentation

[jiesun, 2011/02/22 21:37:03]

Done.

+                        unsigned char* const* source_data_rows,
+                        int pixel_width,
+                        unsigned char* out_row) {
+#ifdef ARCH_CPU_X86_FAMILY
+  int width = pixel_width & ~3;
+
+  __m128i zero = _mm_setzero_si128();
+  __m128i accum0, accum1, accum2, accum3, coeff16;
+  const __m128i* src;
+  for (int i = 0; i < width; i += 4) {  // Four pixels per iteration.
+    accum0 = _mm_setzero_si128();
+    accum1 = _mm_setzero_si128();
+    accum2 = _mm_setzero_si128();
+    accum3 = _mm_setzero_si128();
+    for (int j = 0; j < filter_length; ++j) {
+      coeff16 = _mm_set1_epi16(filter_values[j]);
+
+      // aligned load due to row_buffer is 16 byte aligned.
+      // [8] a3 b3 g3 r3 a2 b2 g2 r2 a1 b1 g1 r1 a0 b0 g0 r0
+      src = reinterpret_cast<const __m128i*>(&source_data_rows[j][i<<2]);
+      __m128i src8 = _mm_loadu_si128(src);
+      // [16] a1 b1 g1 r1 a0 b0 g0 r0
+      __m128i src16 = _mm_unpacklo_epi8(src8, zero);
+      __m128i mul_hi = _mm_mulhi_epi16(src16, coeff16);
+      __m128i mul_lo = _mm_mullo_epi16(src16, coeff16);
+      // [32] a0 b0 g0 r0
+      __m128i t = _mm_unpacklo_epi16(mul_lo, mul_hi);
+      accum0 = _mm_add_epi32(accum0, t);
+      // [32] a1 b1 g1 r1
+      t = _mm_unpackhi_epi16(mul_lo, mul_hi);
+      accum1 = _mm_add_epi32(accum1, t);
+      // [16] a3 b3 g3 r3 a2 b2 g2 r2
+      src16 = _mm_unpackhi_epi8(src8, zero);
+      mul_hi = _mm_mulhi_epi16(src16, coeff16);
+      mul_lo = _mm_mullo_epi16(src16, coeff16);
+      // [32] a2 b2 g2 r2
+      t = _mm_unpacklo_epi16(mul_lo, mul_hi);
+      accum2 = _mm_add_epi32(accum2, t);
+      // [32] a3 b3 g3 r3
+      t = _mm_unpackhi_epi16(mul_lo, mul_hi);
+      accum3 = _mm_add_epi32(accum3, t);
+    }
+
+    accum0 = _mm_srai_epi32(accum0, ConvolutionFilter1D::kShiftBits);
+    accum1 = _mm_srai_epi32(accum1, ConvolutionFilter1D::kShiftBits);
+    accum2 = _mm_srai_epi32(accum2, ConvolutionFilter1D::kShiftBits);
+    accum3 = _mm_srai_epi32(accum3, ConvolutionFilter1D::kShiftBits);
+    // [16] a1 b1 g1 r1 a0 b0 g0 r0
+    accum0 = _mm_packs_epi32(accum0, accum1);
+    // [16] a3 b3 g3 r3 a2 b2 g2 r2
+    accum2 = _mm_packs_epi32(accum2, accum3);
+    // [8] a3 b3 g3 r3 a2 b2 g2 r2 a1 b1 g1 r1 a0 b0 g0 r0
+    accum0 = _mm_packus_epi16(accum0, accum2);
+    if (has_alpha) {
+      // [8] xx a3 b3 g3 xx a2 b2 g2 xx a1 b1 g1 xx a0 b0 g0
+      __m128i a = _mm_srli_epi32(accum0, 8);
+      // [8] xx xx xx max3 xx xx xx max2 xx xx xx max1 xx xx xx max0
+      __m128i b = _mm_max_epu8(a, accum0);  // Max of r and g.
+      // [8] xx xx a3 b3 xx xx a2 b2 xx xx a1 b1 xx xx a0 b0
+      a = _mm_srli_epi32(accum0, 16);
+      // [8] xx xx xx max3 xx xx xx max2 xx xx xx max1 xx xx xx max0
+      b = _mm_max_epu8(a, b);  // Max of r and g and b.
+      // [8] max3 00 00 00 max2 00 00 00 max1 00 00 00 max0 00 00 00
+      b = _mm_slli_epi32(b, 24);
+      accum0 = _mm_max_epu8(b, accum0);
+    } else {
+      __m128i mask = _mm_set1_epi32(0xff000000);
+      accum0 = _mm_or_si128(accum0, mask);
+    }
+    _mm_storeu_si128(reinterpret_cast<__m128i*>(out_row), accum0);
+    out_row += 16;
+  }
+
+  if (pixel_width & 3) {
+    accum0 = _mm_setzero_si128();
+    accum1 = _mm_setzero_si128();
+    accum2 = _mm_setzero_si128();
+    for (int j = 0; j < filter_length; ++j) {
+      coeff16 = _mm_set1_epi16(filter_values[j]);
+      // [8] a3 b3 g3 r3 a2 b2 g2 r2 a1 b1 g1 r1 a0 b0 g0 r0
+      src = reinterpret_cast<const __m128i*>(&source_data_rows[j][width<<2]);
+      __m128i src8 = _mm_loadu_si128(src);
+      // [16] a1 b1 g1 r1 a0 b0 g0 r0
+      __m128i src16 = _mm_unpacklo_epi8(src8, zero);
+      __m128i mul_hi = _mm_mulhi_epi16(src16, coeff16);
+      __m128i mul_lo = _mm_mullo_epi16(src16, coeff16);
+      // [32] a0 b0 g0 r0
+      __m128i t = _mm_unpacklo_epi16(mul_lo, mul_hi);
+      accum0 = _mm_add_epi32(accum0, t);
+      // [32] a1 b1 g1 r1
+      t = _mm_unpackhi_epi16(mul_lo, mul_hi);
+      accum1 = _mm_add_epi32(accum1, t);
+      // [16] a3 b3 g3 r3 a2 b2 g2 r2
+      src16 = _mm_unpackhi_epi8(src8, zero);
+      mul_hi = _mm_mulhi_epi16(src16, coeff16);
+      mul_lo = _mm_mullo_epi16(src16, coeff16);
+      // [32] a2 b2 g2 r2
+      t = _mm_unpacklo_epi16(mul_lo, mul_hi);
+      accum2 = _mm_add_epi32(accum2, t);
+    }
+
+    accum0 = _mm_srai_epi32(accum0, ConvolutionFilter1D::kShiftBits);
+    accum1 = _mm_srai_epi32(accum1, ConvolutionFilter1D::kShiftBits);
+    accum2 = _mm_srai_epi32(accum2, ConvolutionFilter1D::kShiftBits);
+    // [16] a1 b1 g1 r1 a0 b0 g0 r0
+    accum0 = _mm_packs_epi32(accum0, accum1);
+    // [16] a3 b3 g3 r3 a2 b2 g2 r2
+    accum2 = _mm_packs_epi32(accum2, zero);
+    // [8] a3 b3 g3 r3 a2 b2 g2 r2 a1 b1 g1 r1 a0 b0 g0 r0
+    accum0 = _mm_packus_epi16(accum0, accum2);
+    if (has_alpha) {
+      // [8] xx a3 b3 g3 xx a2 b2 g2 xx a1 b1 g1 xx a0 b0 g0
+      __m128i a = _mm_srli_epi32(accum0, 8);
+      // [8] xx xx xx max3 xx xx xx max2 xx xx xx max1 xx xx xx max0
+      __m128i b = _mm_max_epu8(a, accum0);  // Max of r and g.
+      // [8] xx xx a3 b3 xx xx a2 b2 xx xx a1 b1 xx xx a0 b0
+      a = _mm_srli_epi32(accum0, 16);
+      // [8] xx xx xx max3 xx xx xx max2 xx xx xx max1 xx xx xx max0
+      b = _mm_max_epu8(a, b);  // Max of r and g and b.
+      // [8] max3 00 00 00 max2 00 00 00 max1 00 00 00 max0 00 00 00
+      b = _mm_slli_epi32(b, 24);
+      accum0 = _mm_max_epu8(b, accum0);
+    } else {
+      __m128i mask = _mm_set1_epi32(0xff000000);
+      accum0 = _mm_or_si128(accum0, mask);
+    }
+
+    for (int i = width; i < pixel_width; ++i) {
+      *(reinterpret_cast<int*>(out_row)) = _mm_cvtsi128_si32(accum0);
+      accum0 = _mm_srli_si128(accum0, 4);
+      out_row += 4;
+    }
+  }
+#endif
+}
+
 }  // namespace
 
 // ConvolutionFilter1D ---------------------------------------------------------
@@ -284,15 +649,13 @@ void ConvolutionFilter1D::AddFilter(int filter_offset,
   max_filter_ = std::max(max_filter_, filter_length);
 }
 
-// BGRAConvolve2D -------------------------------------------------------------
-
-void BGRAConvolve2D(const unsigned char* source_data,
-                    int source_byte_row_stride,
-                    bool source_has_alpha,
-                    const ConvolutionFilter1D& filter_x,
-                    const ConvolutionFilter1D& filter_y,
-                    int output_byte_row_stride,
-                    unsigned char* output) {
+void BGRAConvolve2D_C(const unsigned char* source_data,
+                      int source_byte_row_stride,
+                      bool source_has_alpha,
+                      const ConvolutionFilter1D& filter_x,
+                      const ConvolutionFilter1D& filter_y,
+                      int output_byte_row_stride,
+                      unsigned char* output) {
   int max_y_filter_size = filter_y.max_filter();
 
   // The next row in the input that we will generate a horizontally
@@ -317,6 +680,11 @@ void BGRAConvolve2D(const unsigned char* source_data,
   // convolutions to run each subsequent vertical convolution.
   SkASSERT(output_byte_row_stride >= filter_x.num_values() * 4);
   int num_output_rows = filter_y.num_values();
+
+  int last_filter_offset, last_filter_length;
+  filter_y.FilterForValue(num_output_rows-1, &last_filter_offset,
+                          &last_filter_length);
+
   for (int out_y = 0; out_y < num_output_rows; out_y++) {
     filter_values = filter_y.FilterForValue(out_y,
                                             &filter_offset, &filter_length);
@@ -354,10 +722,132 @@ void BGRAConvolve2D(const unsigned char* source_data,
                                filter_x.num_values(), cur_output_row);
     } else {
       ConvolveVertically<false>(filter_values, filter_length,
-                                first_row_for_filter,
-                                filter_x.num_values(), cur_output_row);
+                               first_row_for_filter,
+                               filter_x.num_values(), cur_output_row);
     }
   }
 }
 
+// BGRAConvolve2D -------------------------------------------------------------
+
+void BGRAConvolve2D_SSE2(const unsigned char* source_data,
+                         int source_byte_row_stride,
+                         bool source_has_alpha,
+                         const ConvolutionFilter1D& filter_x,
+                         const ConvolutionFilter1D& filter_y,
+                         int output_byte_row_stride,
+                         unsigned char* output) {
+  int max_y_filter_size = filter_y.max_filter();
+
+  // The next row in the input that we will generate a horizontally
+  // convolved row for. If the filter doesn't start at the beginning of the
+  // image (this is the case when we are only resizing a subset), then we
+  // don't want to generate any output rows before that. Compute the starting
+  // row for convolution as the first pixel for the first vertical filter.
+  int filter_offset, filter_length;
+  const ConvolutionFilter1D::Fixed* filter_values =
+      filter_y.FilterForValue(0, &filter_offset, &filter_length);
+  int next_x_row = filter_offset;
+
+  // We loop over each row in the input doing a horizontal convolution. This
+  // will result in a horizontally convolved image. We write the results into
+  // a circular buffer of convolved rows and do vertical convolution as rows
+  // are available. This prevents us from having to store the entire
+  // intermediate image and helps cache coherency.
+  // We will need four extra rows to allow horizontal convolution could be done
+  // simultaneously.
+  int row_buffer_width = (filter_x.num_values() + 15) & ~0xF;
+  int row_buffer_height = max_y_filter_size + 4;
+  CircularRowBuffer row_buffer(row_buffer_width,
+                               row_buffer_height,
+                               filter_offset);
+
+  // Loop over every possible output row, processing just enough horizontal
+  // convolutions to run each subsequent vertical convolution.
+  SkASSERT(output_byte_row_stride >= filter_x.num_values() * 4);
+  int num_output_rows = filter_y.num_values();
+
+  int last_filter_offset, last_filter_length;
+  filter_y.FilterForValue(num_output_rows-1, &last_filter_offset,
+                          &last_filter_length);
+
+  for (int out_y = 0; out_y < num_output_rows; out_y++) {
+    filter_values = filter_y.FilterForValue(out_y,
+                                            &filter_offset, &filter_length);
+
+    // Generate output rows until we have enough to run the current filter.
+    while (next_x_row < filter_offset + filter_length) {
+      if (next_x_row + 3 < last_filter_offset + last_filter_length - 1) {
+        const unsigned char* src[4];
+        unsigned char* out_row[4];
+        for (int i = 0; i < 4; ++i) {
+          src[i] = &source_data[(next_x_row+i) * source_byte_row_stride];
+          out_row[i] = row_buffer.AdvanceRow();
+        }
+        ConvolveHorizontally4_SSE2(src, filter_x, out_row);
+        next_x_row+=4;
+      } else {
+        // For the last row, SSE2 load possibly to access data beyond the
+        // image area. therefore we use C version here. Hacking into skia
+        // to add line paddings is not something in my mind.
+        if (next_x_row == last_filter_offset + last_filter_length - 1) {
+          if (source_has_alpha)
+            ConvolveHorizontally<true>(
+                &source_data[next_x_row * source_byte_row_stride],
+                filter_x, row_buffer.AdvanceRow());
+          else
+            ConvolveHorizontally<false>(
+                &source_data[next_x_row * source_byte_row_stride],
+                filter_x, row_buffer.AdvanceRow());
+        } else {
+          ConvolveHorizontally_SSE2(
+              &source_data[next_x_row * source_byte_row_stride],
+              filter_x, row_buffer.AdvanceRow());
+        }
+        next_x_row++;
+      }
+    }
+
+    // Compute where in the output image this row of final data will go.
+    unsigned char* cur_output_row = &output[out_y * output_byte_row_stride];
+
+    // Get the list of rows that the circular buffer has, in order.
+    int first_row_in_circular_buffer;
+    unsigned char* const* rows_to_convolve =
+        row_buffer.GetRowAddresses(&first_row_in_circular_buffer);
+
+    // Now compute the start of the subset of those rows that the filter
+    // needs.
+    unsigned char* const* first_row_for_filter =
+        &rows_to_convolve[filter_offset - first_row_in_circular_buffer];
+
+    if (source_has_alpha) {
+      ConvolveVertically_SSE2<true>(filter_values, filter_length,
+                                    first_row_for_filter,
+                                    filter_x.num_values(), cur_output_row);
+    } else {
+      ConvolveVertically_SSE2<false>(filter_values, filter_length,
+                                     first_row_for_filter,
+                                     filter_x.num_values(), cur_output_row);
+    }
+  }
+}
+
+void BGRAConvolve2D(const unsigned char* source_data,
+                    int source_byte_row_stride,
+                    bool source_has_alpha,
+                    const ConvolutionFilter1D& filter_x,
+                    const ConvolutionFilter1D& filter_y,
+                    int output_byte_row_stride,
+                    unsigned char* output) {
+  base::CPU cpu;
+  if (cpu.has_sse2()) {
+    BGRAConvolve2D_SSE2(source_data, source_byte_row_stride, source_has_alpha,
+                        filter_x, filter_y, output_byte_row_stride, output);
+  } else {
+    BGRAConvolve2D_C(source_data, source_byte_row_stride, source_has_alpha,
+                     filter_x, filter_y, output_byte_row_stride, output);
+  }
+}
+
 }  // namespace skia