SIMD implementation of Convolver for Lanczos filter etc.

skia/ext/convolver.cc

 // Copyright (c) 2011 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include <algorithm>
 
 #include "skia/ext/convolver.h"
 #include "third_party/skia/include/core/SkTypes.h"
 
+#if defined(ARCH_CPU_X86_FAMILY)
+#include <emmintrin.h>  // ARCH_CPU_X86_FAMILY was defined in build/config.h
+#endif
+
 namespace skia {
 
 namespace {
 
 // Converts the argument to an 8-bit unsigned value by clamping to the range
 // 0-255.
 inline unsigned char ClampTo8(int a) {
   if (static_cast<unsigned>(a) < 256)
     return a;  // Avoid the extra check in the common case.
   if (a < 0)
@@ skipping 172 matching lines @@
     if (has_alpha)
       accum[3] >>= ConvolutionFilter1D::kShiftBits;
 
     // Store the new pixel.
     out_row[byte_offset + 0] = ClampTo8(accum[0]);
     out_row[byte_offset + 1] = ClampTo8(accum[1]);
     out_row[byte_offset + 2] = ClampTo8(accum[2]);
     if (has_alpha) {
       unsigned char alpha = ClampTo8(accum[3]);
 
-      // Make sure the alpha channel doesn't come out larger than any of the
+      // Make sure the alpha channel doesn't come out smaller than any of the
       // color channels. We use premultipled alpha channels, so this should
       // never happen, but rounding errors will cause this from time to time.
       // These "impossible" colors will cause overflows (and hence random pixel
       // values) when the resulting bitmap is drawn to the screen.
       //
       // We only need to do this when generating the final output row (here).
       int max_color_channel = std::max(out_row[byte_offset + 0],
           std::max(out_row[byte_offset + 1], out_row[byte_offset + 2]));
       if (alpha < max_color_channel)
         out_row[byte_offset + 3] = max_color_channel;
       else
         out_row[byte_offset + 3] = alpha;
     } else {
       // No alpha channel, the image is opaque.
       out_row[byte_offset + 3] = 0xff;
     }
   }
 }
 
+
+// Convolves horizontally along a single row. The row data is given in
+// |src_data| and continues for the num_values() of the filter.
+void ConvolveHorizontally_SSE2(const unsigned char* src_data,
+                               const ConvolutionFilter1D& filter,
+                               unsigned char* out_row) {
+#ifdef ARCH_CPU_X86_FAMILY
+  int num_values = filter.num_values();
+
+  int filter_offset, filter_length;
+  __m128i zero = _mm_setzero_si128();
+  __m128i mask[4];

[brettw, 2011/02/21 04:45:45]

What's mask[0] for? Can you provide a comment for what this is for and note you
you don't need mask[0]?

[jiesun, 2011/02/22 21:37:03]

yes, mask[0] is not used.

+  mask[1] = _mm_set_epi16(0, 0, 0, 0, 0, 0, 0, -1);
+  mask[2] = _mm_set_epi16(0, 0, 0, 0, 0, 0, -1, -1);
+  mask[3] = _mm_set_epi16(0, 0, 0, 0, 0, -1, -1, -1);
+
+  for (int out_x = 0; out_x < num_values; out_x += 1) {
+    const ConvolutionFilter1D::Fixed* filter_values =
+        filter.FilterForValue(out_x, &filter_offset, &filter_length);
+
+    __m128i accum = _mm_setzero_si128();
+
+    const __m128i* row_to_filter =

[brettw, 2011/02/21 04:45:45]

Can you comment what this means?

[jiesun, 2011/02/22 21:37:03]

Done.

+        reinterpret_cast<const __m128i*>(&src_data[filter_offset << 2]);
+    // Four filter taps per iteration.
+    for (int j = 0; j < filter_length >> 2; ++j) {
+      __m128i coeff, coeff16;

[brettw, 2011/02/21 04:45:45]

For each of the "blocks" of SSE code you're written, can you provide comments
explaining in English what you're doing and what the contents of the different
variables will be at the end?

[jiesun, 2011/02/22 21:37:03]

Done.

+      // [16] xx xx xx xx c3 c2 c1 c0
+      coeff = _mm_loadl_epi64(reinterpret_cast<const __m128i*>(filter_values));
+      // [16] xx xx xx xx c1 c1 c0 c0
+      coeff16 = _mm_shufflelo_epi16(coeff, _MM_SHUFFLE(1, 1, 0, 0));
+      // [16] c1 c1 c1 c1 c0 c0 c0 c0
+      coeff16 = _mm_unpacklo_epi16(coeff16, coeff16);
+
+      // [8] a3 b3 g3 r3 a2 b2 g2 r2 a1 b1 g1 r1 a0 b0 g0 r0
+      __m128i src8 = _mm_loadu_si128(row_to_filter);
+      // [16] a1 b1 g1 r1 a0 b0 g0 r0
+      __m128i src16 = _mm_unpacklo_epi8(src8, zero);
+      __m128i mul_hi = _mm_mulhi_epi16(src16, coeff16);
+      __m128i mul_lo = _mm_mullo_epi16(src16, coeff16);
+      // [32]  a0*c0 b0*c0 g0*c0 r0*c0
+      __m128i t = _mm_unpacklo_epi16(mul_lo, mul_hi);
+      accum = _mm_add_epi32(accum, t);
+      // [32]  a1*c1 b1*c1 g1*c1 r1*c1
+      t = _mm_unpackhi_epi16(mul_lo, mul_hi);
+      accum = _mm_add_epi32(accum, t);
+
+      // [16] xx xx xx xx c3 c3 c2 c2
+      coeff16 = _mm_shufflelo_epi16(coeff, _MM_SHUFFLE(3, 3, 2, 2));
+      // [16] c3 c3 c3 c3 c2 c2 c2 c2
+      coeff16 = _mm_unpacklo_epi16(coeff16, coeff16);
+      // [16] a3 g3 b3 r3 a2 g2 b2 r2
+      src16 = _mm_unpackhi_epi8(src8, zero);
+      mul_hi = _mm_mulhi_epi16(src16, coeff16);
+      mul_lo = _mm_mullo_epi16(src16, coeff16);
+      // [32]  a2*c2 b2*c2 g2*c2 r2*c2
+      t = _mm_unpacklo_epi16(mul_lo, mul_hi);
+      accum = _mm_add_epi32(accum, t);
+      // [32]  a3*c3 b3*c3 g3*c3 r3*c3
+      t = _mm_unpackhi_epi16(mul_lo, mul_hi);
+      accum = _mm_add_epi32(accum, t);
+
+      row_to_filter += 1;
+      filter_values += 4;
+    }
+
+    // remaining

[brettw, 2011/02/21 04:45:45]

Can you provide a better comment here?

[jiesun, 2011/02/22 21:37:03]

Done.

+    int r = filter_length&3;
+    if (r) {
+      // Note: filter_values must be padded to align_up(filter_offset, 8).
+      __m128i coeff, coeff16;
+      coeff = _mm_loadl_epi64(reinterpret_cast<const __m128i*>(filter_values));
+      // Mask out extra filter taps.
+      coeff = _mm_and_si128(coeff, mask[r]);
+      coeff16 = _mm_shufflelo_epi16(coeff, _MM_SHUFFLE(1, 1, 0, 0));
+      coeff16 = _mm_unpacklo_epi16(coeff16, coeff16);
+
+      // Note: line buffer must be padded to align_up(filter_offset, 16).
+      // We resolve this by use C-version for the last horizontal line.
+      __m128i src8 = _mm_loadu_si128(row_to_filter);
+      __m128i src16 = _mm_unpacklo_epi8(src8, zero);
+      __m128i mul_hi = _mm_mulhi_epi16(src16, coeff16);
+      __m128i mul_lo = _mm_mullo_epi16(src16, coeff16);
+      __m128i t = _mm_unpacklo_epi16(mul_lo, mul_hi);
+      accum = _mm_add_epi32(accum, t);
+      t = _mm_unpackhi_epi16(mul_lo, mul_hi);
+      accum = _mm_add_epi32(accum, t);
+
+      src16 = _mm_unpackhi_epi8(src8, zero);
+      coeff16 = _mm_shufflelo_epi16(coeff, _MM_SHUFFLE(3, 3, 2, 2));
+      coeff16 = _mm_unpacklo_epi16(coeff16, coeff16);
+      mul_hi = _mm_mulhi_epi16(src16, coeff16);
+      mul_lo = _mm_mullo_epi16(src16, coeff16);
+      t = _mm_unpacklo_epi16(mul_lo, mul_hi);
+      accum = _mm_add_epi32(accum, t);
+    }
+
+    // shift right for fix point implementation before saturation.
+    accum = _mm_srai_epi32(accum, ConvolutionFilter1D::kShiftBits);
+    accum = _mm_packs_epi32(accum, zero);
+    accum = _mm_packus_epi16(accum, zero);
+
+    *(reinterpret_cast<int*>(out_row)) = _mm_cvtsi128_si32(accum);
+    out_row += 4;
+  }
+#endif
+}
+
+// Convolves horizontally along four rows. The row data is given in
+// |src_data| and continues for the num_values() of the filter.
+void ConvolveHorizontally4_SSE2(const unsigned char* src_data[4],
+                                const ConvolutionFilter1D& filter,
+                                unsigned char* out_row[4]) {
+#ifdef ARCH_CPU_X86_FAMILY
+  int width = filter.num_values();
+
+  int filter_offset, filter_length;
+  __m128i zero = _mm_setzero_si128();
+  __m128i mask[4];
+  mask[1] = _mm_set_epi16(0, 0, 0, 0, 0, 0, 0, -1);
+  mask[2] = _mm_set_epi16(0, 0, 0, 0, 0, 0, -1, -1);
+  mask[3] = _mm_set_epi16(0, 0, 0, 0, 0, -1, -1, -1);
+
+  for (int i = 0; i < width; ++i) {
+    const ConvolutionFilter1D::Fixed* filter_values =
+        filter.FilterForValue(i, &filter_offset, &filter_length);
+
+    // four pixels in a column per iteration.
+    __m128i accum0 = _mm_setzero_si128();
+    __m128i accum1 = _mm_setzero_si128();
+    __m128i accum2 = _mm_setzero_si128();
+    __m128i accum3 = _mm_setzero_si128();
+    int start = (filter_offset<<2);
+    for (int j = 0; j < (filter_length >> 2); ++j) {
+      __m128i coeff, coeff16lo, coeff16hi;
+      // [16] xx xx xx xx c3 c2 c1 c0
+      coeff = _mm_loadl_epi64(reinterpret_cast<const __m128i*>(filter_values));
+      // [16] xx xx xx xx c1 c1 c0 c0
+      coeff16lo = _mm_shufflelo_epi16(coeff, _MM_SHUFFLE(1, 1, 0, 0));
+      // [16] c1 c1 c1 c1 c0 c0 c0 c0
+      coeff16lo = _mm_unpacklo_epi16(coeff16lo, coeff16lo);
+      // [16] xx xx xx xx c3 c3 c2 c2
+      coeff16hi = _mm_shufflelo_epi16(coeff, _MM_SHUFFLE(3, 3, 2, 2));
+      // [16] c3 c3 c3 c3 c2 c2 c2 c2
+      coeff16hi = _mm_unpacklo_epi16(coeff16hi, coeff16hi);
+
+      __m128i src8, src16, mul_hi, mul_lo, t;
+
+#define ITERATION(src, accum)                                          \
+      src8 = _mm_loadu_si128(reinterpret_cast<const __m128i*>(src));   \
+      src16 = _mm_unpacklo_epi8(src8, zero);                           \
+      mul_hi = _mm_mulhi_epi16(src16, coeff16lo);                      \
+      mul_lo = _mm_mullo_epi16(src16, coeff16lo);                      \
+      t = _mm_unpacklo_epi16(mul_lo, mul_hi);                          \
+      accum = _mm_add_epi32(accum, t);                                 \
+      t = _mm_unpackhi_epi16(mul_lo, mul_hi);                          \
+      accum = _mm_add_epi32(accum, t);                                 \
+      src16 = _mm_unpackhi_epi8(src8, zero);                           \
+      mul_hi = _mm_mulhi_epi16(src16, coeff16hi);                      \
+      mul_lo = _mm_mullo_epi16(src16, coeff16hi);                      \
+      t = _mm_unpacklo_epi16(mul_lo, mul_hi);                          \
+      accum = _mm_add_epi32(accum, t);                                 \
+      t = _mm_unpackhi_epi16(mul_lo, mul_hi);                          \
+      accum = _mm_add_epi32(accum, t)
+
+      ITERATION(src_data[0]+start, accum0);
+      ITERATION(src_data[1]+start, accum1);
+      ITERATION(src_data[2]+start, accum2);
+      ITERATION(src_data[3]+start, accum3);
+
+      start += 16;
+      filter_values += 4;
+    }
+
+    int r = filter_length&3;
+    if (r) {
+      // Note: filter_values must be padded to align_up(filter_offset, 8);
+      __m128i coeff;
+      coeff = _mm_loadl_epi64(reinterpret_cast<const __m128i*>(filter_values));
+      // Mask out extra filter taps.
+      coeff = _mm_and_si128(coeff, mask[r]);
+
+      __m128i coeff16lo = _mm_shufflelo_epi16(coeff, _MM_SHUFFLE(1, 1, 0, 0));
+      /* c1 c1 c1 c1 c0 c0 c0 c0 */
+      coeff16lo = _mm_unpacklo_epi16(coeff16lo, coeff16lo);
+      __m128i coeff16hi = _mm_shufflelo_epi16(coeff, _MM_SHUFFLE(3, 3, 2, 2));
+      coeff16hi = _mm_unpacklo_epi16(coeff16hi, coeff16hi);
+
+      __m128i src8, src16, mul_hi, mul_lo, t;
+
+      ITERATION(src_data[0]+start, accum0);
+      ITERATION(src_data[1]+start, accum1);
+      ITERATION(src_data[2]+start, accum2);
+      ITERATION(src_data[3]+start, accum3);
+    }
+
+    accum0 = _mm_srai_epi32(accum0, ConvolutionFilter1D::kShiftBits);
+    accum0 = _mm_packs_epi32(accum0, zero);
+    accum0 = _mm_packus_epi16(accum0, zero);
+    accum1 = _mm_srai_epi32(accum1, ConvolutionFilter1D::kShiftBits);
+    accum1 = _mm_packs_epi32(accum1, zero);
+    accum1 = _mm_packus_epi16(accum1, zero);
+    accum2 = _mm_srai_epi32(accum2, ConvolutionFilter1D::kShiftBits);
+    accum2 = _mm_packs_epi32(accum2, zero);
+    accum2 = _mm_packus_epi16(accum2, zero);
+    accum3 = _mm_srai_epi32(accum3, ConvolutionFilter1D::kShiftBits);
+    accum3 = _mm_packs_epi32(accum3, zero);
+    accum3 = _mm_packus_epi16(accum3, zero);
+
+    *(reinterpret_cast<int*>(out_row[0])) = _mm_cvtsi128_si32(accum0);
+    *(reinterpret_cast<int*>(out_row[1])) = _mm_cvtsi128_si32(accum1);
+    *(reinterpret_cast<int*>(out_row[2])) = _mm_cvtsi128_si32(accum2);
+    *(reinterpret_cast<int*>(out_row[3])) = _mm_cvtsi128_si32(accum3);
+
+    out_row[0] += 4;
+    out_row[1] += 4;
+    out_row[2] += 4;
+    out_row[3] += 4;
+  }
+#endif
+}
+
+template<bool has_alpha>
+void ConvolveVertically_SSE2(const ConvolutionFilter1D::Fixed* filter_values,
+                        int filter_length,

[brettw, 2011/02/21 04:45:45]

Check indentation

[jiesun, 2011/02/22 21:37:03]

Done.

+                        unsigned char* const* source_data_rows,
+                        int pixel_width,
+                        unsigned char* out_row) {
+#ifdef ARCH_CPU_X86_FAMILY
+  int width = pixel_width & ~3;
+
+  __m128i zero = _mm_setzero_si128();
+  __m128i accum0, accum1, accum2, accum3, coeff16;
+  const __m128i* src;
+  for (int i = 0; i < width; i += 4) {  // Four pixels per iteration.
+    accum0 = _mm_setzero_si128();
+    accum1 = _mm_setzero_si128();
+    accum2 = _mm_setzero_si128();
+    accum3 = _mm_setzero_si128();
+    for (int j = 0; j < filter_length; ++j) {
+      coeff16 = _mm_set1_epi16(filter_values[j]);
+
+      // aligned load due to row_buffer is 16 byte aligned.
+      // [8] a3 b3 g3 r3 a2 b2 g2 r2 a1 b1 g1 r1 a0 b0 g0 r0
+      src = reinterpret_cast<const __m128i*>(&source_data_rows[j][i<<2]);
+      __m128i src8 = _mm_loadu_si128(src);
+      // [16] a1 b1 g1 r1 a0 b0 g0 r0
+      __m128i src16 = _mm_unpacklo_epi8(src8, zero);
+      __m128i mul_hi = _mm_mulhi_epi16(src16, coeff16);
+      __m128i mul_lo = _mm_mullo_epi16(src16, coeff16);
+      // [32] a0 b0 g0 r0
+      __m128i t = _mm_unpacklo_epi16(mul_lo, mul_hi);
+      accum0 = _mm_add_epi32(accum0, t);
+      // [32] a1 b1 g1 r1
+      t = _mm_unpackhi_epi16(mul_lo, mul_hi);
+      accum1 = _mm_add_epi32(accum1, t);
+      // [16] a3 b3 g3 r3 a2 b2 g2 r2
+      src16 = _mm_unpackhi_epi8(src8, zero);
+      mul_hi = _mm_mulhi_epi16(src16, coeff16);
+      mul_lo = _mm_mullo_epi16(src16, coeff16);
+      // [32] a2 b2 g2 r2
+      t = _mm_unpacklo_epi16(mul_lo, mul_hi);
+      accum2 = _mm_add_epi32(accum2, t);
+      // [32] a3 b3 g3 r3
+      t = _mm_unpackhi_epi16(mul_lo, mul_hi);
+      accum3 = _mm_add_epi32(accum3, t);
+    }
+
+    accum0 = _mm_srai_epi32(accum0, ConvolutionFilter1D::kShiftBits);
+    accum1 = _mm_srai_epi32(accum1, ConvolutionFilter1D::kShiftBits);
+    accum2 = _mm_srai_epi32(accum2, ConvolutionFilter1D::kShiftBits);
+    accum3 = _mm_srai_epi32(accum3, ConvolutionFilter1D::kShiftBits);
+    // [16] a1 b1 g1 r1 a0 b0 g0 r0
+    accum0 = _mm_packs_epi32(accum0, accum1);
+    // [16] a3 b3 g3 r3 a2 b2 g2 r2
+    accum2 = _mm_packs_epi32(accum2, accum3);
+    // [8] a3 b3 g3 r3 a2 b2 g2 r2 a1 b1 g1 r1 a0 b0 g0 r0
+    accum0 = _mm_packus_epi16(accum0, accum2);
+    if (has_alpha) {
+      // [8] xx a3 b3 g3 xx a2 b2 g2 xx a1 b1 g1 xx a0 b0 g0
+      __m128i a = _mm_srli_epi32(accum0, 8);
+      // [8] xx xx xx max3 xx xx xx max2 xx xx xx max1 xx xx xx max0
+      __m128i b = _mm_max_epu8(a, accum0);  // Max of r and g.
+      // [8] xx xx a3 b3 xx xx a2 b2 xx xx a1 b1 xx xx a0 b0
+      a = _mm_srli_epi32(accum0, 16);
+      // [8] xx xx xx max3 xx xx xx max2 xx xx xx max1 xx xx xx max0
+      b = _mm_max_epu8(a, b);  // Max of r and g and b.
+      // [8] max3 00 00 00 max2 00 00 00 max1 00 00 00 max0 00 00 00
+      b = _mm_slli_epi32(b, 24);
+      accum0 = _mm_max_epu8(b, accum0);
+    } else {
+      __m128i mask = _mm_set1_epi32(0xff000000);
+      accum0 = _mm_or_si128(accum0, mask);
+    }
+    _mm_storeu_si128(reinterpret_cast<__m128i*>(out_row), accum0);
+    out_row += 16;
+  }
+
+  if (pixel_width & 3) {
+    accum0 = _mm_setzero_si128();
+    accum1 = _mm_setzero_si128();
+    accum2 = _mm_setzero_si128();
+    for (int j = 0; j < filter_length; ++j) {
+      coeff16 = _mm_set1_epi16(filter_values[j]);
+      // [8] a3 b3 g3 r3 a2 b2 g2 r2 a1 b1 g1 r1 a0 b0 g0 r0
+      src = reinterpret_cast<const __m128i*>(&source_data_rows[j][width<<2]);
+      __m128i src8 = _mm_loadu_si128(src);
+      // [16] a1 b1 g1 r1 a0 b0 g0 r0
+      __m128i src16 = _mm_unpacklo_epi8(src8, zero);
+      __m128i mul_hi = _mm_mulhi_epi16(src16, coeff16);
+      __m128i mul_lo = _mm_mullo_epi16(src16, coeff16);
+      // [32] a0 b0 g0 r0
+      __m128i t = _mm_unpacklo_epi16(mul_lo, mul_hi);
+      accum0 = _mm_add_epi32(accum0, t);
+      // [32] a1 b1 g1 r1
+      t = _mm_unpackhi_epi16(mul_lo, mul_hi);
+      accum1 = _mm_add_epi32(accum1, t);
+      // [16] a3 b3 g3 r3 a2 b2 g2 r2
+      src16 = _mm_unpackhi_epi8(src8, zero);
+      mul_hi = _mm_mulhi_epi16(src16, coeff16);
+      mul_lo = _mm_mullo_epi16(src16, coeff16);
+      // [32] a2 b2 g2 r2
+      t = _mm_unpacklo_epi16(mul_lo, mul_hi);
+      accum2 = _mm_add_epi32(accum2, t);
+    }
+
+    accum0 = _mm_srai_epi32(accum0, ConvolutionFilter1D::kShiftBits);
+    accum1 = _mm_srai_epi32(accum1, ConvolutionFilter1D::kShiftBits);
+    accum2 = _mm_srai_epi32(accum2, ConvolutionFilter1D::kShiftBits);
+    // [16] a1 b1 g1 r1 a0 b0 g0 r0
+    accum0 = _mm_packs_epi32(accum0, accum1);
+    // [16] a3 b3 g3 r3 a2 b2 g2 r2
+    accum2 = _mm_packs_epi32(accum2, zero);
+    // [8] a3 b3 g3 r3 a2 b2 g2 r2 a1 b1 g1 r1 a0 b0 g0 r0
+    accum0 = _mm_packus_epi16(accum0, accum2);
+    if (has_alpha) {
+      // [8] xx a3 b3 g3 xx a2 b2 g2 xx a1 b1 g1 xx a0 b0 g0
+      __m128i a = _mm_srli_epi32(accum0, 8);
+      // [8] xx xx xx max3 xx xx xx max2 xx xx xx max1 xx xx xx max0
+      __m128i b = _mm_max_epu8(a, accum0);  // Max of r and g.
+      // [8] xx xx a3 b3 xx xx a2 b2 xx xx a1 b1 xx xx a0 b0
+      a = _mm_srli_epi32(accum0, 16);
+      // [8] xx xx xx max3 xx xx xx max2 xx xx xx max1 xx xx xx max0
+      b = _mm_max_epu8(a, b);  // Max of r and g and b.
+      // [8] max3 00 00 00 max2 00 00 00 max1 00 00 00 max0 00 00 00
+      b = _mm_slli_epi32(b, 24);
+      accum0 = _mm_max_epu8(b, accum0);
+    } else {
+      __m128i mask = _mm_set1_epi32(0xff000000);
+      accum0 = _mm_or_si128(accum0, mask);
+    }
+
+    for (int i = width; i < pixel_width; ++i) {
+      *(reinterpret_cast<int*>(out_row)) = _mm_cvtsi128_si32(accum0);
+      accum0 = _mm_srli_si128(accum0, 4);
+      out_row += 4;
+    }
+  }
+#endif
+}
+
 }  // namespace
 
 // ConvolutionFilter1D ---------------------------------------------------------
 
 ConvolutionFilter1D::ConvolutionFilter1D()
     : max_filter_(0) {
 }
 
 ConvolutionFilter1D::~ConvolutionFilter1D() {
 }
@@ skipping 45 matching lines @@
   // We pushed filter_length elements onto filter_values_
   instance.data_location = (static_cast<int>(filter_values_.size()) -
                             filter_length);
   instance.offset = filter_offset;
   instance.length = filter_length;
   filters_.push_back(instance);
 
   max_filter_ = std::max(max_filter_, filter_length);
 }
 
-// BGRAConvolve2D -------------------------------------------------------------
-
-void BGRAConvolve2D(const unsigned char* source_data,
-                    int source_byte_row_stride,
-                    bool source_has_alpha,
-                    const ConvolutionFilter1D& filter_x,
-                    const ConvolutionFilter1D& filter_y,
-                    int output_byte_row_stride,
-                    unsigned char* output) {
+void BGRAConvolve2D_C(const unsigned char* source_data,
+                      int source_byte_row_stride,
+                      bool source_has_alpha,
+                      const ConvolutionFilter1D& filter_x,
+                      const ConvolutionFilter1D& filter_y,
+                      int output_byte_row_stride,
+                      unsigned char* output) {
   int max_y_filter_size = filter_y.max_filter();
 
   // The next row in the input that we will generate a horizontally
   // convolved row for. If the filter doesn't start at the beginning of the
   // image (this is the case when we are only resizing a subset), then we
   // don't want to generate any output rows before that. Compute the starting
   // row for convolution as the first pixel for the first vertical filter.
   int filter_offset, filter_length;
   const ConvolutionFilter1D::Fixed* filter_values =
       filter_y.FilterForValue(0, &filter_offset, &filter_length);
   int next_x_row = filter_offset;
 
   // We loop over each row in the input doing a horizontal convolution. This
   // will result in a horizontally convolved image. We write the results into
   // a circular buffer of convolved rows and do vertical convolution as rows
   // are available. This prevents us from having to store the entire
   // intermediate image and helps cache coherency.
   CircularRowBuffer row_buffer(filter_x.num_values(), max_y_filter_size,
                                filter_offset);
 
   // Loop over every possible output row, processing just enough horizontal
   // convolutions to run each subsequent vertical convolution.
   SkASSERT(output_byte_row_stride >= filter_x.num_values() * 4);
   int num_output_rows = filter_y.num_values();
+
+  int last_filter_offset, last_filter_length;
+  filter_y.FilterForValue(num_output_rows-1, &last_filter_offset,
+                          &last_filter_length);
+
   for (int out_y = 0; out_y < num_output_rows; out_y++) {
     filter_values = filter_y.FilterForValue(out_y,
                                             &filter_offset, &filter_length);
 
     // Generate output rows until we have enough to run the current filter.
     while (next_x_row < filter_offset + filter_length) {
       if (source_has_alpha) {
         ConvolveHorizontally<true>(
             &source_data[next_x_row * source_byte_row_stride],
             filter_x, row_buffer.AdvanceRow());
@@ skipping 17 matching lines @@
     // needs.
     unsigned char* const* first_row_for_filter =
         &rows_to_convolve[filter_offset - first_row_in_circular_buffer];
 
     if (source_has_alpha) {
       ConvolveVertically<true>(filter_values, filter_length,
                                first_row_for_filter,
                                filter_x.num_values(), cur_output_row);
     } else {
       ConvolveVertically<false>(filter_values, filter_length,
-                                first_row_for_filter,
-                                filter_x.num_values(), cur_output_row);
+                               first_row_for_filter,
+                               filter_x.num_values(), cur_output_row);
     }
   }
 }
 
+// BGRAConvolve2D -------------------------------------------------------------
+
+void BGRAConvolve2D_SSE2(const unsigned char* source_data,
+                         int source_byte_row_stride,
+                         bool source_has_alpha,
+                         const ConvolutionFilter1D& filter_x,
+                         const ConvolutionFilter1D& filter_y,
+                         int output_byte_row_stride,
+                         unsigned char* output) {
+  int max_y_filter_size = filter_y.max_filter();
+
+  // The next row in the input that we will generate a horizontally
+  // convolved row for. If the filter doesn't start at the beginning of the
+  // image (this is the case when we are only resizing a subset), then we
+  // don't want to generate any output rows before that. Compute the starting
+  // row for convolution as the first pixel for the first vertical filter.
+  int filter_offset, filter_length;
+  const ConvolutionFilter1D::Fixed* filter_values =
+      filter_y.FilterForValue(0, &filter_offset, &filter_length);
+  int next_x_row = filter_offset;
+
+  // We loop over each row in the input doing a horizontal convolution. This
+  // will result in a horizontally convolved image. We write the results into
+  // a circular buffer of convolved rows and do vertical convolution as rows
+  // are available. This prevents us from having to store the entire
+  // intermediate image and helps cache coherency.
+  // We will need four extra rows to allow horizontal convolution could be done
+  // simultaneously.
+  int row_buffer_width = (filter_x.num_values() + 15) & ~0xF;
+  int row_buffer_height = max_y_filter_size + 4;
+  CircularRowBuffer row_buffer(row_buffer_width,
+                               row_buffer_height,
+                               filter_offset);
+
+  // Loop over every possible output row, processing just enough horizontal
+  // convolutions to run each subsequent vertical convolution.
+  SkASSERT(output_byte_row_stride >= filter_x.num_values() * 4);
+  int num_output_rows = filter_y.num_values();
+
+  int last_filter_offset, last_filter_length;
+  filter_y.FilterForValue(num_output_rows-1, &last_filter_offset,
+                          &last_filter_length);
+
+  for (int out_y = 0; out_y < num_output_rows; out_y++) {
+    filter_values = filter_y.FilterForValue(out_y,
+                                            &filter_offset, &filter_length);
+
+    // Generate output rows until we have enough to run the current filter.
+    while (next_x_row < filter_offset + filter_length) {
+      if (next_x_row + 3 < last_filter_offset + last_filter_length - 1) {
+        const unsigned char* src[4];
+        unsigned char* out_row[4];
+        for (int i = 0; i < 4; ++i) {
+          src[i] = &source_data[(next_x_row+i) * source_byte_row_stride];
+          out_row[i] = row_buffer.AdvanceRow();
+        }
+        ConvolveHorizontally4_SSE2(src, filter_x, out_row);
+        next_x_row+=4;
+      } else {
+        // For the last row, SSE2 load possibly to access data beyond the
+        // image area. therefore we use C version here. Hacking into skia
+        // to add line paddings is not something in my mind.
+        if (next_x_row == last_filter_offset + last_filter_length - 1) {
+          if (source_has_alpha)
+            ConvolveHorizontally<true>(
+                &source_data[next_x_row * source_byte_row_stride],
+                filter_x, row_buffer.AdvanceRow());
+          else
+            ConvolveHorizontally<false>(
+                &source_data[next_x_row * source_byte_row_stride],
+                filter_x, row_buffer.AdvanceRow());
+        } else {
+          ConvolveHorizontally_SSE2(
+              &source_data[next_x_row * source_byte_row_stride],
+              filter_x, row_buffer.AdvanceRow());
+        }
+        next_x_row++;
+      }
+    }
+
+    // Compute where in the output image this row of final data will go.
+    unsigned char* cur_output_row = &output[out_y * output_byte_row_stride];
+
+    // Get the list of rows that the circular buffer has, in order.
+    int first_row_in_circular_buffer;
+    unsigned char* const* rows_to_convolve =
+        row_buffer.GetRowAddresses(&first_row_in_circular_buffer);
+
+    // Now compute the start of the subset of those rows that the filter
+    // needs.
+    unsigned char* const* first_row_for_filter =
+        &rows_to_convolve[filter_offset - first_row_in_circular_buffer];
+
+    if (source_has_alpha) {
+      ConvolveVertically_SSE2<true>(filter_values, filter_length,
+                                    first_row_for_filter,
+                                    filter_x.num_values(), cur_output_row);
+    } else {
+      ConvolveVertically_SSE2<false>(filter_values, filter_length,
+                                     first_row_for_filter,
+                                     filter_x.num_values(), cur_output_row);
+    }
+  }
+}
+
+void BGRAConvolve2D(const unsigned char* source_data,
+                    int source_byte_row_stride,
+                    bool source_has_alpha,
+                    const ConvolutionFilter1D& filter_x,
+                    const ConvolutionFilter1D& filter_y,
+                    int output_byte_row_stride,
+                    unsigned char* output) {
+  base::CPU cpu;
+  if (cpu.has_sse2()) {
+    BGRAConvolve2D_SSE2(source_data, source_byte_row_stride, source_has_alpha,
+                        filter_x, filter_y, output_byte_row_stride, output);
+  } else {
+    BGRAConvolve2D_C(source_data, source_byte_row_stride, source_has_alpha,
+                     filter_x, filter_y, output_byte_row_stride, output);
+  }
+}
+
 }  // namespace skia