libvpx: Pull from upstream

source/libvpx/vp9/encoder/x86/vp9_dct_sse2.c

 /*
  *  Copyright (c) 2012 The WebM project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
 #include <emmintrin.h>  // SSE2
 #include "vp9/common/vp9_idct.h"  // for cospi constants
 #include "vpx_ports/mem.h"
 
+void vp9_fdct4x4_1_sse2(const int16_t *input, int16_t *output, int stride) {
+  __m128i in0, in1;
+  __m128i tmp;
+  const __m128i zero = _mm_setzero_si128();
+  in0  = _mm_loadl_epi64((const __m128i *)(input +  0 * stride));
+  in1  = _mm_loadl_epi64((const __m128i *)(input +  1 * stride));
+  in1  = _mm_unpacklo_epi64(in1, _mm_loadl_epi64((const __m128i *)
+         (input +  2 * stride)));
+  in0  = _mm_unpacklo_epi64(in0, _mm_loadl_epi64((const __m128i *)
+         (input +  3 * stride)));
+
+  tmp = _mm_add_epi16(in0, in1);
+  in0 = _mm_unpacklo_epi16(zero, tmp);
+  in1 = _mm_unpackhi_epi16(zero, tmp);
+  in0 = _mm_srai_epi32(in0, 16);
+  in1 = _mm_srai_epi32(in1, 16);
+
+  tmp = _mm_add_epi32(in0, in1);
+  in0 = _mm_unpacklo_epi32(tmp, zero);
+  in1 = _mm_unpackhi_epi32(tmp, zero);
+
+  tmp = _mm_add_epi32(in0, in1);
+  in0 = _mm_srli_si128(tmp, 8);
+
+  in1 = _mm_add_epi32(tmp, in0);
+  in0 = _mm_slli_epi32(in1, 1);
+  _mm_store_si128((__m128i *)(output), in0);
+}
+
 void vp9_fdct4x4_sse2(const int16_t *input, int16_t *output, int stride) {
   // This 2D transform implements 4 vertical 1D transforms followed
   // by 4 horizontal 1D transforms.  The multiplies and adds are as given
   // by Chen, Smith and Fralick ('77).  The commands for moving the data
   // around have been minimized by hand.
   // For the purposes of the comments, the 16 inputs are referred to at i0
   // through iF (in raster order), intermediate variables are a0, b0, c0
   // through f, and correspond to the in-place computations mapped to input
   // locations.  The outputs, o0 through oF are labeled according to the
   // output locations.
@@ skipping 345 matching lines @@
       fadst4_sse2(in);
       fadst4_sse2(in);
       write_buffer_4x4(output, in);
       break;
    default:
      assert(0);
      break;
   }
 }
 
+void vp9_fdct8x8_1_sse2(const int16_t *input, int16_t *output, int stride) {
+  __m128i in0  = _mm_load_si128((const __m128i *)(input + 0 * stride));
+  __m128i in1  = _mm_load_si128((const __m128i *)(input + 1 * stride));
+  __m128i in2  = _mm_load_si128((const __m128i *)(input + 2 * stride));
+  __m128i in3  = _mm_load_si128((const __m128i *)(input + 3 * stride));
+  __m128i u0, u1, sum;
+
+  u0 = _mm_add_epi16(in0, in1);
+  u1 = _mm_add_epi16(in2, in3);
+
+  in0  = _mm_load_si128((const __m128i *)(input + 4 * stride));
+  in1  = _mm_load_si128((const __m128i *)(input + 5 * stride));
+  in2  = _mm_load_si128((const __m128i *)(input + 6 * stride));
+  in3  = _mm_load_si128((const __m128i *)(input + 7 * stride));
+
+  sum = _mm_add_epi16(u0, u1);
+
+  in0 = _mm_add_epi16(in0, in1);
+  in2 = _mm_add_epi16(in2, in3);
+  sum = _mm_add_epi16(sum, in0);
+
+  u0  = _mm_setzero_si128();
+  sum = _mm_add_epi16(sum, in2);
+
+  in0 = _mm_unpacklo_epi16(u0, sum);
+  in1 = _mm_unpackhi_epi16(u0, sum);
+  in0 = _mm_srai_epi32(in0, 16);
+  in1 = _mm_srai_epi32(in1, 16);
+
+  sum = _mm_add_epi32(in0, in1);
+  in0 = _mm_unpacklo_epi32(sum, u0);
+  in1 = _mm_unpackhi_epi32(sum, u0);
+
+  sum = _mm_add_epi32(in0, in1);
+  in0 = _mm_srli_si128(sum, 8);
+
+  in1 = _mm_add_epi32(sum, in0);
+  _mm_store_si128((__m128i *)(output), in1);
+}
+
 void vp9_fdct8x8_sse2(const int16_t *input, int16_t *output, int stride) {
   int pass;
   // Constants
   //    When we use them, in one case, they are all the same. In all others
   //    it's a pair of them that we need to repeat four times. This is done
   //    by constructing the 32 bit constant corresponding to that pair.
   const __m128i k__cospi_p16_p16 = _mm_set1_epi16(cospi_16_64);
   const __m128i k__cospi_p16_m16 = pair_set_epi16(cospi_16_64, -cospi_16_64);
   const __m128i k__cospi_p24_p08 = pair_set_epi16(cospi_24_64, cospi_8_64);
   const __m128i k__cospi_m08_p24 = pair_set_epi16(-cospi_8_64, cospi_24_64);
@@ skipping 771 matching lines @@
       fadst8_sse2(in);
       right_shift_8x8(in, 1);
       write_buffer_8x8(output, in, 8);
       break;
     default:
       assert(0);
       break;
   }
 }
 
+void vp9_fdct16x16_1_sse2(const int16_t *input, int16_t *output, int stride) {
+  __m128i in0, in1, in2, in3;
+  __m128i u0, u1;
+  __m128i sum = _mm_setzero_si128();
+  int i;
+
+  for (i = 0; i < 2; ++i) {
+    input += 8 * i;
+    in0  = _mm_load_si128((const __m128i *)(input +  0 * stride));
+    in1  = _mm_load_si128((const __m128i *)(input +  1 * stride));
+    in2  = _mm_load_si128((const __m128i *)(input +  2 * stride));
+    in3  = _mm_load_si128((const __m128i *)(input +  3 * stride));
+
+    u0 = _mm_add_epi16(in0, in1);
+    u1 = _mm_add_epi16(in2, in3);
+    sum = _mm_add_epi16(sum, u0);
+
+    in0  = _mm_load_si128((const __m128i *)(input +  4 * stride));
+    in1  = _mm_load_si128((const __m128i *)(input +  5 * stride));
+    in2  = _mm_load_si128((const __m128i *)(input +  6 * stride));
+    in3  = _mm_load_si128((const __m128i *)(input +  7 * stride));
+
+    sum = _mm_add_epi16(sum, u1);
+    u0  = _mm_add_epi16(in0, in1);
+    u1  = _mm_add_epi16(in2, in3);
+    sum = _mm_add_epi16(sum, u0);
+
+    in0  = _mm_load_si128((const __m128i *)(input +  8 * stride));
+    in1  = _mm_load_si128((const __m128i *)(input +  9 * stride));
+    in2  = _mm_load_si128((const __m128i *)(input + 10 * stride));
+    in3  = _mm_load_si128((const __m128i *)(input + 11 * stride));
+
+    sum = _mm_add_epi16(sum, u1);
+    u0  = _mm_add_epi16(in0, in1);
+    u1  = _mm_add_epi16(in2, in3);
+    sum = _mm_add_epi16(sum, u0);
+
+    in0  = _mm_load_si128((const __m128i *)(input + 12 * stride));
+    in1  = _mm_load_si128((const __m128i *)(input + 13 * stride));
+    in2  = _mm_load_si128((const __m128i *)(input + 14 * stride));
+    in3  = _mm_load_si128((const __m128i *)(input + 15 * stride));
+
+    sum = _mm_add_epi16(sum, u1);
+    u0  = _mm_add_epi16(in0, in1);
+    u1  = _mm_add_epi16(in2, in3);
+    sum = _mm_add_epi16(sum, u0);
+
+    sum = _mm_add_epi16(sum, u1);
+  }
+
+  u0  = _mm_setzero_si128();
+  in0 = _mm_unpacklo_epi16(u0, sum);
+  in1 = _mm_unpackhi_epi16(u0, sum);
+  in0 = _mm_srai_epi32(in0, 16);
+  in1 = _mm_srai_epi32(in1, 16);
+
+  sum = _mm_add_epi32(in0, in1);
+  in0 = _mm_unpacklo_epi32(sum, u0);
+  in1 = _mm_unpackhi_epi32(sum, u0);
+
+  sum = _mm_add_epi32(in0, in1);
+  in0 = _mm_srli_si128(sum, 8);
+
+  in1 = _mm_add_epi32(sum, in0);
+  in1 = _mm_srai_epi32(in1, 1);
+  _mm_store_si128((__m128i *)(output), in1);
+}
+
 void vp9_fdct16x16_sse2(const int16_t *input, int16_t *output, int stride) {
   // The 2D transform is done with two passes which are actually pretty
   // similar. In the first one, we transform the columns and transpose
   // the results. In the second one, we transform the rows. To achieve that,
   // as the first pass results are transposed, we transpose the columns (that
   // is the transposed rows) and transpose the results (so that it goes back
   // in normal/row positions).
   int pass;
   // We need an intermediate buffer between passes.
   DECLARE_ALIGNED_ARRAY(16, int16_t, intermediate, 256);
   const int16_t *in = input;
   int16_t *out = intermediate;
   // Constants
   //    When we use them, in one case, they are all the same. In all others
   //    it's a pair of them that we need to repeat four times. This is done
   //    by constructing the 32 bit constant corresponding to that pair.
   const __m128i k__cospi_p16_p16 = _mm_set1_epi16(cospi_16_64);
   const __m128i k__cospi_p16_m16 = pair_set_epi16(cospi_16_64, -cospi_16_64);
   const __m128i k__cospi_p24_p08 = pair_set_epi16(cospi_24_64, cospi_8_64);
-  const __m128i k__cospi_m24_m08 = pair_set_epi16(-cospi_24_64, -cospi_8_64);
+  const __m128i k__cospi_p08_m24 = pair_set_epi16(cospi_8_64, -cospi_24_64);
   const __m128i k__cospi_m08_p24 = pair_set_epi16(-cospi_8_64, cospi_24_64);
   const __m128i k__cospi_p28_p04 = pair_set_epi16(cospi_28_64, cospi_4_64);
   const __m128i k__cospi_m04_p28 = pair_set_epi16(-cospi_4_64, cospi_28_64);
   const __m128i k__cospi_p12_p20 = pair_set_epi16(cospi_12_64, cospi_20_64);
   const __m128i k__cospi_m20_p12 = pair_set_epi16(-cospi_20_64, cospi_12_64);
   const __m128i k__cospi_p30_p02 = pair_set_epi16(cospi_30_64, cospi_2_64);
   const __m128i k__cospi_p14_p18 = pair_set_epi16(cospi_14_64, cospi_18_64);
   const __m128i k__cospi_m02_p30 = pair_set_epi16(-cospi_2_64, cospi_30_64);
   const __m128i k__cospi_m18_p14 = pair_set_epi16(-cospi_18_64, cospi_14_64);
   const __m128i k__cospi_p22_p10 = pair_set_epi16(cospi_22_64, cospi_10_64);
@@ skipping 305 matching lines @@
           step3_7 = _mm_add_epi16(step1_7, step2_4);
         }
         // step 4
         {
           const __m128i t0 = _mm_unpacklo_epi16(step3_1, step3_6);
           const __m128i t1 = _mm_unpackhi_epi16(step3_1, step3_6);
           const __m128i t2 = _mm_unpacklo_epi16(step3_2, step3_5);
           const __m128i t3 = _mm_unpackhi_epi16(step3_2, step3_5);
           const __m128i u0 = _mm_madd_epi16(t0, k__cospi_m08_p24);
           const __m128i u1 = _mm_madd_epi16(t1, k__cospi_m08_p24);
-          const __m128i u2 = _mm_madd_epi16(t2, k__cospi_m24_m08);
-          const __m128i u3 = _mm_madd_epi16(t3, k__cospi_m24_m08);
+          const __m128i u2 = _mm_madd_epi16(t2, k__cospi_p24_p08);
+          const __m128i u3 = _mm_madd_epi16(t3, k__cospi_p24_p08);
           // dct_const_round_shift
           const __m128i v0 = _mm_add_epi32(u0, k__DCT_CONST_ROUNDING);
           const __m128i v1 = _mm_add_epi32(u1, k__DCT_CONST_ROUNDING);
           const __m128i v2 = _mm_add_epi32(u2, k__DCT_CONST_ROUNDING);
           const __m128i v3 = _mm_add_epi32(u3, k__DCT_CONST_ROUNDING);
           const __m128i w0 = _mm_srai_epi32(v0, DCT_CONST_BITS);
           const __m128i w1 = _mm_srai_epi32(v1, DCT_CONST_BITS);
           const __m128i w2 = _mm_srai_epi32(v2, DCT_CONST_BITS);
           const __m128i w3 = _mm_srai_epi32(v3, DCT_CONST_BITS);
           // Combine
           step2_1 = _mm_packs_epi32(w0, w1);
           step2_2 = _mm_packs_epi32(w2, w3);
         }
         {
           const __m128i t0 = _mm_unpacklo_epi16(step3_1, step3_6);
           const __m128i t1 = _mm_unpackhi_epi16(step3_1, step3_6);
           const __m128i t2 = _mm_unpacklo_epi16(step3_2, step3_5);
           const __m128i t3 = _mm_unpackhi_epi16(step3_2, step3_5);
           const __m128i u0 = _mm_madd_epi16(t0, k__cospi_p24_p08);
           const __m128i u1 = _mm_madd_epi16(t1, k__cospi_p24_p08);
-          const __m128i u2 = _mm_madd_epi16(t2, k__cospi_m08_p24);
-          const __m128i u3 = _mm_madd_epi16(t3, k__cospi_m08_p24);
+          const __m128i u2 = _mm_madd_epi16(t2, k__cospi_p08_m24);
+          const __m128i u3 = _mm_madd_epi16(t3, k__cospi_p08_m24);
           // dct_const_round_shift
           const __m128i v0 = _mm_add_epi32(u0, k__DCT_CONST_ROUNDING);
           const __m128i v1 = _mm_add_epi32(u1, k__DCT_CONST_ROUNDING);
           const __m128i v2 = _mm_add_epi32(u2, k__DCT_CONST_ROUNDING);
           const __m128i v3 = _mm_add_epi32(u3, k__DCT_CONST_ROUNDING);
           const __m128i w0 = _mm_srai_epi32(v0, DCT_CONST_BITS);
           const __m128i w1 = _mm_srai_epi32(v1, DCT_CONST_BITS);
           const __m128i w2 = _mm_srai_epi32(v2, DCT_CONST_BITS);
           const __m128i w3 = _mm_srai_epi32(v3, DCT_CONST_BITS);
           // Combine
           step2_6 = _mm_packs_epi32(w0, w1);
           step2_5 = _mm_packs_epi32(w2, w3);
         }
         // step 5
         {
           step1_0 = _mm_add_epi16(step3_0, step2_1);
           step1_1 = _mm_sub_epi16(step3_0, step2_1);
-          step1_2 = _mm_sub_epi16(step3_3, step2_2);
-          step1_3 = _mm_add_epi16(step3_3, step2_2);
-          step1_4 = _mm_add_epi16(step3_4, step2_5);
-          step1_5 = _mm_sub_epi16(step3_4, step2_5);
+          step1_2 = _mm_add_epi16(step3_3, step2_2);
+          step1_3 = _mm_sub_epi16(step3_3, step2_2);
+          step1_4 = _mm_sub_epi16(step3_4, step2_5);
+          step1_5 = _mm_add_epi16(step3_4, step2_5);
           step1_6 = _mm_sub_epi16(step3_7, step2_6);
           step1_7 = _mm_add_epi16(step3_7, step2_6);
         }
         // step 6
         {
           const __m128i t0 = _mm_unpacklo_epi16(step1_0, step1_7);
           const __m128i t1 = _mm_unpackhi_epi16(step1_0, step1_7);
           const __m128i t2 = _mm_unpacklo_epi16(step1_1, step1_6);
           const __m128i t3 = _mm_unpackhi_epi16(step1_1, step1_6);
           const __m128i u0 = _mm_madd_epi16(t0, k__cospi_p30_p02);
@@ skipping 270 matching lines @@
   right_shift_8x8(res1 + 8, 2);
 }
 
 void fdct16_8col(__m128i *in) {
   // perform 16x16 1-D DCT for 8 columns
   __m128i i[8], s[8], p[8], t[8], u[16], v[16];
   const __m128i k__cospi_p16_p16 = _mm_set1_epi16(cospi_16_64);
   const __m128i k__cospi_p16_m16 = pair_set_epi16(cospi_16_64, -cospi_16_64);
   const __m128i k__cospi_m16_p16 = pair_set_epi16(-cospi_16_64, cospi_16_64);
   const __m128i k__cospi_p24_p08 = pair_set_epi16(cospi_24_64, cospi_8_64);
-  const __m128i k__cospi_m24_m08 = pair_set_epi16(-cospi_24_64, -cospi_8_64);
+  const __m128i k__cospi_p08_m24 = pair_set_epi16(cospi_8_64, -cospi_24_64);
   const __m128i k__cospi_m08_p24 = pair_set_epi16(-cospi_8_64, cospi_24_64);
   const __m128i k__cospi_p28_p04 = pair_set_epi16(cospi_28_64, cospi_4_64);
   const __m128i k__cospi_m04_p28 = pair_set_epi16(-cospi_4_64, cospi_28_64);
   const __m128i k__cospi_p12_p20 = pair_set_epi16(cospi_12_64, cospi_20_64);
   const __m128i k__cospi_m20_p12 = pair_set_epi16(-cospi_20_64, cospi_12_64);
   const __m128i k__cospi_p30_p02 = pair_set_epi16(cospi_30_64, cospi_2_64);
   const __m128i k__cospi_p14_p18 = pair_set_epi16(cospi_14_64, cospi_18_64);
   const __m128i k__cospi_m02_p30 = pair_set_epi16(-cospi_2_64, cospi_30_64);
   const __m128i k__cospi_m18_p14 = pair_set_epi16(-cospi_18_64, cospi_14_64);
   const __m128i k__cospi_p22_p10 = pair_set_epi16(cospi_22_64, cospi_10_64);
@@ skipping 183 matching lines @@
   p[7] = _mm_add_epi16(s[7], t[4]);
 
   // stage 4
   u[0] = _mm_unpacklo_epi16(p[1], p[6]);
   u[1] = _mm_unpackhi_epi16(p[1], p[6]);
   u[2] = _mm_unpacklo_epi16(p[2], p[5]);
   u[3] = _mm_unpackhi_epi16(p[2], p[5]);
 
   v[0] = _mm_madd_epi16(u[0], k__cospi_m08_p24);
   v[1] = _mm_madd_epi16(u[1], k__cospi_m08_p24);
-  v[2] = _mm_madd_epi16(u[2], k__cospi_m24_m08);
-  v[3] = _mm_madd_epi16(u[3], k__cospi_m24_m08);
-  v[4] = _mm_madd_epi16(u[2], k__cospi_m08_p24);
-  v[5] = _mm_madd_epi16(u[3], k__cospi_m08_p24);
+  v[2] = _mm_madd_epi16(u[2], k__cospi_p24_p08);
+  v[3] = _mm_madd_epi16(u[3], k__cospi_p24_p08);
+  v[4] = _mm_madd_epi16(u[2], k__cospi_p08_m24);
+  v[5] = _mm_madd_epi16(u[3], k__cospi_p08_m24);
   v[6] = _mm_madd_epi16(u[0], k__cospi_p24_p08);
   v[7] = _mm_madd_epi16(u[1], k__cospi_p24_p08);
 
   u[0] = _mm_add_epi32(v[0], k__DCT_CONST_ROUNDING);
   u[1] = _mm_add_epi32(v[1], k__DCT_CONST_ROUNDING);
   u[2] = _mm_add_epi32(v[2], k__DCT_CONST_ROUNDING);
   u[3] = _mm_add_epi32(v[3], k__DCT_CONST_ROUNDING);
   u[4] = _mm_add_epi32(v[4], k__DCT_CONST_ROUNDING);
   u[5] = _mm_add_epi32(v[5], k__DCT_CONST_ROUNDING);
   u[6] = _mm_add_epi32(v[6], k__DCT_CONST_ROUNDING);
   u[7] = _mm_add_epi32(v[7], k__DCT_CONST_ROUNDING);
 
   v[0] = _mm_srai_epi32(u[0], DCT_CONST_BITS);
   v[1] = _mm_srai_epi32(u[1], DCT_CONST_BITS);
   v[2] = _mm_srai_epi32(u[2], DCT_CONST_BITS);
   v[3] = _mm_srai_epi32(u[3], DCT_CONST_BITS);
   v[4] = _mm_srai_epi32(u[4], DCT_CONST_BITS);
   v[5] = _mm_srai_epi32(u[5], DCT_CONST_BITS);
   v[6] = _mm_srai_epi32(u[6], DCT_CONST_BITS);
   v[7] = _mm_srai_epi32(u[7], DCT_CONST_BITS);
 
   t[1] = _mm_packs_epi32(v[0], v[1]);
   t[2] = _mm_packs_epi32(v[2], v[3]);
   t[5] = _mm_packs_epi32(v[4], v[5]);
   t[6] = _mm_packs_epi32(v[6], v[7]);
 
   // stage 5
   s[0] = _mm_add_epi16(p[0], t[1]);
   s[1] = _mm_sub_epi16(p[0], t[1]);
-  s[2] = _mm_sub_epi16(p[3], t[2]);
-  s[3] = _mm_add_epi16(p[3], t[2]);
-  s[4] = _mm_add_epi16(p[4], t[5]);
-  s[5] = _mm_sub_epi16(p[4], t[5]);
+  s[2] = _mm_add_epi16(p[3], t[2]);
+  s[3] = _mm_sub_epi16(p[3], t[2]);
+  s[4] = _mm_sub_epi16(p[4], t[5]);
+  s[5] = _mm_add_epi16(p[4], t[5]);
   s[6] = _mm_sub_epi16(p[7], t[6]);
   s[7] = _mm_add_epi16(p[7], t[6]);
 
   // stage 6
   u[0] = _mm_unpacklo_epi16(s[0], s[7]);
   u[1] = _mm_unpackhi_epi16(s[0], s[7]);
   u[2] = _mm_unpacklo_epi16(s[1], s[6]);
   u[3] = _mm_unpackhi_epi16(s[1], s[6]);
   u[4] = _mm_unpacklo_epi16(s[2], s[5]);
   u[5] = _mm_unpackhi_epi16(s[2], s[5]);
@@ skipping 571 matching lines @@
       right_shift_16x16(in0, in1);
       fadst16_sse2(in0, in1);
       write_buffer_16x16(output, in0, in1, 16);
       break;
     default:
       assert(0);
       break;
   }
 }
 
+void vp9_fdct32x32_1_sse2(const int16_t *input, int16_t *output, int stride) {
+  __m128i in0, in1, in2, in3;
+  __m128i u0, u1;
+  __m128i sum = _mm_setzero_si128();
+  int i;
+
+  for (i = 0; i < 8; ++i) {
+    in0  = _mm_load_si128((const __m128i *)(input +  0));
+    in1  = _mm_load_si128((const __m128i *)(input +  8));
+    in2  = _mm_load_si128((const __m128i *)(input + 16));
+    in3  = _mm_load_si128((const __m128i *)(input + 24));
+
+    input += stride;
+    u0 = _mm_add_epi16(in0, in1);
+    u1 = _mm_add_epi16(in2, in3);
+    sum = _mm_add_epi16(sum, u0);
+
+    in0  = _mm_load_si128((const __m128i *)(input +  0));
+    in1  = _mm_load_si128((const __m128i *)(input +  8));
+    in2  = _mm_load_si128((const __m128i *)(input + 16));
+    in3  = _mm_load_si128((const __m128i *)(input + 24));
+
+    input += stride;
+    sum = _mm_add_epi16(sum, u1);
+    u0  = _mm_add_epi16(in0, in1);
+    u1  = _mm_add_epi16(in2, in3);
+    sum = _mm_add_epi16(sum, u0);
+
+    in0  = _mm_load_si128((const __m128i *)(input +  0));
+    in1  = _mm_load_si128((const __m128i *)(input +  8));
+    in2  = _mm_load_si128((const __m128i *)(input + 16));
+    in3  = _mm_load_si128((const __m128i *)(input + 24));
+
+    input += stride;
+    sum = _mm_add_epi16(sum, u1);
+    u0  = _mm_add_epi16(in0, in1);
+    u1  = _mm_add_epi16(in2, in3);
+    sum = _mm_add_epi16(sum, u0);
+
+    in0  = _mm_load_si128((const __m128i *)(input +  0));
+    in1  = _mm_load_si128((const __m128i *)(input +  8));
+    in2  = _mm_load_si128((const __m128i *)(input + 16));
+    in3  = _mm_load_si128((const __m128i *)(input + 24));
+
+    input += stride;
+    sum = _mm_add_epi16(sum, u1);
+    u0  = _mm_add_epi16(in0, in1);
+    u1  = _mm_add_epi16(in2, in3);
+    sum = _mm_add_epi16(sum, u0);
+
+    sum = _mm_add_epi16(sum, u1);
+  }
+
+  u0  = _mm_setzero_si128();
+  in0 = _mm_unpacklo_epi16(u0, sum);
+  in1 = _mm_unpackhi_epi16(u0, sum);
+  in0 = _mm_srai_epi32(in0, 16);
+  in1 = _mm_srai_epi32(in1, 16);
+
+  sum = _mm_add_epi32(in0, in1);
+  in0 = _mm_unpacklo_epi32(sum, u0);
+  in1 = _mm_unpackhi_epi32(sum, u0);
+
+  sum = _mm_add_epi32(in0, in1);
+  in0 = _mm_srli_si128(sum, 8);
+
+  in1 = _mm_add_epi32(sum, in0);
+  in1 = _mm_srai_epi32(in1, 3);
+  _mm_store_si128((__m128i *)(output), in1);
+}
+
 #define FDCT32x32_2D vp9_fdct32x32_rd_sse2
 #define FDCT32x32_HIGH_PRECISION 0
 #include "vp9/encoder/x86/vp9_dct32x32_sse2.c"
 #undef  FDCT32x32_2D
 #undef  FDCT32x32_HIGH_PRECISION
 
 #define FDCT32x32_2D vp9_fdct32x32_sse2
 #define FDCT32x32_HIGH_PRECISION 1
 #include "vp9/encoder/x86/vp9_dct32x32_sse2.c" // NOLINT
 #undef  FDCT32x32_2D
 #undef  FDCT32x32_HIGH_PRECISION