libvpx: Pull from upstream

source/libvpx/vp9/encoder/x86/vp9_dct32x32_sse2.c

 /*
  *  Copyright (c) 2012 The WebM project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
 #include <emmintrin.h>  // SSE2
 #include "vp9/common/vp9_idct.h"  // for cospi constants
 #include "vpx_ports/mem.h"
 
 #if FDCT32x32_HIGH_PRECISION
 static INLINE __m128i k_madd_epi32(__m128i a, __m128i b) {
   __m128i buf0, buf1;
   buf0 = _mm_mul_epu32(a, b);
   a = _mm_srli_epi64(a, 32);
   b = _mm_srli_epi64(b, 32);
   buf1 = _mm_mul_epu32(a, b);
   return _mm_add_epi64(buf0, buf1);
 }
 
 static INLINE __m128i k_packs_epi64(__m128i a, __m128i b) {
   __m128i buf0 = _mm_shuffle_epi32(a, _MM_SHUFFLE(0, 0, 2, 0));
   __m128i buf1 = _mm_shuffle_epi32(b, _MM_SHUFFLE(0, 0, 2, 0));
   return _mm_unpacklo_epi64(buf0, buf1);
 }
-
-static INLINE __m128i k_cvtlo_epi16(__m128i a, __m128i mask16, __m128i kZero) {
-  // convert the lower 4 signed 16-bit integers into 4 signed 32-bit integers
-  __m128i sign_bit = _mm_and_si128(a, mask16);
-  __m128i b = _mm_unpacklo_epi16(a, kZero);
-  sign_bit = _mm_cmplt_epi16(sign_bit, kZero);
-  sign_bit = _mm_unpacklo_epi16(kZero, sign_bit);
-  return _mm_or_si128(sign_bit, b);
-}
-
-static INLINE __m128i k_cvthi_epi16(__m128i a, __m128i mask16, __m128i kZero) {
-  // convert the lower 4 signed 16-bit integers into 4 signed 32-bit integers
-  __m128i sign_bit = _mm_and_si128(a, mask16);
-  __m128i b = _mm_unpackhi_epi16(a, kZero);
-  sign_bit = _mm_cmplt_epi16(sign_bit, kZero);
-  sign_bit = _mm_unpackhi_epi16(kZero, sign_bit);
-  return _mm_or_si128(sign_bit, b);
-}
 #endif
 
-void FDCT32x32_2D(int16_t *input,
-                  int16_t *output_org, int pitch) {
+void FDCT32x32_2D(const int16_t *input,
+                  int16_t *output_org, int stride) {
   // Calculate pre-multiplied strides
-  const int str1 = pitch >> 1;
-  const int str2 = pitch;
-  const int str3 = pitch + str1;
+  const int str1 = stride;
+  const int str2 = 2 * stride;
+  const int str3 = 2 * stride + str1;
   // We need an intermediate buffer between passes.
   DECLARE_ALIGNED(16, int16_t, intermediate[32 * 32]);
   // Constants
   //    When we use them, in one case, they are all the same. In all others
   //    it's a pair of them that we need to repeat four times. This is done
   //    by constructing the 32 bit constant corresponding to that pair.
   const __m128i k__cospi_p16_p16 = _mm_set1_epi16(+cospi_16_64);
   const __m128i k__cospi_p16_m16 = pair_set_epi16(+cospi_16_64, -cospi_16_64);
   const __m128i k__cospi_m08_p24 = pair_set_epi16(-cospi_8_64,   cospi_24_64);
   const __m128i k__cospi_m24_m08 = pair_set_epi16(-cospi_24_64, -cospi_8_64);
@@ skipping 38 matching lines @@
     int column_start;
     for (column_start = 0; column_start < 32; column_start += 8) {
       __m128i step1[32];
       __m128i step2[32];
       __m128i step3[32];
       __m128i out[32];
       // Stage 1
       // Note: even though all the loads below are aligned, using the aligned
       //       intrinsic make the code slightly slower.
       if (0 == pass) {
-        int16_t *in  = &input[column_start];
+        const int16_t *in  = &input[column_start];
         // step1[i] =  (in[ 0 * stride] + in[(32 -  1) * stride]) << 2;
         // Note: the next four blocks could be in a loop. That would help the
         //       instruction cache but is actually slower.
         {
-          int16_t *ina =  in +  0 * str1;
-          int16_t *inb =  in + 31 * str1;
+          const int16_t *ina =  in +  0 * str1;
+          const int16_t *inb =  in + 31 * str1;
           __m128i *step1a = &step1[ 0];
           __m128i *step1b = &step1[31];
           const __m128i ina0  = _mm_loadu_si128((const __m128i *)(ina));
           const __m128i ina1  = _mm_loadu_si128((const __m128i *)(ina + str1));
           const __m128i ina2  = _mm_loadu_si128((const __m128i *)(ina + str2));
           const __m128i ina3  = _mm_loadu_si128((const __m128i *)(ina + str3));
           const __m128i inb3  = _mm_loadu_si128((const __m128i *)(inb - str3));
           const __m128i inb2  = _mm_loadu_si128((const __m128i *)(inb - str2));
           const __m128i inb1  = _mm_loadu_si128((const __m128i *)(inb - str1));
           const __m128i inb0  = _mm_loadu_si128((const __m128i *)(inb));
           step1a[ 0] = _mm_add_epi16(ina0, inb0);
           step1a[ 1] = _mm_add_epi16(ina1, inb1);
           step1a[ 2] = _mm_add_epi16(ina2, inb2);
           step1a[ 3] = _mm_add_epi16(ina3, inb3);
           step1b[-3] = _mm_sub_epi16(ina3, inb3);
           step1b[-2] = _mm_sub_epi16(ina2, inb2);
           step1b[-1] = _mm_sub_epi16(ina1, inb1);
           step1b[-0] = _mm_sub_epi16(ina0, inb0);
           step1a[ 0] = _mm_slli_epi16(step1a[ 0], 2);
           step1a[ 1] = _mm_slli_epi16(step1a[ 1], 2);
           step1a[ 2] = _mm_slli_epi16(step1a[ 2], 2);
           step1a[ 3] = _mm_slli_epi16(step1a[ 3], 2);
           step1b[-3] = _mm_slli_epi16(step1b[-3], 2);
           step1b[-2] = _mm_slli_epi16(step1b[-2], 2);
           step1b[-1] = _mm_slli_epi16(step1b[-1], 2);
           step1b[-0] = _mm_slli_epi16(step1b[-0], 2);
         }
         {
-          int16_t *ina =  in +  4 * str1;
-          int16_t *inb =  in + 27 * str1;
+          const int16_t *ina =  in +  4 * str1;
+          const int16_t *inb =  in + 27 * str1;
           __m128i *step1a = &step1[ 4];
           __m128i *step1b = &step1[27];
           const __m128i ina0  = _mm_loadu_si128((const __m128i *)(ina));
           const __m128i ina1  = _mm_loadu_si128((const __m128i *)(ina + str1));
           const __m128i ina2  = _mm_loadu_si128((const __m128i *)(ina + str2));
           const __m128i ina3  = _mm_loadu_si128((const __m128i *)(ina + str3));
           const __m128i inb3  = _mm_loadu_si128((const __m128i *)(inb - str3));
           const __m128i inb2  = _mm_loadu_si128((const __m128i *)(inb - str2));
           const __m128i inb1  = _mm_loadu_si128((const __m128i *)(inb - str1));
           const __m128i inb0  = _mm_loadu_si128((const __m128i *)(inb));
           step1a[ 0] = _mm_add_epi16(ina0, inb0);
           step1a[ 1] = _mm_add_epi16(ina1, inb1);
           step1a[ 2] = _mm_add_epi16(ina2, inb2);
           step1a[ 3] = _mm_add_epi16(ina3, inb3);
           step1b[-3] = _mm_sub_epi16(ina3, inb3);
           step1b[-2] = _mm_sub_epi16(ina2, inb2);
           step1b[-1] = _mm_sub_epi16(ina1, inb1);
           step1b[-0] = _mm_sub_epi16(ina0, inb0);
           step1a[ 0] = _mm_slli_epi16(step1a[ 0], 2);
           step1a[ 1] = _mm_slli_epi16(step1a[ 1], 2);
           step1a[ 2] = _mm_slli_epi16(step1a[ 2], 2);
           step1a[ 3] = _mm_slli_epi16(step1a[ 3], 2);
           step1b[-3] = _mm_slli_epi16(step1b[-3], 2);
           step1b[-2] = _mm_slli_epi16(step1b[-2], 2);
           step1b[-1] = _mm_slli_epi16(step1b[-1], 2);
           step1b[-0] = _mm_slli_epi16(step1b[-0], 2);
         }
         {
-          int16_t *ina =  in +  8 * str1;
-          int16_t *inb =  in + 23 * str1;
+          const int16_t *ina =  in +  8 * str1;
+          const int16_t *inb =  in + 23 * str1;
           __m128i *step1a = &step1[ 8];
           __m128i *step1b = &step1[23];
           const __m128i ina0  = _mm_loadu_si128((const __m128i *)(ina));
           const __m128i ina1  = _mm_loadu_si128((const __m128i *)(ina + str1));
           const __m128i ina2  = _mm_loadu_si128((const __m128i *)(ina + str2));
           const __m128i ina3  = _mm_loadu_si128((const __m128i *)(ina + str3));
           const __m128i inb3  = _mm_loadu_si128((const __m128i *)(inb - str3));
           const __m128i inb2  = _mm_loadu_si128((const __m128i *)(inb - str2));
           const __m128i inb1  = _mm_loadu_si128((const __m128i *)(inb - str1));
           const __m128i inb0  = _mm_loadu_si128((const __m128i *)(inb));
           step1a[ 0] = _mm_add_epi16(ina0, inb0);
           step1a[ 1] = _mm_add_epi16(ina1, inb1);
           step1a[ 2] = _mm_add_epi16(ina2, inb2);
           step1a[ 3] = _mm_add_epi16(ina3, inb3);
           step1b[-3] = _mm_sub_epi16(ina3, inb3);
           step1b[-2] = _mm_sub_epi16(ina2, inb2);
           step1b[-1] = _mm_sub_epi16(ina1, inb1);
           step1b[-0] = _mm_sub_epi16(ina0, inb0);
           step1a[ 0] = _mm_slli_epi16(step1a[ 0], 2);
           step1a[ 1] = _mm_slli_epi16(step1a[ 1], 2);
           step1a[ 2] = _mm_slli_epi16(step1a[ 2], 2);
           step1a[ 3] = _mm_slli_epi16(step1a[ 3], 2);
           step1b[-3] = _mm_slli_epi16(step1b[-3], 2);
           step1b[-2] = _mm_slli_epi16(step1b[-2], 2);
           step1b[-1] = _mm_slli_epi16(step1b[-1], 2);
           step1b[-0] = _mm_slli_epi16(step1b[-0], 2);
         }
         {
-          int16_t *ina =  in + 12 * str1;
-          int16_t *inb =  in + 19 * str1;
+          const int16_t *ina =  in + 12 * str1;
+          const int16_t *inb =  in + 19 * str1;
           __m128i *step1a = &step1[12];
           __m128i *step1b = &step1[19];
           const __m128i ina0  = _mm_loadu_si128((const __m128i *)(ina));
           const __m128i ina1  = _mm_loadu_si128((const __m128i *)(ina + str1));
           const __m128i ina2  = _mm_loadu_si128((const __m128i *)(ina + str2));
           const __m128i ina3  = _mm_loadu_si128((const __m128i *)(ina + str3));
           const __m128i inb3  = _mm_loadu_si128((const __m128i *)(inb - str3));
           const __m128i inb2  = _mm_loadu_si128((const __m128i *)(inb - str2));
           const __m128i inb1  = _mm_loadu_si128((const __m128i *)(inb - str1));
           const __m128i inb0  = _mm_loadu_si128((const __m128i *)(inb));
@@ skipping 931 matching lines @@
         out[29] = _mm_packs_epi32(out_29_6, out_29_7);
         out[ 3] = _mm_packs_epi32(out_03_6, out_03_7);
         out[19] = _mm_packs_epi32(out_19_6, out_19_7);
         out[11] = _mm_packs_epi32(out_11_6, out_11_7);
         out[27] = _mm_packs_epi32(out_27_6, out_27_7);
       }
 #if FDCT32x32_HIGH_PRECISION
       } else {
         __m128i lstep1[64], lstep2[64], lstep3[64];
         __m128i u[32], v[32], sign[16];
-        const __m128i mask16 = _mm_set1_epi32(0x80008000);
         const __m128i K32One = _mm_set_epi32(1, 1, 1, 1);
         // start using 32-bit operations
         // stage 3
         {
           // expanding to 32-bit length priori to addition operations
-          lstep2[ 0] = k_cvtlo_epi16(step2[ 0], mask16, kZero);
-          lstep2[ 1] = k_cvthi_epi16(step2[ 0], mask16, kZero);
-          lstep2[ 2] = k_cvtlo_epi16(step2[ 1], mask16, kZero);
-          lstep2[ 3] = k_cvthi_epi16(step2[ 1], mask16, kZero);
-          lstep2[ 4] = k_cvtlo_epi16(step2[ 2], mask16, kZero);
-          lstep2[ 5] = k_cvthi_epi16(step2[ 2], mask16, kZero);
-          lstep2[ 6] = k_cvtlo_epi16(step2[ 3], mask16, kZero);
-          lstep2[ 7] = k_cvthi_epi16(step2[ 3], mask16, kZero);
-          lstep2[ 8] = k_cvtlo_epi16(step2[ 4], mask16, kZero);
-          lstep2[ 9] = k_cvthi_epi16(step2[ 4], mask16, kZero);
-          lstep2[10] = k_cvtlo_epi16(step2[ 5], mask16, kZero);
-          lstep2[11] = k_cvthi_epi16(step2[ 5], mask16, kZero);
-          lstep2[12] = k_cvtlo_epi16(step2[ 6], mask16, kZero);
-          lstep2[13] = k_cvthi_epi16(step2[ 6], mask16, kZero);
-          lstep2[14] = k_cvtlo_epi16(step2[ 7], mask16, kZero);
-          lstep2[15] = k_cvthi_epi16(step2[ 7], mask16, kZero);
+          lstep2[ 0] = _mm_unpacklo_epi16(step2[ 0], kZero);
+          lstep2[ 1] = _mm_unpackhi_epi16(step2[ 0], kZero);
+          lstep2[ 2] = _mm_unpacklo_epi16(step2[ 1], kZero);
+          lstep2[ 3] = _mm_unpackhi_epi16(step2[ 1], kZero);
+          lstep2[ 4] = _mm_unpacklo_epi16(step2[ 2], kZero);
+          lstep2[ 5] = _mm_unpackhi_epi16(step2[ 2], kZero);
+          lstep2[ 6] = _mm_unpacklo_epi16(step2[ 3], kZero);
+          lstep2[ 7] = _mm_unpackhi_epi16(step2[ 3], kZero);
+          lstep2[ 8] = _mm_unpacklo_epi16(step2[ 4], kZero);
+          lstep2[ 9] = _mm_unpackhi_epi16(step2[ 4], kZero);
+          lstep2[10] = _mm_unpacklo_epi16(step2[ 5], kZero);
+          lstep2[11] = _mm_unpackhi_epi16(step2[ 5], kZero);
+          lstep2[12] = _mm_unpacklo_epi16(step2[ 6], kZero);
+          lstep2[13] = _mm_unpackhi_epi16(step2[ 6], kZero);
+          lstep2[14] = _mm_unpacklo_epi16(step2[ 7], kZero);
+          lstep2[15] = _mm_unpackhi_epi16(step2[ 7], kZero);
+          lstep2[ 0] = _mm_madd_epi16(lstep2[ 0], kOne);
+          lstep2[ 1] = _mm_madd_epi16(lstep2[ 1], kOne);
+          lstep2[ 2] = _mm_madd_epi16(lstep2[ 2], kOne);
+          lstep2[ 3] = _mm_madd_epi16(lstep2[ 3], kOne);
+          lstep2[ 4] = _mm_madd_epi16(lstep2[ 4], kOne);
+          lstep2[ 5] = _mm_madd_epi16(lstep2[ 5], kOne);
+          lstep2[ 6] = _mm_madd_epi16(lstep2[ 6], kOne);
+          lstep2[ 7] = _mm_madd_epi16(lstep2[ 7], kOne);
+          lstep2[ 8] = _mm_madd_epi16(lstep2[ 8], kOne);
+          lstep2[ 9] = _mm_madd_epi16(lstep2[ 9], kOne);
+          lstep2[10] = _mm_madd_epi16(lstep2[10], kOne);
+          lstep2[11] = _mm_madd_epi16(lstep2[11], kOne);
+          lstep2[12] = _mm_madd_epi16(lstep2[12], kOne);
+          lstep2[13] = _mm_madd_epi16(lstep2[13], kOne);
+          lstep2[14] = _mm_madd_epi16(lstep2[14], kOne);
+          lstep2[15] = _mm_madd_epi16(lstep2[15], kOne);
 
           lstep3[ 0] = _mm_add_epi32(lstep2[14], lstep2[ 0]);
           lstep3[ 1] = _mm_add_epi32(lstep2[15], lstep2[ 1]);
           lstep3[ 2] = _mm_add_epi32(lstep2[12], lstep2[ 2]);
           lstep3[ 3] = _mm_add_epi32(lstep2[13], lstep2[ 3]);
           lstep3[ 4] = _mm_add_epi32(lstep2[10], lstep2[ 4]);
           lstep3[ 5] = _mm_add_epi32(lstep2[11], lstep2[ 5]);
           lstep3[ 6] = _mm_add_epi32(lstep2[ 8], lstep2[ 6]);
           lstep3[ 7] = _mm_add_epi32(lstep2[ 9], lstep2[ 7]);
           lstep3[ 8] = _mm_sub_epi32(lstep2[ 6], lstep2[ 8]);
@@ skipping 30 matching lines @@
           lstep3[20] = _mm_srai_epi32(s3_10_4, DCT_CONST_BITS);
           lstep3[21] = _mm_srai_epi32(s3_10_5, DCT_CONST_BITS);
           lstep3[22] = _mm_srai_epi32(s3_11_4, DCT_CONST_BITS);
           lstep3[23] = _mm_srai_epi32(s3_11_5, DCT_CONST_BITS);
           lstep3[24] = _mm_srai_epi32(s3_12_4, DCT_CONST_BITS);
           lstep3[25] = _mm_srai_epi32(s3_12_5, DCT_CONST_BITS);
           lstep3[26] = _mm_srai_epi32(s3_13_4, DCT_CONST_BITS);
           lstep3[27] = _mm_srai_epi32(s3_13_5, DCT_CONST_BITS);
         }
         {
-          lstep2[40] = k_cvtlo_epi16(step2[20], mask16, kZero);
-          lstep2[41] = k_cvthi_epi16(step2[20], mask16, kZero);
-          lstep2[42] = k_cvtlo_epi16(step2[21], mask16, kZero);
-          lstep2[43] = k_cvthi_epi16(step2[21], mask16, kZero);
-          lstep2[44] = k_cvtlo_epi16(step2[22], mask16, kZero);
-          lstep2[45] = k_cvthi_epi16(step2[22], mask16, kZero);
-          lstep2[46] = k_cvtlo_epi16(step2[23], mask16, kZero);
-          lstep2[47] = k_cvthi_epi16(step2[23], mask16, kZero);
-          lstep2[48] = k_cvtlo_epi16(step2[24], mask16, kZero);
-          lstep2[49] = k_cvthi_epi16(step2[24], mask16, kZero);
-          lstep2[50] = k_cvtlo_epi16(step2[25], mask16, kZero);
-          lstep2[51] = k_cvthi_epi16(step2[25], mask16, kZero);
-          lstep2[52] = k_cvtlo_epi16(step2[26], mask16, kZero);
-          lstep2[53] = k_cvthi_epi16(step2[26], mask16, kZero);
-          lstep2[54] = k_cvtlo_epi16(step2[27], mask16, kZero);
-          lstep2[55] = k_cvthi_epi16(step2[27], mask16, kZero);
+          lstep2[40] = _mm_unpacklo_epi16(step2[20], kZero);
+          lstep2[41] = _mm_unpackhi_epi16(step2[20], kZero);
+          lstep2[42] = _mm_unpacklo_epi16(step2[21], kZero);
+          lstep2[43] = _mm_unpackhi_epi16(step2[21], kZero);
+          lstep2[44] = _mm_unpacklo_epi16(step2[22], kZero);
+          lstep2[45] = _mm_unpackhi_epi16(step2[22], kZero);
+          lstep2[46] = _mm_unpacklo_epi16(step2[23], kZero);
+          lstep2[47] = _mm_unpackhi_epi16(step2[23], kZero);
+          lstep2[48] = _mm_unpacklo_epi16(step2[24], kZero);
+          lstep2[49] = _mm_unpackhi_epi16(step2[24], kZero);
+          lstep2[50] = _mm_unpacklo_epi16(step2[25], kZero);
+          lstep2[51] = _mm_unpackhi_epi16(step2[25], kZero);
+          lstep2[52] = _mm_unpacklo_epi16(step2[26], kZero);
+          lstep2[53] = _mm_unpackhi_epi16(step2[26], kZero);
+          lstep2[54] = _mm_unpacklo_epi16(step2[27], kZero);
+          lstep2[55] = _mm_unpackhi_epi16(step2[27], kZero);
+          lstep2[40] = _mm_madd_epi16(lstep2[40], kOne);
+          lstep2[41] = _mm_madd_epi16(lstep2[41], kOne);
+          lstep2[42] = _mm_madd_epi16(lstep2[42], kOne);
+          lstep2[43] = _mm_madd_epi16(lstep2[43], kOne);
+          lstep2[44] = _mm_madd_epi16(lstep2[44], kOne);
+          lstep2[45] = _mm_madd_epi16(lstep2[45], kOne);
+          lstep2[46] = _mm_madd_epi16(lstep2[46], kOne);
+          lstep2[47] = _mm_madd_epi16(lstep2[47], kOne);
+          lstep2[48] = _mm_madd_epi16(lstep2[48], kOne);
+          lstep2[49] = _mm_madd_epi16(lstep2[49], kOne);
+          lstep2[50] = _mm_madd_epi16(lstep2[50], kOne);
+          lstep2[51] = _mm_madd_epi16(lstep2[51], kOne);
+          lstep2[52] = _mm_madd_epi16(lstep2[52], kOne);
+          lstep2[53] = _mm_madd_epi16(lstep2[53], kOne);
+          lstep2[54] = _mm_madd_epi16(lstep2[54], kOne);
+          lstep2[55] = _mm_madd_epi16(lstep2[55], kOne);
 
-          lstep1[32] = k_cvtlo_epi16(step1[16], mask16, kZero);
-          lstep1[33] = k_cvthi_epi16(step1[16], mask16, kZero);
-          lstep1[34] = k_cvtlo_epi16(step1[17], mask16, kZero);
-          lstep1[35] = k_cvthi_epi16(step1[17], mask16, kZero);
-          lstep1[36] = k_cvtlo_epi16(step1[18], mask16, kZero);
-          lstep1[37] = k_cvthi_epi16(step1[18], mask16, kZero);
-          lstep1[38] = k_cvtlo_epi16(step1[19], mask16, kZero);
-          lstep1[39] = k_cvthi_epi16(step1[19], mask16, kZero);
-          lstep1[56] = k_cvtlo_epi16(step1[28], mask16, kZero);
-          lstep1[57] = k_cvthi_epi16(step1[28], mask16, kZero);
-          lstep1[58] = k_cvtlo_epi16(step1[29], mask16, kZero);
-          lstep1[59] = k_cvthi_epi16(step1[29], mask16, kZero);
-          lstep1[60] = k_cvtlo_epi16(step1[30], mask16, kZero);
-          lstep1[61] = k_cvthi_epi16(step1[30], mask16, kZero);
-          lstep1[62] = k_cvtlo_epi16(step1[31], mask16, kZero);
-          lstep1[63] = k_cvthi_epi16(step1[31], mask16, kZero);
+          lstep1[32] = _mm_unpacklo_epi16(step1[16], kZero);
+          lstep1[33] = _mm_unpackhi_epi16(step1[16], kZero);
+          lstep1[34] = _mm_unpacklo_epi16(step1[17], kZero);
+          lstep1[35] = _mm_unpackhi_epi16(step1[17], kZero);
+          lstep1[36] = _mm_unpacklo_epi16(step1[18], kZero);
+          lstep1[37] = _mm_unpackhi_epi16(step1[18], kZero);
+          lstep1[38] = _mm_unpacklo_epi16(step1[19], kZero);
+          lstep1[39] = _mm_unpackhi_epi16(step1[19], kZero);
+          lstep1[56] = _mm_unpacklo_epi16(step1[28], kZero);
+          lstep1[57] = _mm_unpackhi_epi16(step1[28], kZero);
+          lstep1[58] = _mm_unpacklo_epi16(step1[29], kZero);
+          lstep1[59] = _mm_unpackhi_epi16(step1[29], kZero);
+          lstep1[60] = _mm_unpacklo_epi16(step1[30], kZero);
+          lstep1[61] = _mm_unpackhi_epi16(step1[30], kZero);
+          lstep1[62] = _mm_unpacklo_epi16(step1[31], kZero);
+          lstep1[63] = _mm_unpackhi_epi16(step1[31], kZero);
+          lstep1[32] = _mm_madd_epi16(lstep1[32], kOne);
+          lstep1[33] = _mm_madd_epi16(lstep1[33], kOne);
+          lstep1[34] = _mm_madd_epi16(lstep1[34], kOne);
+          lstep1[35] = _mm_madd_epi16(lstep1[35], kOne);
+          lstep1[36] = _mm_madd_epi16(lstep1[36], kOne);
+          lstep1[37] = _mm_madd_epi16(lstep1[37], kOne);
+          lstep1[38] = _mm_madd_epi16(lstep1[38], kOne);
+          lstep1[39] = _mm_madd_epi16(lstep1[39], kOne);
+          lstep1[56] = _mm_madd_epi16(lstep1[56], kOne);
+          lstep1[57] = _mm_madd_epi16(lstep1[57], kOne);
+          lstep1[58] = _mm_madd_epi16(lstep1[58], kOne);
+          lstep1[59] = _mm_madd_epi16(lstep1[59], kOne);
+          lstep1[60] = _mm_madd_epi16(lstep1[60], kOne);
+          lstep1[61] = _mm_madd_epi16(lstep1[61], kOne);
+          lstep1[62] = _mm_madd_epi16(lstep1[62], kOne);
+          lstep1[63] = _mm_madd_epi16(lstep1[63], kOne);
 
           lstep3[32] = _mm_add_epi32(lstep2[46], lstep1[32]);
           lstep3[33] = _mm_add_epi32(lstep2[47], lstep1[33]);
+
           lstep3[34] = _mm_add_epi32(lstep2[44], lstep1[34]);
           lstep3[35] = _mm_add_epi32(lstep2[45], lstep1[35]);
           lstep3[36] = _mm_add_epi32(lstep2[42], lstep1[36]);
           lstep3[37] = _mm_add_epi32(lstep2[43], lstep1[37]);
           lstep3[38] = _mm_add_epi32(lstep2[40], lstep1[38]);
           lstep3[39] = _mm_add_epi32(lstep2[41], lstep1[39]);
           lstep3[40] = _mm_sub_epi32(lstep1[38], lstep2[40]);
           lstep3[41] = _mm_sub_epi32(lstep1[39], lstep2[41]);
           lstep3[42] = _mm_sub_epi32(lstep1[36], lstep2[42]);
           lstep3[43] = _mm_sub_epi32(lstep1[37], lstep2[43]);
@@ skipping 15 matching lines @@
           lstep3[59] = _mm_add_epi32(lstep2[53], lstep1[59]);
           lstep3[60] = _mm_add_epi32(lstep2[50], lstep1[60]);
           lstep3[61] = _mm_add_epi32(lstep2[51], lstep1[61]);
           lstep3[62] = _mm_add_epi32(lstep2[48], lstep1[62]);
           lstep3[63] = _mm_add_epi32(lstep2[49], lstep1[63]);
         }
 
         // stage 4
         {
           // expanding to 32-bit length priori to addition operations
-          lstep2[16] = k_cvtlo_epi16(step2[ 8], mask16, kZero);
-          lstep2[17] = k_cvthi_epi16(step2[ 8], mask16, kZero);
-          lstep2[18] = k_cvtlo_epi16(step2[ 9], mask16, kZero);
-          lstep2[19] = k_cvthi_epi16(step2[ 9], mask16, kZero);
-          lstep2[28] = k_cvtlo_epi16(step2[14], mask16, kZero);
-          lstep2[29] = k_cvthi_epi16(step2[14], mask16, kZero);
-          lstep2[30] = k_cvtlo_epi16(step2[15], mask16, kZero);
-          lstep2[31] = k_cvthi_epi16(step2[15], mask16, kZero);
+          lstep2[16] = _mm_unpacklo_epi16(step2[ 8], kZero);
+          lstep2[17] = _mm_unpackhi_epi16(step2[ 8], kZero);
+          lstep2[18] = _mm_unpacklo_epi16(step2[ 9], kZero);
+          lstep2[19] = _mm_unpackhi_epi16(step2[ 9], kZero);
+          lstep2[28] = _mm_unpacklo_epi16(step2[14], kZero);
+          lstep2[29] = _mm_unpackhi_epi16(step2[14], kZero);
+          lstep2[30] = _mm_unpacklo_epi16(step2[15], kZero);
+          lstep2[31] = _mm_unpackhi_epi16(step2[15], kZero);
+          lstep2[16] = _mm_madd_epi16(lstep2[16], kOne);
+          lstep2[17] = _mm_madd_epi16(lstep2[17], kOne);
+          lstep2[18] = _mm_madd_epi16(lstep2[18], kOne);
+          lstep2[19] = _mm_madd_epi16(lstep2[19], kOne);
+          lstep2[28] = _mm_madd_epi16(lstep2[28], kOne);
+          lstep2[29] = _mm_madd_epi16(lstep2[29], kOne);
+          lstep2[30] = _mm_madd_epi16(lstep2[30], kOne);
+          lstep2[31] = _mm_madd_epi16(lstep2[31], kOne);
 
           lstep1[ 0] = _mm_add_epi32(lstep3[ 6], lstep3[ 0]);
           lstep1[ 1] = _mm_add_epi32(lstep3[ 7], lstep3[ 1]);
           lstep1[ 2] = _mm_add_epi32(lstep3[ 4], lstep3[ 2]);
           lstep1[ 3] = _mm_add_epi32(lstep3[ 5], lstep3[ 3]);
           lstep1[ 4] = _mm_sub_epi32(lstep3[ 2], lstep3[ 4]);
           lstep1[ 5] = _mm_sub_epi32(lstep3[ 3], lstep3[ 5]);
           lstep1[ 6] = _mm_sub_epi32(lstep3[ 0], lstep3[ 6]);
           lstep1[ 7] = _mm_sub_epi32(lstep3[ 1], lstep3[ 7]);
           lstep1[16] = _mm_add_epi32(lstep3[22], lstep2[16]);
           lstep1[17] = _mm_add_epi32(lstep3[23], lstep2[17]);
           lstep1[18] = _mm_add_epi32(lstep3[20], lstep2[18]);
           lstep1[19] = _mm_add_epi32(lstep3[21], lstep2[19]);
           lstep1[20] = _mm_sub_epi32(lstep2[18], lstep3[20]);
           lstep1[21] = _mm_sub_epi32(lstep2[19], lstep3[21]);
           lstep1[22] = _mm_sub_epi32(lstep2[16], lstep3[22]);
           lstep1[23] = _mm_sub_epi32(lstep2[17], lstep3[23]);
           lstep1[24] = _mm_sub_epi32(lstep2[30], lstep3[24]);
           lstep1[25] = _mm_sub_epi32(lstep2[31], lstep3[25]);
           lstep1[26] = _mm_sub_epi32(lstep2[28], lstep3[26]);
           lstep1[27] = _mm_sub_epi32(lstep2[29], lstep3[27]);
           lstep1[28] = _mm_add_epi32(lstep3[26], lstep2[28]);
           lstep1[29] = _mm_add_epi32(lstep3[27], lstep2[29]);
           lstep1[30] = _mm_add_epi32(lstep3[24], lstep2[30]);
           lstep1[31] = _mm_add_epi32(lstep3[25], lstep2[31]);
         }
         {
-          // to be continued...
-          //
-          const __m128i k32_p16_p16 = pair_set_epi32(cospi_16_64, cospi_16_64);
-          const __m128i k32_p16_m16 = pair_set_epi32(cospi_16_64, -cospi_16_64);
+        // to be continued...
+        //
+        const __m128i k32_p16_p16 = pair_set_epi32(cospi_16_64, cospi_16_64);
+        const __m128i k32_p16_m16 = pair_set_epi32(cospi_16_64, -cospi_16_64);
 
-          u[0] = _mm_unpacklo_epi32(lstep3[12], lstep3[10]);
-          u[1] = _mm_unpackhi_epi32(lstep3[12], lstep3[10]);
-          u[2] = _mm_unpacklo_epi32(lstep3[13], lstep3[11]);
-          u[3] = _mm_unpackhi_epi32(lstep3[13], lstep3[11]);
+        u[0] = _mm_unpacklo_epi32(lstep3[12], lstep3[10]);
+        u[1] = _mm_unpackhi_epi32(lstep3[12], lstep3[10]);
+        u[2] = _mm_unpacklo_epi32(lstep3[13], lstep3[11]);
+        u[3] = _mm_unpackhi_epi32(lstep3[13], lstep3[11]);
 
-          // TODO(jingning): manually inline k_madd_epi32_ to further hide
-          // instruction latency.
-          v[ 0] = k_madd_epi32(u[0], k32_p16_m16);
-          v[ 1] = k_madd_epi32(u[1], k32_p16_m16);
-          v[ 2] = k_madd_epi32(u[2], k32_p16_m16);
-          v[ 3] = k_madd_epi32(u[3], k32_p16_m16);
-          v[ 4] = k_madd_epi32(u[0], k32_p16_p16);
-          v[ 5] = k_madd_epi32(u[1], k32_p16_p16);
-          v[ 6] = k_madd_epi32(u[2], k32_p16_p16);
-          v[ 7] = k_madd_epi32(u[3], k32_p16_p16);
+        // TODO(jingning): manually inline k_madd_epi32_ to further hide
+        // instruction latency.
+        v[ 0] = k_madd_epi32(u[0], k32_p16_m16);
+        v[ 1] = k_madd_epi32(u[1], k32_p16_m16);
+        v[ 2] = k_madd_epi32(u[2], k32_p16_m16);
+        v[ 3] = k_madd_epi32(u[3], k32_p16_m16);
+        v[ 4] = k_madd_epi32(u[0], k32_p16_p16);
+        v[ 5] = k_madd_epi32(u[1], k32_p16_p16);
+        v[ 6] = k_madd_epi32(u[2], k32_p16_p16);
+        v[ 7] = k_madd_epi32(u[3], k32_p16_p16);
 
-          u[0] = k_packs_epi64(v[0], v[1]);
-          u[1] = k_packs_epi64(v[2], v[3]);
-          u[2] = k_packs_epi64(v[4], v[5]);
-          u[3] = k_packs_epi64(v[6], v[7]);
+        u[0] = k_packs_epi64(v[0], v[1]);
+        u[1] = k_packs_epi64(v[2], v[3]);
+        u[2] = k_packs_epi64(v[4], v[5]);
+        u[3] = k_packs_epi64(v[6], v[7]);
 
-          v[0] = _mm_add_epi32(u[0], k__DCT_CONST_ROUNDING);
-          v[1] = _mm_add_epi32(u[1], k__DCT_CONST_ROUNDING);
-          v[2] = _mm_add_epi32(u[2], k__DCT_CONST_ROUNDING);
-          v[3] = _mm_add_epi32(u[3], k__DCT_CONST_ROUNDING);
+        v[0] = _mm_add_epi32(u[0], k__DCT_CONST_ROUNDING);
+        v[1] = _mm_add_epi32(u[1], k__DCT_CONST_ROUNDING);
+        v[2] = _mm_add_epi32(u[2], k__DCT_CONST_ROUNDING);
+        v[3] = _mm_add_epi32(u[3], k__DCT_CONST_ROUNDING);
 
-          lstep1[10] = _mm_srai_epi32(v[0], DCT_CONST_BITS);
-          lstep1[11] = _mm_srai_epi32(v[1], DCT_CONST_BITS);
-          lstep1[12] = _mm_srai_epi32(v[2], DCT_CONST_BITS);
-          lstep1[13] = _mm_srai_epi32(v[3], DCT_CONST_BITS);
+        lstep1[10] = _mm_srai_epi32(v[0], DCT_CONST_BITS);
+        lstep1[11] = _mm_srai_epi32(v[1], DCT_CONST_BITS);
+        lstep1[12] = _mm_srai_epi32(v[2], DCT_CONST_BITS);
+        lstep1[13] = _mm_srai_epi32(v[3], DCT_CONST_BITS);
         }
         {
           const __m128i k32_m08_p24 = pair_set_epi32(-cospi_8_64, cospi_24_64);
           const __m128i k32_m24_m08 = pair_set_epi32(-cospi_24_64, -cospi_8_64);
           const __m128i k32_p24_p08 = pair_set_epi32(cospi_24_64, cospi_8_64);
 
           u[ 0] = _mm_unpacklo_epi32(lstep3[36], lstep3[58]);
           u[ 1] = _mm_unpackhi_epi32(lstep3[36], lstep3[58]);
           u[ 2] = _mm_unpacklo_epi32(lstep3[37], lstep3[59]);
           u[ 3] = _mm_unpackhi_epi32(lstep3[37], lstep3[59]);
@@ skipping 1255 matching lines @@
           _mm_storeu_si128((__m128i *)(output + 4 * 32), tr2_4);
           _mm_storeu_si128((__m128i *)(output + 5 * 32), tr2_5);
           _mm_storeu_si128((__m128i *)(output + 6 * 32), tr2_6);
           _mm_storeu_si128((__m128i *)(output + 7 * 32), tr2_7);
           // Process next 8x8
           output += 8;
         }
       }
     }
   }
-}
+}  // NOLINT