libvpx: Pull from upstream

source/libvpx/vp9/encoder/x86/vp9_dct_sse2.c

 /*
  *  Copyright (c) 2012 The WebM project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
 #include <emmintrin.h>  // SSE2
 #include "vp9/common/vp9_idct.h"  // for cospi constants
 #include "vpx_ports/mem.h"
 
 void vp9_fdct4x4_sse2(const int16_t *input, int16_t *output, int stride) {
   // The 2D transform is done with two passes which are actually pretty
   // similar. In the first one, we transform the columns and transpose
   // the results. In the second one, we transform the rows. To achieve that,
   // as the first pass results are transposed, we tranpose the columns (that
   // is the transposed rows) and transpose the results (so that it goes back
   // in normal/row positions).
   int pass;
   // Constants
   //    When we use them, in one case, they are all the same. In all others
   //    it's a pair of them that we need to repeat four times. This is done
   //    by constructing the 32 bit constant corresponding to that pair.
   const __m128i k__cospi_p16_p16 = _mm_set1_epi16(cospi_16_64);
   const __m128i k__cospi_p16_m16 = pair_set_epi16(cospi_16_64, -cospi_16_64);
-  const __m128i k__cospi_p24_p08 = pair_set_epi16(cospi_24_64, cospi_8_64);
-  const __m128i k__cospi_m08_p24 = pair_set_epi16(-cospi_8_64, cospi_24_64);
+  const __m128i k__cospi_p08_p24 = pair_set_epi16(cospi_8_64, cospi_24_64);
+  const __m128i k__cospi_p24_m08 = pair_set_epi16(cospi_24_64, -cospi_8_64);
   const __m128i k__DCT_CONST_ROUNDING = _mm_set1_epi32(DCT_CONST_ROUNDING);
   const __m128i k__nonzero_bias_a = _mm_setr_epi16(0, 1, 1, 1, 1, 1, 1, 1);
   const __m128i k__nonzero_bias_b = _mm_setr_epi16(1, 0, 0, 0, 0, 0, 0, 0);
   const __m128i kOne = _mm_set1_epi16(1);
-  __m128i in0, in1, in2, in3;
+  __m128i in0, in1;
   // Load inputs.
   {
     in0  = _mm_loadl_epi64((const __m128i *)(input +  0 * stride));
-    in1  = _mm_loadl_epi64((const __m128i *)(input +  1 * stride));
-    in2  = _mm_loadl_epi64((const __m128i *)(input +  2 * stride));
-    in3  = _mm_loadl_epi64((const __m128i *)(input +  3 * stride));
+    in0  = _mm_unpacklo_epi64(in0, _mm_loadl_epi64((const __m128i *)
+           (input +  1 * stride)));
+    in1  = _mm_loadl_epi64((const __m128i *)(input +  2 * stride));
+    in1  = _mm_unpacklo_epi64(_mm_loadl_epi64((const __m128i *)
+           (input +  3 * stride)), in1);
+
     // x = x << 4
     in0 = _mm_slli_epi16(in0, 4);
     in1 = _mm_slli_epi16(in1, 4);
-    in2 = _mm_slli_epi16(in2, 4);
-    in3 = _mm_slli_epi16(in3, 4);
     // if (i == 0 && input[0]) input[0] += 1;
     {
       // The mask will only contain wether the first value is zero, all
       // other comparison will fail as something shifted by 4 (above << 4)
       // can never be equal to one. To increment in the non-zero case, we
       // add the mask and one for the first element:
       //   - if zero, mask = -1, v = v - 1 + 1 = v
       //   - if non-zero, mask = 0, v = v + 0 + 1 = v + 1
       __m128i mask = _mm_cmpeq_epi16(in0, k__nonzero_bias_a);
       in0 = _mm_add_epi16(in0, mask);
       in0 = _mm_add_epi16(in0, k__nonzero_bias_b);
     }
   }
   // Do the two transform/transpose passes
   for (pass = 0; pass < 2; ++pass) {
     // Transform 1/2: Add/substract
-    const __m128i r0 = _mm_add_epi16(in0, in3);
-    const __m128i r1 = _mm_add_epi16(in1, in2);
-    const __m128i r2 = _mm_sub_epi16(in1, in2);
-    const __m128i r3 = _mm_sub_epi16(in0, in3);
+    const __m128i r0 = _mm_add_epi16(in0, in1);
+    const __m128i r1 = _mm_sub_epi16(in0, in1);
+    const __m128i r2 = _mm_unpacklo_epi64(r0, r1);
+    const __m128i r3 = _mm_unpackhi_epi64(r0, r1);
     // Transform 1/2: Interleave to do the multiply by constants which gets us
     //                into 32 bits.
-    const __m128i t0 = _mm_unpacklo_epi16(r0, r1);
-    const __m128i t2 = _mm_unpacklo_epi16(r2, r3);
+    const __m128i t0 = _mm_unpacklo_epi16(r2, r3);
+    const __m128i t2 = _mm_unpackhi_epi16(r2, r3);
     const __m128i u0 = _mm_madd_epi16(t0, k__cospi_p16_p16);
     const __m128i u2 = _mm_madd_epi16(t0, k__cospi_p16_m16);
-    const __m128i u4 = _mm_madd_epi16(t2, k__cospi_p24_p08);
-    const __m128i u6 = _mm_madd_epi16(t2, k__cospi_m08_p24);
+    const __m128i u4 = _mm_madd_epi16(t2, k__cospi_p08_p24);
+    const __m128i u6 = _mm_madd_epi16(t2, k__cospi_p24_m08);
     const __m128i v0 = _mm_add_epi32(u0, k__DCT_CONST_ROUNDING);
     const __m128i v2 = _mm_add_epi32(u2, k__DCT_CONST_ROUNDING);
     const __m128i v4 = _mm_add_epi32(u4, k__DCT_CONST_ROUNDING);
     const __m128i v6 = _mm_add_epi32(u6, k__DCT_CONST_ROUNDING);
     const __m128i w0 = _mm_srai_epi32(v0, DCT_CONST_BITS);
     const __m128i w2 = _mm_srai_epi32(v2, DCT_CONST_BITS);
     const __m128i w4 = _mm_srai_epi32(v4, DCT_CONST_BITS);
     const __m128i w6 = _mm_srai_epi32(v6, DCT_CONST_BITS);
     // Combine and transpose
     const __m128i res0 = _mm_packs_epi32(w0, w2);
     const __m128i res1 = _mm_packs_epi32(w4, w6);
     // 00 01 02 03 20 21 22 23
     // 10 11 12 13 30 31 32 33
     const __m128i tr0_0 = _mm_unpacklo_epi16(res0, res1);
     const __m128i tr0_1 = _mm_unpackhi_epi16(res0, res1);
     // 00 10 01 11 02 12 03 13
     // 20 30 21 31 22 32 23 33
     in0 = _mm_unpacklo_epi32(tr0_0, tr0_1);
-    in2 = _mm_unpackhi_epi32(tr0_0, tr0_1);
+    in1 = _mm_unpackhi_epi32(tr0_0, tr0_1);
+    in1 = _mm_shuffle_epi32(in1, 0x4E);
     // 00 10 20 30 01 11 21 31      in0 contains 0 followed by 1
-    // 02 12 22 32 03 13 23 33      in2 contains 2 followed by 3
-    if (0 == pass) {
-      // Extract values in the high part for second pass as transform code
-      // only uses the first four values.
-      in1 = _mm_unpackhi_epi64(in0, in0);
-      in3 = _mm_unpackhi_epi64(in2, in2);
-    } else {
-      // Post-condition output and store it (v + 1) >> 2, taking advantage
-      // of the fact 1/3 are stored just after 0/2.
-      __m128i out01 = _mm_add_epi16(in0, kOne);
-      __m128i out23 = _mm_add_epi16(in2, kOne);
-      out01 = _mm_srai_epi16(out01, 2);
-      out23 = _mm_srai_epi16(out23, 2);
-      _mm_storeu_si128((__m128i *)(output + 0 * 4), out01);
-      _mm_storeu_si128((__m128i *)(output + 2 * 4), out23);
-    }
+    // 02 12 22 32 03 13 23 33      in1 contains 2 followed by 3
+  }
+  in1 = _mm_shuffle_epi32(in1, 0x4E);
+  // Post-condition output and store it (v + 1) >> 2, taking advantage
+  // of the fact 1/3 are stored just after 0/2.
+  {
+     __m128i out01 = _mm_add_epi16(in0, kOne);
+     __m128i out23 = _mm_add_epi16(in1, kOne);
+     out01 = _mm_srai_epi16(out01, 2);
+     out23 = _mm_srai_epi16(out23, 2);
+     _mm_storeu_si128((__m128i *)(output + 0 * 4), out01);
+     _mm_storeu_si128((__m128i *)(output + 2 * 4), out23);
   }
 }
 
 static INLINE void load_buffer_4x4(const int16_t *input, __m128i *in,
                                    int stride) {
   const __m128i k__nonzero_bias_a = _mm_setr_epi16(0, 1, 1, 1, 1, 1, 1, 1);
   const __m128i k__nonzero_bias_b = _mm_setr_epi16(1, 0, 0, 0, 0, 0, 0, 0);
   __m128i mask;
 
   in[0] = _mm_loadl_epi64((const __m128i *)(input + 0 * stride));
@@ skipping 78 matching lines @@
 void fadst4_1d_sse2(__m128i *in) {
   const __m128i k__sinpi_p01_p02 = pair_set_epi16(sinpi_1_9, sinpi_2_9);
   const __m128i k__sinpi_p04_m01 = pair_set_epi16(sinpi_4_9, -sinpi_1_9);
   const __m128i k__sinpi_p03_p04 = pair_set_epi16(sinpi_3_9, sinpi_4_9);
   const __m128i k__sinpi_m03_p02 = pair_set_epi16(-sinpi_3_9, sinpi_2_9);
   const __m128i k__sinpi_p03_p03 = _mm_set1_epi16(sinpi_3_9);
   const __m128i kZero = _mm_set1_epi16(0);
   const __m128i k__DCT_CONST_ROUNDING = _mm_set1_epi32(DCT_CONST_ROUNDING);
   __m128i u[8], v[8];
   __m128i in7 = _mm_add_epi16(in[0], in[1]);
-  in7 = _mm_sub_epi16(in7, in[3]);
 
   u[0] = _mm_unpacklo_epi16(in[0], in[1]);
   u[1] = _mm_unpacklo_epi16(in[2], in[3]);
   u[2] = _mm_unpacklo_epi16(in7, kZero);
   u[3] = _mm_unpacklo_epi16(in[2], kZero);
+  u[4] = _mm_unpacklo_epi16(in[3], kZero);
 
   v[0] = _mm_madd_epi16(u[0], k__sinpi_p01_p02);  // s0 + s2
   v[1] = _mm_madd_epi16(u[1], k__sinpi_p03_p04);  // s4 + s5
   v[2] = _mm_madd_epi16(u[2], k__sinpi_p03_p03);  // x1
   v[3] = _mm_madd_epi16(u[0], k__sinpi_p04_m01);  // s1 - s3
   v[4] = _mm_madd_epi16(u[1], k__sinpi_m03_p02);  // -s4 + s6
   v[5] = _mm_madd_epi16(u[3], k__sinpi_p03_p03);  // s4
+  v[6] = _mm_madd_epi16(u[4], k__sinpi_p03_p03);
 
   u[0] = _mm_add_epi32(v[0], v[1]);
-  u[1] = v[2];
+  u[1] = _mm_sub_epi32(v[2], v[6]);
   u[2] = _mm_add_epi32(v[3], v[4]);
   u[3] = _mm_sub_epi32(u[2], u[0]);
   u[4] = _mm_slli_epi32(v[5], 2);
   u[5] = _mm_sub_epi32(u[4], v[5]);
   u[6] = _mm_add_epi32(u[3], u[5]);
 
   v[0] = _mm_add_epi32(u[0], k__DCT_CONST_ROUNDING);
   v[1] = _mm_add_epi32(u[1], k__DCT_CONST_ROUNDING);
   v[2] = _mm_add_epi32(u[2], k__DCT_CONST_ROUNDING);
   v[3] = _mm_add_epi32(u[6], k__DCT_CONST_ROUNDING);
@@ skipping 2334 matching lines @@
 #define FDCT32x32_HIGH_PRECISION 0
 #include "vp9/encoder/x86/vp9_dct32x32_sse2.c"
 #undef  FDCT32x32_2D
 #undef  FDCT32x32_HIGH_PRECISION
 
 #define FDCT32x32_2D vp9_fdct32x32_sse2
 #define FDCT32x32_HIGH_PRECISION 1
 #include "vp9/encoder/x86/vp9_dct32x32_sse2.c" // NOLINT
 #undef  FDCT32x32_2D
 #undef  FDCT32x32_HIGH_PRECISION