| Index: source/libvpx/vp9/encoder/x86/vp9_highbd_sad_sse2.asm
|
| ===================================================================
|
| --- source/libvpx/vp9/encoder/x86/vp9_highbd_sad_sse2.asm (revision 0)
|
| +++ source/libvpx/vp9/encoder/x86/vp9_highbd_sad_sse2.asm (working copy)
|
| @@ -0,0 +1,363 @@
|
| +;
|
| +; Copyright (c) 2014 The WebM project authors. All Rights Reserved.
|
| +;
|
| +; Use of this source code is governed by a BSD-style license
|
| +; that can be found in the LICENSE file in the root of the source
|
| +; tree. An additional intellectual property rights grant can be found
|
| +; in the file PATENTS. All contributing project authors may
|
| +; be found in the AUTHORS file in the root of the source tree.
|
| +;
|
| +
|
| +%include "third_party/x86inc/x86inc.asm"
|
| +
|
| +SECTION .text
|
| +
|
| +%macro HIGH_SAD_FN 4
|
| +%if %4 == 0
|
| +%if %3 == 5
|
| +cglobal highbd_sad%1x%2, 4, %3, 7, src, src_stride, ref, ref_stride, n_rows
|
| +%else ; %3 == 7
|
| +cglobal highbd_sad%1x%2, 4, %3, 7, src, src_stride, ref, ref_stride, \
|
| + src_stride3, ref_stride3, n_rows
|
| +%endif ; %3 == 5/7
|
| +%else ; avg
|
| +%if %3 == 5
|
| +cglobal highbd_sad%1x%2_avg, 5, 1 + %3, 7, src, src_stride, ref, ref_stride, \
|
| + second_pred, n_rows
|
| +%else ; %3 == 7
|
| +cglobal highbd_sad%1x%2_avg, 5, ARCH_X86_64 + %3, 7, src, src_stride, \
|
| + ref, ref_stride, \
|
| + second_pred, \
|
| + src_stride3, ref_stride3
|
| +%if ARCH_X86_64
|
| +%define n_rowsd r7d
|
| +%else ; x86-32
|
| +%define n_rowsd dword r0m
|
| +%endif ; x86-32/64
|
| +%endif ; %3 == 5/7
|
| +%endif ; avg/sad
|
| + movsxdifnidn src_strideq, src_strided
|
| + movsxdifnidn ref_strideq, ref_strided
|
| +%if %3 == 7
|
| + lea src_stride3q, [src_strideq*3]
|
| + lea ref_stride3q, [ref_strideq*3]
|
| +%endif ; %3 == 7
|
| +; convert src, ref & second_pred to short ptrs (from byte ptrs)
|
| + shl srcq, 1
|
| + shl refq, 1
|
| +%if %4 == 1
|
| + shl second_predq, 1
|
| +%endif
|
| +%endmacro
|
| +
|
| +; unsigned int vp9_highbd_sad64x{16,32,64}_sse2(uint8_t *src, int src_stride,
|
| +; uint8_t *ref, int ref_stride);
|
| +%macro HIGH_SAD64XN 1-2 0
|
| + HIGH_SAD_FN 64, %1, 5, %2
|
| + mov n_rowsd, %1
|
| + pxor m0, m0
|
| + pxor m6, m6
|
| +
|
| +.loop:
|
| + ; first half of each row
|
| + movu m1, [refq]
|
| + movu m2, [refq+16]
|
| + movu m3, [refq+32]
|
| + movu m4, [refq+48]
|
| +%if %2 == 1
|
| + pavgw m1, [second_predq+mmsize*0]
|
| + pavgw m2, [second_predq+mmsize*1]
|
| + pavgw m3, [second_predq+mmsize*2]
|
| + pavgw m4, [second_predq+mmsize*3]
|
| + lea second_predq, [second_predq+mmsize*4]
|
| +%endif
|
| + mova m5, [srcq]
|
| + psubusw m5, m1
|
| + psubusw m1, [srcq]
|
| + por m1, m5
|
| + mova m5, [srcq+16]
|
| + psubusw m5, m2
|
| + psubusw m2, [srcq+16]
|
| + por m2, m5
|
| + mova m5, [srcq+32]
|
| + psubusw m5, m3
|
| + psubusw m3, [srcq+32]
|
| + por m3, m5
|
| + mova m5, [srcq+48]
|
| + psubusw m5, m4
|
| + psubusw m4, [srcq+48]
|
| + por m4, m5
|
| + paddw m1, m2
|
| + paddw m3, m4
|
| + movhlps m2, m1
|
| + movhlps m4, m3
|
| + paddw m1, m2
|
| + paddw m3, m4
|
| + punpcklwd m1, m6
|
| + punpcklwd m3, m6
|
| + paddd m0, m1
|
| + paddd m0, m3
|
| + ; second half of each row
|
| + movu m1, [refq+64]
|
| + movu m2, [refq+80]
|
| + movu m3, [refq+96]
|
| + movu m4, [refq+112]
|
| +%if %2 == 1
|
| + pavgw m1, [second_predq+mmsize*0]
|
| + pavgw m2, [second_predq+mmsize*1]
|
| + pavgw m3, [second_predq+mmsize*2]
|
| + pavgw m4, [second_predq+mmsize*3]
|
| + lea second_predq, [second_predq+mmsize*4]
|
| +%endif
|
| + mova m5, [srcq+64]
|
| + psubusw m5, m1
|
| + psubusw m1, [srcq+64]
|
| + por m1, m5
|
| + mova m5, [srcq+80]
|
| + psubusw m5, m2
|
| + psubusw m2, [srcq+80]
|
| + por m2, m5
|
| + mova m5, [srcq+96]
|
| + psubusw m5, m3
|
| + psubusw m3, [srcq+96]
|
| + por m3, m5
|
| + mova m5, [srcq+112]
|
| + psubusw m5, m4
|
| + psubusw m4, [srcq+112]
|
| + por m4, m5
|
| + paddw m1, m2
|
| + paddw m3, m4
|
| + movhlps m2, m1
|
| + movhlps m4, m3
|
| + paddw m1, m2
|
| + paddw m3, m4
|
| + punpcklwd m1, m6
|
| + punpcklwd m3, m6
|
| + lea refq, [refq+ref_strideq*2]
|
| + paddd m0, m1
|
| + lea srcq, [srcq+src_strideq*2]
|
| + paddd m0, m3
|
| +
|
| + dec n_rowsd
|
| + jg .loop
|
| +
|
| + movhlps m1, m0
|
| + paddd m0, m1
|
| + punpckldq m0, m6
|
| + movhlps m1, m0
|
| + paddd m0, m1
|
| + movd eax, m0
|
| + RET
|
| +%endmacro
|
| +
|
| +INIT_XMM sse2
|
| +HIGH_SAD64XN 64 ; highbd_sad64x64_sse2
|
| +HIGH_SAD64XN 32 ; highbd_sad64x32_sse2
|
| +HIGH_SAD64XN 64, 1 ; highbd_sad64x64_avg_sse2
|
| +HIGH_SAD64XN 32, 1 ; highbd_sad64x32_avg_sse2
|
| +
|
| +
|
| +; unsigned int vp9_highbd_sad32x{16,32,64}_sse2(uint8_t *src, int src_stride,
|
| +; uint8_t *ref, int ref_stride);
|
| +%macro HIGH_SAD32XN 1-2 0
|
| + HIGH_SAD_FN 32, %1, 5, %2
|
| + mov n_rowsd, %1
|
| + pxor m0, m0
|
| + pxor m6, m6
|
| +
|
| +.loop:
|
| + movu m1, [refq]
|
| + movu m2, [refq+16]
|
| + movu m3, [refq+32]
|
| + movu m4, [refq+48]
|
| +%if %2 == 1
|
| + pavgw m1, [second_predq+mmsize*0]
|
| + pavgw m2, [second_predq+mmsize*1]
|
| + pavgw m3, [second_predq+mmsize*2]
|
| + pavgw m4, [second_predq+mmsize*3]
|
| + lea second_predq, [second_predq+mmsize*4]
|
| +%endif
|
| + mova m5, [srcq]
|
| + psubusw m5, m1
|
| + psubusw m1, [srcq]
|
| + por m1, m5
|
| + mova m5, [srcq+16]
|
| + psubusw m5, m2
|
| + psubusw m2, [srcq+16]
|
| + por m2, m5
|
| + mova m5, [srcq+32]
|
| + psubusw m5, m3
|
| + psubusw m3, [srcq+32]
|
| + por m3, m5
|
| + mova m5, [srcq+48]
|
| + psubusw m5, m4
|
| + psubusw m4, [srcq+48]
|
| + por m4, m5
|
| + paddw m1, m2
|
| + paddw m3, m4
|
| + movhlps m2, m1
|
| + movhlps m4, m3
|
| + paddw m1, m2
|
| + paddw m3, m4
|
| + punpcklwd m1, m6
|
| + punpcklwd m3, m6
|
| + lea refq, [refq+ref_strideq*2]
|
| + paddd m0, m1
|
| + lea srcq, [srcq+src_strideq*2]
|
| + paddd m0, m3
|
| + dec n_rowsd
|
| + jg .loop
|
| +
|
| + movhlps m1, m0
|
| + paddd m0, m1
|
| + punpckldq m0, m6
|
| + movhlps m1, m0
|
| + paddd m0, m1
|
| + movd eax, m0
|
| + RET
|
| +%endmacro
|
| +
|
| +INIT_XMM sse2
|
| +HIGH_SAD32XN 64 ; highbd_sad32x64_sse2
|
| +HIGH_SAD32XN 32 ; highbd_sad32x32_sse2
|
| +HIGH_SAD32XN 16 ; highbd_sad32x16_sse2
|
| +HIGH_SAD32XN 64, 1 ; highbd_sad32x64_avg_sse2
|
| +HIGH_SAD32XN 32, 1 ; highbd_sad32x32_avg_sse2
|
| +HIGH_SAD32XN 16, 1 ; highbd_sad32x16_avg_sse2
|
| +
|
| +; unsigned int vp9_highbd_sad16x{8,16,32}_sse2(uint8_t *src, int src_stride,
|
| +; uint8_t *ref, int ref_stride);
|
| +%macro HIGH_SAD16XN 1-2 0
|
| + HIGH_SAD_FN 16, %1, 5, %2
|
| + mov n_rowsd, %1/2
|
| + pxor m0, m0
|
| + pxor m6, m6
|
| +
|
| +.loop:
|
| + movu m1, [refq]
|
| + movu m2, [refq+16]
|
| + movu m3, [refq+ref_strideq*2]
|
| + movu m4, [refq+ref_strideq*2+16]
|
| +%if %2 == 1
|
| + pavgw m1, [second_predq+mmsize*0]
|
| + pavgw m2, [second_predq+16]
|
| + pavgw m3, [second_predq+mmsize*2]
|
| + pavgw m4, [second_predq+mmsize*2+16]
|
| + lea second_predq, [second_predq+mmsize*4]
|
| +%endif
|
| + mova m5, [srcq]
|
| + psubusw m5, m1
|
| + psubusw m1, [srcq]
|
| + por m1, m5
|
| + mova m5, [srcq+16]
|
| + psubusw m5, m2
|
| + psubusw m2, [srcq+16]
|
| + por m2, m5
|
| + mova m5, [srcq+src_strideq*2]
|
| + psubusw m5, m3
|
| + psubusw m3, [srcq+src_strideq*2]
|
| + por m3, m5
|
| + mova m5, [srcq+src_strideq*2+16]
|
| + psubusw m5, m4
|
| + psubusw m4, [srcq+src_strideq*2+16]
|
| + por m4, m5
|
| + paddw m1, m2
|
| + paddw m3, m4
|
| + movhlps m2, m1
|
| + movhlps m4, m3
|
| + paddw m1, m2
|
| + paddw m3, m4
|
| + punpcklwd m1, m6
|
| + punpcklwd m3, m6
|
| + lea refq, [refq+ref_strideq*4]
|
| + paddd m0, m1
|
| + lea srcq, [srcq+src_strideq*4]
|
| + paddd m0, m3
|
| + dec n_rowsd
|
| + jg .loop
|
| +
|
| + movhlps m1, m0
|
| + paddd m0, m1
|
| + punpckldq m0, m6
|
| + movhlps m1, m0
|
| + paddd m0, m1
|
| + movd eax, m0
|
| + RET
|
| +%endmacro
|
| +
|
| +INIT_XMM sse2
|
| +HIGH_SAD16XN 32 ; highbd_sad16x32_sse2
|
| +HIGH_SAD16XN 16 ; highbd_sad16x16_sse2
|
| +HIGH_SAD16XN 8 ; highbd_sad16x8_sse2
|
| +HIGH_SAD16XN 32, 1 ; highbd_sad16x32_avg_sse2
|
| +HIGH_SAD16XN 16, 1 ; highbd_sad16x16_avg_sse2
|
| +HIGH_SAD16XN 8, 1 ; highbd_sad16x8_avg_sse2
|
| +
|
| +
|
| +; unsigned int vp9_highbd_sad8x{4,8,16}_sse2(uint8_t *src, int src_stride,
|
| +; uint8_t *ref, int ref_stride);
|
| +%macro HIGH_SAD8XN 1-2 0
|
| + HIGH_SAD_FN 8, %1, 7, %2
|
| + mov n_rowsd, %1/4
|
| + pxor m0, m0
|
| + pxor m6, m6
|
| +
|
| +.loop:
|
| + movu m1, [refq]
|
| + movu m2, [refq+ref_strideq*2]
|
| + movu m3, [refq+ref_strideq*4]
|
| + movu m4, [refq+ref_stride3q*2]
|
| +%if %2 == 1
|
| + pavgw m1, [second_predq+mmsize*0]
|
| + pavgw m2, [second_predq+mmsize*1]
|
| + pavgw m3, [second_predq+mmsize*2]
|
| + pavgw m4, [second_predq+mmsize*3]
|
| + lea second_predq, [second_predq+mmsize*4]
|
| +%endif
|
| + mova m5, [srcq]
|
| + psubusw m5, m1
|
| + psubusw m1, [srcq]
|
| + por m1, m5
|
| + mova m5, [srcq+src_strideq*2]
|
| + psubusw m5, m2
|
| + psubusw m2, [srcq+src_strideq*2]
|
| + por m2, m5
|
| + mova m5, [srcq+src_strideq*4]
|
| + psubusw m5, m3
|
| + psubusw m3, [srcq+src_strideq*4]
|
| + por m3, m5
|
| + mova m5, [srcq+src_stride3q*2]
|
| + psubusw m5, m4
|
| + psubusw m4, [srcq+src_stride3q*2]
|
| + por m4, m5
|
| + paddw m1, m2
|
| + paddw m3, m4
|
| + movhlps m2, m1
|
| + movhlps m4, m3
|
| + paddw m1, m2
|
| + paddw m3, m4
|
| + punpcklwd m1, m6
|
| + punpcklwd m3, m6
|
| + lea refq, [refq+ref_strideq*8]
|
| + paddd m0, m1
|
| + lea srcq, [srcq+src_strideq*8]
|
| + paddd m0, m3
|
| + dec n_rowsd
|
| + jg .loop
|
| +
|
| + movhlps m1, m0
|
| + paddd m0, m1
|
| + punpckldq m0, m6
|
| + movhlps m1, m0
|
| + paddd m0, m1
|
| + movd eax, m0
|
| + RET
|
| +%endmacro
|
| +
|
| +INIT_XMM sse2
|
| +HIGH_SAD8XN 16 ; highbd_sad8x16_sse2
|
| +HIGH_SAD8XN 8 ; highbd_sad8x8_sse2
|
| +HIGH_SAD8XN 4 ; highbd_sad8x4_sse2
|
| +HIGH_SAD8XN 16, 1 ; highbd_sad8x16_avg_sse2
|
| +HIGH_SAD8XN 8, 1 ; highbd_sad8x8_avg_sse2
|
| +HIGH_SAD8XN 4, 1 ; highbd_sad8x4_avg_sse2
|
|
|