| Index: source/libvpx/vp9/encoder/x86/vp9_dct_sse2.asm
|
| ===================================================================
|
| --- source/libvpx/vp9/encoder/x86/vp9_dct_sse2.asm (revision 0)
|
| +++ source/libvpx/vp9/encoder/x86/vp9_dct_sse2.asm (revision 0)
|
| @@ -0,0 +1,432 @@
|
| +;
|
| +; Copyright (c) 2010 The WebM project authors. All Rights Reserved.
|
| +;
|
| +; Use of this source code is governed by a BSD-style license
|
| +; that can be found in the LICENSE file in the root of the source
|
| +; tree. An additional intellectual property rights grant can be found
|
| +; in the file PATENTS. All contributing project authors may
|
| +; be found in the AUTHORS file in the root of the source tree.
|
| +;
|
| +
|
| +
|
| +%include "vpx_ports/x86_abi_support.asm"
|
| +
|
| +%macro STACK_FRAME_CREATE 0
|
| +%if ABI_IS_32BIT
|
| + %define input rsi
|
| + %define output rdi
|
| + %define pitch rax
|
| + push rbp
|
| + mov rbp, rsp
|
| + GET_GOT rbx
|
| + push rsi
|
| + push rdi
|
| + ; end prolog
|
| +
|
| + mov rsi, arg(0)
|
| + mov rdi, arg(1)
|
| +
|
| + movsxd rax, dword ptr arg(2)
|
| + lea rcx, [rsi + rax*2]
|
| +%else
|
| + %ifidn __OUTPUT_FORMAT__,x64
|
| + %define input rcx
|
| + %define output rdx
|
| + %define pitch r8
|
| + SAVE_XMM 7, u
|
| + %else
|
| + %define input rdi
|
| + %define output rsi
|
| + %define pitch rdx
|
| + %endif
|
| +%endif
|
| +%endmacro
|
| +
|
| +%macro STACK_FRAME_DESTROY 0
|
| + %define input
|
| + %define output
|
| + %define pitch
|
| +
|
| +%if ABI_IS_32BIT
|
| + pop rdi
|
| + pop rsi
|
| + RESTORE_GOT
|
| + pop rbp
|
| +%else
|
| + %ifidn __OUTPUT_FORMAT__,x64
|
| + RESTORE_XMM
|
| + %endif
|
| +%endif
|
| + ret
|
| +%endmacro
|
| +
|
| +;void vp9_short_fdct4x4_sse2(short *input, short *output, int pitch)
|
| +global sym(vp9_short_fdct4x4_sse2)
|
| +sym(vp9_short_fdct4x4_sse2):
|
| +
|
| + STACK_FRAME_CREATE
|
| +
|
| + movq xmm0, MMWORD PTR[input ] ;03 02 01 00
|
| + movq xmm2, MMWORD PTR[input+ pitch] ;13 12 11 10
|
| + lea input, [input+2*pitch]
|
| + movq xmm1, MMWORD PTR[input ] ;23 22 21 20
|
| + movq xmm3, MMWORD PTR[input+ pitch] ;33 32 31 30
|
| +
|
| + punpcklqdq xmm0, xmm2 ;13 12 11 10 03 02 01 00
|
| + punpcklqdq xmm1, xmm3 ;33 32 31 30 23 22 21 20
|
| +
|
| + movdqa xmm2, xmm0
|
| + punpckldq xmm0, xmm1 ;23 22 03 02 21 20 01 00
|
| + punpckhdq xmm2, xmm1 ;33 32 13 12 31 30 11 10
|
| + movdqa xmm1, xmm0
|
| + punpckldq xmm0, xmm2 ;31 21 30 20 11 10 01 00
|
| + pshufhw xmm1, xmm1, 0b1h ;22 23 02 03 xx xx xx xx
|
| + pshufhw xmm2, xmm2, 0b1h ;32 33 12 13 xx xx xx xx
|
| +
|
| + punpckhdq xmm1, xmm2 ;32 33 22 23 12 13 02 03
|
| + movdqa xmm3, xmm0
|
| + paddw xmm0, xmm1 ;b1 a1 b1 a1 b1 a1 b1 a1
|
| + psubw xmm3, xmm1 ;c1 d1 c1 d1 c1 d1 c1 d1
|
| + psllw xmm0, 3 ;b1 <<= 3 a1 <<= 3
|
| + psllw xmm3, 3 ;c1 <<= 3 d1 <<= 3
|
| +
|
| + movdqa xmm1, xmm0
|
| + pmaddwd xmm0, XMMWORD PTR[GLOBAL(_mult_add)] ;a1 + b1
|
| + pmaddwd xmm1, XMMWORD PTR[GLOBAL(_mult_sub)] ;a1 - b1
|
| + movdqa xmm4, xmm3
|
| + pmaddwd xmm3, XMMWORD PTR[GLOBAL(_5352_2217)] ;c1*2217 + d1*5352
|
| + pmaddwd xmm4, XMMWORD PTR[GLOBAL(_2217_neg5352)];d1*2217 - c1*5352
|
| +
|
| + paddd xmm3, XMMWORD PTR[GLOBAL(_14500)]
|
| + paddd xmm4, XMMWORD PTR[GLOBAL(_7500)]
|
| + psrad xmm3, 12 ;(c1 * 2217 + d1 * 5352 + 14500)>>12
|
| + psrad xmm4, 12 ;(d1 * 2217 - c1 * 5352 + 7500)>>12
|
| +
|
| + packssdw xmm0, xmm1 ;op[2] op[0]
|
| + packssdw xmm3, xmm4 ;op[3] op[1]
|
| + ; 23 22 21 20 03 02 01 00
|
| + ;
|
| + ; 33 32 31 30 13 12 11 10
|
| + ;
|
| + movdqa xmm2, xmm0
|
| + punpcklqdq xmm0, xmm3 ;13 12 11 10 03 02 01 00
|
| + punpckhqdq xmm2, xmm3 ;23 22 21 20 33 32 31 30
|
| +
|
| + movdqa xmm3, xmm0
|
| + punpcklwd xmm0, xmm2 ;32 30 22 20 12 10 02 00
|
| + punpckhwd xmm3, xmm2 ;33 31 23 21 13 11 03 01
|
| + movdqa xmm2, xmm0
|
| + punpcklwd xmm0, xmm3 ;13 12 11 10 03 02 01 00
|
| + punpckhwd xmm2, xmm3 ;33 32 31 30 23 22 21 20
|
| +
|
| + movdqa xmm5, XMMWORD PTR[GLOBAL(_7)]
|
| + pshufd xmm2, xmm2, 04eh
|
| + movdqa xmm3, xmm0
|
| + paddw xmm0, xmm2 ;b1 b1 b1 b1 a1 a1 a1 a1
|
| + psubw xmm3, xmm2 ;c1 c1 c1 c1 d1 d1 d1 d1
|
| +
|
| + pshufd xmm0, xmm0, 0d8h ;b1 b1 a1 a1 b1 b1 a1 a1
|
| + movdqa xmm2, xmm3 ;save d1 for compare
|
| + pshufd xmm3, xmm3, 0d8h ;c1 c1 d1 d1 c1 c1 d1 d1
|
| + pshuflw xmm0, xmm0, 0d8h ;b1 b1 a1 a1 b1 a1 b1 a1
|
| + pshuflw xmm3, xmm3, 0d8h ;c1 c1 d1 d1 c1 d1 c1 d1
|
| + pshufhw xmm0, xmm0, 0d8h ;b1 a1 b1 a1 b1 a1 b1 a1
|
| + pshufhw xmm3, xmm3, 0d8h ;c1 d1 c1 d1 c1 d1 c1 d1
|
| + movdqa xmm1, xmm0
|
| + pmaddwd xmm0, XMMWORD PTR[GLOBAL(_mult_add)] ;a1 + b1
|
| + pmaddwd xmm1, XMMWORD PTR[GLOBAL(_mult_sub)] ;a1 - b1
|
| +
|
| + pxor xmm4, xmm4 ;zero out for compare
|
| + paddd xmm0, xmm5
|
| + paddd xmm1, xmm5
|
| + pcmpeqw xmm2, xmm4
|
| + psrad xmm0, 4 ;(a1 + b1 + 7)>>4
|
| + psrad xmm1, 4 ;(a1 - b1 + 7)>>4
|
| + pandn xmm2, XMMWORD PTR[GLOBAL(_cmp_mask)] ;clear upper,
|
| + ;and keep bit 0 of lower
|
| +
|
| + movdqa xmm4, xmm3
|
| + pmaddwd xmm3, XMMWORD PTR[GLOBAL(_5352_2217)] ;c1*2217 + d1*5352
|
| + pmaddwd xmm4, XMMWORD PTR[GLOBAL(_2217_neg5352)] ;d1*2217 - c1*5352
|
| + paddd xmm3, XMMWORD PTR[GLOBAL(_12000)]
|
| + paddd xmm4, XMMWORD PTR[GLOBAL(_51000)]
|
| + packssdw xmm0, xmm1 ;op[8] op[0]
|
| + psrad xmm3, 16 ;(c1 * 2217 + d1 * 5352 + 12000)>>16
|
| + psrad xmm4, 16 ;(d1 * 2217 - c1 * 5352 + 51000)>>16
|
| +
|
| + packssdw xmm3, xmm4 ;op[12] op[4]
|
| + movdqa xmm1, xmm0
|
| + paddw xmm3, xmm2 ;op[4] += (d1!=0)
|
| + punpcklqdq xmm0, xmm3 ;op[4] op[0]
|
| + punpckhqdq xmm1, xmm3 ;op[12] op[8]
|
| +
|
| + movdqa XMMWORD PTR[output + 0], xmm0
|
| + movdqa XMMWORD PTR[output + 16], xmm1
|
| +
|
| + STACK_FRAME_DESTROY
|
| +
|
| +;void vp9_short_fdct8x4_sse2(short *input, short *output, int pitch)
|
| +global sym(vp9_short_fdct8x4_sse2)
|
| +sym(vp9_short_fdct8x4_sse2):
|
| +
|
| + STACK_FRAME_CREATE
|
| +
|
| + ; read the input data
|
| + movdqa xmm0, [input ]
|
| + movdqa xmm2, [input+ pitch]
|
| + lea input, [input+2*pitch]
|
| + movdqa xmm4, [input ]
|
| + movdqa xmm3, [input+ pitch]
|
| +
|
| + ; transpose for the first stage
|
| + movdqa xmm1, xmm0 ; 00 01 02 03 04 05 06 07
|
| + movdqa xmm5, xmm4 ; 20 21 22 23 24 25 26 27
|
| +
|
| + punpcklwd xmm0, xmm2 ; 00 10 01 11 02 12 03 13
|
| + punpckhwd xmm1, xmm2 ; 04 14 05 15 06 16 07 17
|
| +
|
| + punpcklwd xmm4, xmm3 ; 20 30 21 31 22 32 23 33
|
| + punpckhwd xmm5, xmm3 ; 24 34 25 35 26 36 27 37
|
| +
|
| + movdqa xmm2, xmm0 ; 00 10 01 11 02 12 03 13
|
| + punpckldq xmm0, xmm4 ; 00 10 20 30 01 11 21 31
|
| +
|
| + punpckhdq xmm2, xmm4 ; 02 12 22 32 03 13 23 33
|
| +
|
| + movdqa xmm4, xmm1 ; 04 14 05 15 06 16 07 17
|
| + punpckldq xmm4, xmm5 ; 04 14 24 34 05 15 25 35
|
| +
|
| + punpckhdq xmm1, xmm5 ; 06 16 26 36 07 17 27 37
|
| + movdqa xmm3, xmm2 ; 02 12 22 32 03 13 23 33
|
| +
|
| + punpckhqdq xmm3, xmm1 ; 03 13 23 33 07 17 27 37
|
| + punpcklqdq xmm2, xmm1 ; 02 12 22 32 06 16 26 36
|
| +
|
| + movdqa xmm1, xmm0 ; 00 10 20 30 01 11 21 31
|
| + punpcklqdq xmm0, xmm4 ; 00 10 20 30 04 14 24 34
|
| +
|
| + punpckhqdq xmm1, xmm4 ; 01 11 21 32 05 15 25 35
|
| +
|
| + ; xmm0 0
|
| + ; xmm1 1
|
| + ; xmm2 2
|
| + ; xmm3 3
|
| +
|
| + ; first stage
|
| + movdqa xmm5, xmm0
|
| + movdqa xmm4, xmm1
|
| +
|
| + paddw xmm0, xmm3 ; a1 = 0 + 3
|
| + paddw xmm1, xmm2 ; b1 = 1 + 2
|
| +
|
| + psubw xmm4, xmm2 ; c1 = 1 - 2
|
| + psubw xmm5, xmm3 ; d1 = 0 - 3
|
| +
|
| + psllw xmm5, 3
|
| + psllw xmm4, 3
|
| +
|
| + psllw xmm0, 3
|
| + psllw xmm1, 3
|
| +
|
| + ; output 0 and 2
|
| + movdqa xmm2, xmm0 ; a1
|
| +
|
| + paddw xmm0, xmm1 ; op[0] = a1 + b1
|
| + psubw xmm2, xmm1 ; op[2] = a1 - b1
|
| +
|
| + ; output 1 and 3
|
| + ; interleave c1, d1
|
| + movdqa xmm1, xmm5 ; d1
|
| + punpcklwd xmm1, xmm4 ; c1 d1
|
| + punpckhwd xmm5, xmm4 ; c1 d1
|
| +
|
| + movdqa xmm3, xmm1
|
| + movdqa xmm4, xmm5
|
| +
|
| + pmaddwd xmm1, XMMWORD PTR[GLOBAL (_5352_2217)] ; c1*2217 + d1*5352
|
| + pmaddwd xmm4, XMMWORD PTR[GLOBAL (_5352_2217)] ; c1*2217 + d1*5352
|
| +
|
| + pmaddwd xmm3, XMMWORD PTR[GLOBAL(_2217_neg5352)] ; d1*2217 - c1*5352
|
| + pmaddwd xmm5, XMMWORD PTR[GLOBAL(_2217_neg5352)] ; d1*2217 - c1*5352
|
| +
|
| + paddd xmm1, XMMWORD PTR[GLOBAL(_14500)]
|
| + paddd xmm4, XMMWORD PTR[GLOBAL(_14500)]
|
| + paddd xmm3, XMMWORD PTR[GLOBAL(_7500)]
|
| + paddd xmm5, XMMWORD PTR[GLOBAL(_7500)]
|
| +
|
| + psrad xmm1, 12 ; (c1 * 2217 + d1 * 5352 + 14500)>>12
|
| + psrad xmm4, 12 ; (c1 * 2217 + d1 * 5352 + 14500)>>12
|
| + psrad xmm3, 12 ; (d1 * 2217 - c1 * 5352 + 7500)>>12
|
| + psrad xmm5, 12 ; (d1 * 2217 - c1 * 5352 + 7500)>>12
|
| +
|
| + packssdw xmm1, xmm4 ; op[1]
|
| + packssdw xmm3, xmm5 ; op[3]
|
| +
|
| + ; done with vertical
|
| + ; transpose for the second stage
|
| + movdqa xmm4, xmm0 ; 00 10 20 30 04 14 24 34
|
| + movdqa xmm5, xmm2 ; 02 12 22 32 06 16 26 36
|
| +
|
| + punpcklwd xmm0, xmm1 ; 00 01 10 11 20 21 30 31
|
| + punpckhwd xmm4, xmm1 ; 04 05 14 15 24 25 34 35
|
| +
|
| + punpcklwd xmm2, xmm3 ; 02 03 12 13 22 23 32 33
|
| + punpckhwd xmm5, xmm3 ; 06 07 16 17 26 27 36 37
|
| +
|
| + movdqa xmm1, xmm0 ; 00 01 10 11 20 21 30 31
|
| + punpckldq xmm0, xmm2 ; 00 01 02 03 10 11 12 13
|
| +
|
| + punpckhdq xmm1, xmm2 ; 20 21 22 23 30 31 32 33
|
| +
|
| + movdqa xmm2, xmm4 ; 04 05 14 15 24 25 34 35
|
| + punpckldq xmm2, xmm5 ; 04 05 06 07 14 15 16 17
|
| +
|
| + punpckhdq xmm4, xmm5 ; 24 25 26 27 34 35 36 37
|
| + movdqa xmm3, xmm1 ; 20 21 22 23 30 31 32 33
|
| +
|
| + punpckhqdq xmm3, xmm4 ; 30 31 32 33 34 35 36 37
|
| + punpcklqdq xmm1, xmm4 ; 20 21 22 23 24 25 26 27
|
| +
|
| + movdqa xmm4, xmm0 ; 00 01 02 03 10 11 12 13
|
| + punpcklqdq xmm0, xmm2 ; 00 01 02 03 04 05 06 07
|
| +
|
| + punpckhqdq xmm4, xmm2 ; 10 11 12 13 14 15 16 17
|
| +
|
| + ; xmm0 0
|
| + ; xmm1 4
|
| + ; xmm2 1
|
| + ; xmm3 3
|
| +
|
| + movdqa xmm5, xmm0
|
| + movdqa xmm2, xmm1
|
| +
|
| + paddw xmm0, xmm3 ; a1 = 0 + 3
|
| + paddw xmm1, xmm4 ; b1 = 1 + 2
|
| +
|
| + psubw xmm4, xmm2 ; c1 = 1 - 2
|
| + psubw xmm5, xmm3 ; d1 = 0 - 3
|
| +
|
| + pxor xmm6, xmm6 ; zero out for compare
|
| +
|
| + pcmpeqw xmm6, xmm5 ; d1 != 0
|
| +
|
| + pandn xmm6, XMMWORD PTR[GLOBAL(_cmp_mask8x4)] ; clear upper,
|
| + ; and keep bit 0 of lower
|
| +
|
| + ; output 0 and 2
|
| + movdqa xmm2, xmm0 ; a1
|
| +
|
| + paddw xmm0, xmm1 ; a1 + b1
|
| + psubw xmm2, xmm1 ; a1 - b1
|
| +
|
| + paddw xmm0, XMMWORD PTR[GLOBAL(_7w)]
|
| + paddw xmm2, XMMWORD PTR[GLOBAL(_7w)]
|
| +
|
| + psraw xmm0, 4 ; op[0] = (a1 + b1 + 7)>>4
|
| + psraw xmm2, 4 ; op[8] = (a1 - b1 + 7)>>4
|
| +
|
| + ; output 1 and 3
|
| + ; interleave c1, d1
|
| + movdqa xmm1, xmm5 ; d1
|
| + punpcklwd xmm1, xmm4 ; c1 d1
|
| + punpckhwd xmm5, xmm4 ; c1 d1
|
| +
|
| + movdqa xmm3, xmm1
|
| + movdqa xmm4, xmm5
|
| +
|
| + pmaddwd xmm1, XMMWORD PTR[GLOBAL (_5352_2217)] ; c1*2217 + d1*5352
|
| + pmaddwd xmm4, XMMWORD PTR[GLOBAL (_5352_2217)] ; c1*2217 + d1*5352
|
| +
|
| + pmaddwd xmm3, XMMWORD PTR[GLOBAL(_2217_neg5352)] ; d1*2217 - c1*5352
|
| + pmaddwd xmm5, XMMWORD PTR[GLOBAL(_2217_neg5352)] ; d1*2217 - c1*5352
|
| +
|
| + paddd xmm1, XMMWORD PTR[GLOBAL(_12000)]
|
| + paddd xmm4, XMMWORD PTR[GLOBAL(_12000)]
|
| + paddd xmm3, XMMWORD PTR[GLOBAL(_51000)]
|
| + paddd xmm5, XMMWORD PTR[GLOBAL(_51000)]
|
| +
|
| + psrad xmm1, 16 ; (c1 * 2217 + d1 * 5352 + 14500)>>16
|
| + psrad xmm4, 16 ; (c1 * 2217 + d1 * 5352 + 14500)>>16
|
| + psrad xmm3, 16 ; (d1 * 2217 - c1 * 5352 + 7500)>>16
|
| + psrad xmm5, 16 ; (d1 * 2217 - c1 * 5352 + 7500)>>16
|
| +
|
| + packssdw xmm1, xmm4 ; op[4]
|
| + packssdw xmm3, xmm5 ; op[12]
|
| +
|
| + paddw xmm1, xmm6 ; op[4] += (d1!=0)
|
| +
|
| + movdqa xmm4, xmm0
|
| + movdqa xmm5, xmm2
|
| +
|
| + punpcklqdq xmm0, xmm1
|
| + punpckhqdq xmm4, xmm1
|
| +
|
| + punpcklqdq xmm2, xmm3
|
| + punpckhqdq xmm5, xmm3
|
| +
|
| + movdqa XMMWORD PTR[output + 0 ], xmm0
|
| + movdqa XMMWORD PTR[output + 16], xmm2
|
| + movdqa XMMWORD PTR[output + 32], xmm4
|
| + movdqa XMMWORD PTR[output + 48], xmm5
|
| +
|
| + STACK_FRAME_DESTROY
|
| +
|
| +SECTION_RODATA
|
| +align 16
|
| +_5352_2217:
|
| + dw 5352
|
| + dw 2217
|
| + dw 5352
|
| + dw 2217
|
| + dw 5352
|
| + dw 2217
|
| + dw 5352
|
| + dw 2217
|
| +align 16
|
| +_2217_neg5352:
|
| + dw 2217
|
| + dw -5352
|
| + dw 2217
|
| + dw -5352
|
| + dw 2217
|
| + dw -5352
|
| + dw 2217
|
| + dw -5352
|
| +align 16
|
| +_mult_add:
|
| + times 8 dw 1
|
| +align 16
|
| +_cmp_mask:
|
| + times 4 dw 1
|
| + times 4 dw 0
|
| +align 16
|
| +_cmp_mask8x4:
|
| + times 8 dw 1
|
| +align 16
|
| +_mult_sub:
|
| + dw 1
|
| + dw -1
|
| + dw 1
|
| + dw -1
|
| + dw 1
|
| + dw -1
|
| + dw 1
|
| + dw -1
|
| +align 16
|
| +_7:
|
| + times 4 dd 7
|
| +align 16
|
| +_7w:
|
| + times 8 dw 7
|
| +align 16
|
| +_14500:
|
| + times 4 dd 14500
|
| +align 16
|
| +_7500:
|
| + times 4 dd 7500
|
| +align 16
|
| +_12000:
|
| + times 4 dd 12000
|
| +align 16
|
| +_51000:
|
| + times 4 dd 51000
|
|
|