| OLD | NEW |
|---|
| (Empty) | |
| 1 ;***************************************************************************** |
| 2 ;* x86inc.asm |
| 3 ;***************************************************************************** |
| 4 ;* Copyright (C) 2005-2011 x264 project |
| 5 ;* |
| 6 ;* Authors: Loren Merritt <lorenm@u.washington.edu> |
| 7 ;* Anton Mitrofanov <BugMaster@narod.ru> |
| 8 ;* Jason Garrett-Glaser <darkshikari@gmail.com> |
| 9 ;* |
| 10 ;* Permission to use, copy, modify, and/or distribute this software for any |
| 11 ;* purpose with or without fee is hereby granted, provided that the above |
| 12 ;* copyright notice and this permission notice appear in all copies. |
| 13 ;* |
| 14 ;* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES |
| 15 ;* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF |
| 16 ;* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR |
| 17 ;* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES |
| 18 ;* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN |
| 19 ;* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF |
| 20 ;* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. |
| 21 ;***************************************************************************** |
| 22 |
| 23 ; This is a header file for the x264ASM assembly language, which uses |
| 24 ; NASM/YASM syntax combined with a large number of macros to provide easy |
| 25 ; abstraction between different calling conventions (x86_32, win64, linux64). |
| 26 ; It also has various other useful features to simplify writing the kind of |
| 27 ; DSP functions that are most often used in x264. |
| 28 |
| 29 ; Unlike the rest of x264, this file is available under an ISC license, as it |
| 30 ; has significant usefulness outside of x264 and we want it to be available |
| 31 ; to the largest audience possible. Of course, if you modify it for your own |
| 32 ; purposes to add a new feature, we strongly encourage contributing a patch |
| 33 ; as this feature might be useful for others as well. Send patches or ideas |
| 34 ; to x264-devel@videolan.org . |
| 35 |
| 36 %ifndef MEDIA_BASE_SIMD_X86INC_ASM_ |
| 37 %define MEDIA_BASE_SIMD_X86INC_ASM_ |
| 38 |
| 39 %define program_name ff |
| 40 |
| 41 %ifdef ARCH_X86_64 |
| 42 %ifidn __OUTPUT_FORMAT__,win32 |
| 43 %define WIN64 |
| 44 %else |
| 45 %define UNIX64 |
| 46 %endif |
| 47 %endif |
| 48 |
| 49 %ifdef PREFIX |
| 50 %define mangle(x) _ %+ x |
| 51 %else |
| 52 %define mangle(x) x |
| 53 %endif |
| 54 |
| 55 ; FIXME: All of the 64bit asm functions that take a stride as an argument |
| 56 ; via register, assume that the high dword of that register is filled with 0. |
| 57 ; This is true in practice (since we never do any 64bit arithmetic on strides, |
| 58 ; and x264's strides are all positive), but is not guaranteed by the ABI. |
| 59 |
| 60 ; Name of the .rodata section. |
| 61 ; Kludge: Something on OS X fails to align .rodata even given an align attribute
, |
| 62 ; so use a different read-only section. |
| 63 %ifdef CHROMIUM |
| 64 %macro SECTION_RODATA 0-1 16 |
| 65 %ifidn __OUTPUT_FORMAT__,macho64 |
| 66 SECTION .text align=%1 |
| 67 %elifidn __OUTPUT_FORMAT__,macho |
| 68 SECTION .text align=%1 |
| 69 fakegot: |
| 70 %elifidn __OUTPUT_FORMAT__,aout |
| 71 section .text |
| 72 %else |
| 73 SECTION .rodata align=%1 |
| 74 %endif |
| 75 %endmacro |
| 76 %else |
| 77 %macro SECTION_RODATA 0-1 16 |
| 78 %ifidn __OUTPUT_FORMAT__,aout |
| 79 section .text |
| 80 %else |
| 81 SECTION .rodata align=%1 |
| 82 %endif |
| 83 %endmacro |
| 84 %endif |
| 85 |
| 86 ; aout does not support align= |
| 87 %macro SECTION_TEXT 0-1 16 |
| 88 %ifidn __OUTPUT_FORMAT__,aout |
| 89 SECTION .text |
| 90 %else |
| 91 SECTION .text align=%1 |
| 92 %endif |
| 93 %endmacro |
| 94 |
| 95 %ifdef WIN64 |
| 96 %define PIC |
| 97 %elifndef ARCH_X86_64 |
| 98 ; x86_32 doesn't require PIC. |
| 99 ; Some distros prefer shared objects to be PIC, but nothing breaks if |
| 100 ; the code contains a few textrels, so we'll skip that complexity. |
| 101 %undef PIC |
| 102 %endif |
| 103 %ifdef PIC |
| 104 default rel |
| 105 %endif |
| 106 |
| 107 ; Macros to eliminate most code duplication between x86_32 and x86_64: |
| 108 ; Currently this works only for leaf functions which load all their arguments |
| 109 ; into registers at the start, and make no other use of the stack. Luckily that |
| 110 ; covers most of x264's asm. |
| 111 |
| 112 ; PROLOGUE: |
| 113 ; %1 = number of arguments. loads them from stack if needed. |
| 114 ; %2 = number of registers used. pushes callee-saved regs if needed. |
| 115 ; %3 = number of xmm registers used. pushes callee-saved xmm regs if needed. |
| 116 ; %4 = list of names to define to registers |
| 117 ; PROLOGUE can also be invoked by adding the same options to cglobal |
| 118 |
| 119 ; e.g. |
| 120 ; cglobal foo, 2,3,0, dst, src, tmp |
| 121 ; declares a function (foo), taking two args (dst and src) and one local variabl
e (tmp) |
| 122 |
| 123 ; TODO Some functions can use some args directly from the stack. If they're the |
| 124 ; last args then you can just not declare them, but if they're in the middle |
| 125 ; we need more flexible macro. |
| 126 |
| 127 ; RET: |
| 128 ; Pops anything that was pushed by PROLOGUE |
| 129 |
| 130 ; REP_RET: |
| 131 ; Same, but if it doesn't pop anything it becomes a 2-byte ret, for athlons |
| 132 ; which are slow when a normal ret follows a branch. |
| 133 |
| 134 ; registers: |
| 135 ; rN and rNq are the native-size register holding function argument N |
| 136 ; rNd, rNw, rNb are dword, word, and byte size |
| 137 ; rNm is the original location of arg N (a register or on the stack), dword |
| 138 ; rNmp is native size |
| 139 |
| 140 %macro DECLARE_REG 6 |
| 141 %define r%1q %2 |
| 142 %define r%1d %3 |
| 143 %define r%1w %4 |
| 144 %define r%1b %5 |
| 145 %define r%1m %6 |
| 146 %ifid %6 ; i.e. it's a register |
| 147 %define r%1mp %2 |
| 148 %elifdef ARCH_X86_64 ; memory |
| 149 %define r%1mp qword %6 |
| 150 %else |
| 151 %define r%1mp dword %6 |
| 152 %endif |
| 153 %define r%1 %2 |
| 154 %endmacro |
| 155 |
| 156 %macro DECLARE_REG_SIZE 2 |
| 157 %define r%1q r%1 |
| 158 %define e%1q r%1 |
| 159 %define r%1d e%1 |
| 160 %define e%1d e%1 |
| 161 %define r%1w %1 |
| 162 %define e%1w %1 |
| 163 %define r%1b %2 |
| 164 %define e%1b %2 |
| 165 %ifndef ARCH_X86_64 |
| 166 %define r%1 e%1 |
| 167 %endif |
| 168 %endmacro |
| 169 |
| 170 DECLARE_REG_SIZE ax, al |
| 171 DECLARE_REG_SIZE bx, bl |
| 172 DECLARE_REG_SIZE cx, cl |
| 173 DECLARE_REG_SIZE dx, dl |
| 174 DECLARE_REG_SIZE si, sil |
| 175 DECLARE_REG_SIZE di, dil |
| 176 DECLARE_REG_SIZE bp, bpl |
| 177 |
| 178 ; t# defines for when per-arch register allocation is more complex than just fun
ction arguments |
| 179 |
| 180 %macro DECLARE_REG_TMP 1-* |
| 181 %assign %%i 0 |
| 182 %rep %0 |
| 183 CAT_XDEFINE t, %%i, r%1 |
| 184 %assign %%i %%i+1 |
| 185 %rotate 1 |
| 186 %endrep |
| 187 %endmacro |
| 188 |
| 189 %macro DECLARE_REG_TMP_SIZE 0-* |
| 190 %rep %0 |
| 191 %define t%1q t%1 %+ q |
| 192 %define t%1d t%1 %+ d |
| 193 %define t%1w t%1 %+ w |
| 194 %define t%1b t%1 %+ b |
| 195 %rotate 1 |
| 196 %endrep |
| 197 %endmacro |
| 198 |
| 199 DECLARE_REG_TMP_SIZE 0,1,2,3,4,5,6,7,8,9 |
| 200 |
| 201 %ifdef ARCH_X86_64 |
| 202 %define gprsize 8 |
| 203 %else |
| 204 %define gprsize 4 |
| 205 %endif |
| 206 |
| 207 %macro PUSH 1 |
| 208 push %1 |
| 209 %assign stack_offset stack_offset+gprsize |
| 210 %endmacro |
| 211 |
| 212 %macro POP 1 |
| 213 pop %1 |
| 214 %assign stack_offset stack_offset-gprsize |
| 215 %endmacro |
| 216 |
| 217 %macro SUB 2 |
| 218 sub %1, %2 |
| 219 %ifidn %1, rsp |
| 220 %assign stack_offset stack_offset+(%2) |
| 221 %endif |
| 222 %endmacro |
| 223 |
| 224 %macro ADD 2 |
| 225 add %1, %2 |
| 226 %ifidn %1, rsp |
| 227 %assign stack_offset stack_offset-(%2) |
| 228 %endif |
| 229 %endmacro |
| 230 |
| 231 %macro movifnidn 2 |
| 232 %ifnidn %1, %2 |
| 233 mov %1, %2 |
| 234 %endif |
| 235 %endmacro |
| 236 |
| 237 %macro movsxdifnidn 2 |
| 238 %ifnidn %1, %2 |
| 239 movsxd %1, %2 |
| 240 %endif |
| 241 %endmacro |
| 242 |
| 243 %macro ASSERT 1 |
| 244 %if (%1) == 0 |
| 245 %error assert failed |
| 246 %endif |
| 247 %endmacro |
| 248 |
| 249 %macro DEFINE_ARGS 0-* |
| 250 %ifdef n_arg_names |
| 251 %assign %%i 0 |
| 252 %rep n_arg_names |
| 253 CAT_UNDEF arg_name %+ %%i, q |
| 254 CAT_UNDEF arg_name %+ %%i, d |
| 255 CAT_UNDEF arg_name %+ %%i, w |
| 256 CAT_UNDEF arg_name %+ %%i, b |
| 257 CAT_UNDEF arg_name %+ %%i, m |
| 258 CAT_UNDEF arg_name, %%i |
| 259 %assign %%i %%i+1 |
| 260 %endrep |
| 261 %endif |
| 262 |
| 263 %assign %%i 0 |
| 264 %rep %0 |
| 265 %xdefine %1q r %+ %%i %+ q |
| 266 %xdefine %1d r %+ %%i %+ d |
| 267 %xdefine %1w r %+ %%i %+ w |
| 268 %xdefine %1b r %+ %%i %+ b |
| 269 %xdefine %1m r %+ %%i %+ m |
| 270 CAT_XDEFINE arg_name, %%i, %1 |
| 271 %assign %%i %%i+1 |
| 272 %rotate 1 |
| 273 %endrep |
| 274 %assign n_arg_names %%i |
| 275 %endmacro |
| 276 |
| 277 %ifdef WIN64 ; Windows x64 ;================================================= |
| 278 |
| 279 DECLARE_REG 0, rcx, ecx, cx, cl, ecx |
| 280 DECLARE_REG 1, rdx, edx, dx, dl, edx |
| 281 DECLARE_REG 2, r8, r8d, r8w, r8b, r8d |
| 282 DECLARE_REG 3, r9, r9d, r9w, r9b, r9d |
| 283 DECLARE_REG 4, rdi, edi, di, dil, [rsp + stack_offset + 40] |
| 284 DECLARE_REG 5, rsi, esi, si, sil, [rsp + stack_offset + 48] |
| 285 DECLARE_REG 6, rax, eax, ax, al, [rsp + stack_offset + 56] |
| 286 %define r7m [rsp + stack_offset + 64] |
| 287 %define r8m [rsp + stack_offset + 72] |
| 288 |
| 289 %macro LOAD_IF_USED 2 ; reg_id, number_of_args |
| 290 %if %1 < %2 |
| 291 mov r%1, [rsp + stack_offset + 8 + %1*8] |
| 292 %endif |
| 293 %endmacro |
| 294 |
| 295 %macro PROLOGUE 2-4+ 0 ; #args, #regs, #xmm_regs, arg_names... |
| 296 ASSERT %2 >= %1 |
| 297 %assign regs_used %2 |
| 298 ASSERT regs_used <= 7 |
| 299 %if regs_used > 4 |
| 300 push r4 |
| 301 push r5 |
| 302 %assign stack_offset stack_offset+16 |
| 303 %endif |
| 304 WIN64_SPILL_XMM %3 |
| 305 LOAD_IF_USED 4, %1 |
| 306 LOAD_IF_USED 5, %1 |
| 307 LOAD_IF_USED 6, %1 |
| 308 DEFINE_ARGS %4 |
| 309 %endmacro |
| 310 |
| 311 %macro WIN64_SPILL_XMM 1 |
| 312 %assign xmm_regs_used %1 |
| 313 ASSERT xmm_regs_used <= 16 |
| 314 %if xmm_regs_used > 6 |
| 315 sub rsp, (xmm_regs_used-6)*16+16 |
| 316 %assign stack_offset stack_offset+(xmm_regs_used-6)*16+16 |
| 317 %assign %%i xmm_regs_used |
| 318 %rep (xmm_regs_used-6) |
| 319 %assign %%i %%i-1 |
| 320 movdqa [rsp + (%%i-6)*16+8], xmm %+ %%i |
| 321 %endrep |
| 322 %endif |
| 323 %endmacro |
| 324 |
| 325 %macro WIN64_RESTORE_XMM_INTERNAL 1 |
| 326 %if xmm_regs_used > 6 |
| 327 %assign %%i xmm_regs_used |
| 328 %rep (xmm_regs_used-6) |
| 329 %assign %%i %%i-1 |
| 330 movdqa xmm %+ %%i, [%1 + (%%i-6)*16+8] |
| 331 %endrep |
| 332 add %1, (xmm_regs_used-6)*16+16 |
| 333 %endif |
| 334 %endmacro |
| 335 |
| 336 %macro WIN64_RESTORE_XMM 1 |
| 337 WIN64_RESTORE_XMM_INTERNAL %1 |
| 338 %assign stack_offset stack_offset-(xmm_regs_used-6)*16+16 |
| 339 %assign xmm_regs_used 0 |
| 340 %endmacro |
| 341 |
| 342 %macro RET 0 |
| 343 WIN64_RESTORE_XMM_INTERNAL rsp |
| 344 %if regs_used > 4 |
| 345 pop r5 |
| 346 pop r4 |
| 347 %endif |
| 348 ret |
| 349 %endmacro |
| 350 |
| 351 %macro REP_RET 0 |
| 352 %if regs_used > 4 || xmm_regs_used > 6 |
| 353 RET |
| 354 %else |
| 355 rep ret |
| 356 %endif |
| 357 %endmacro |
| 358 |
| 359 %elifdef ARCH_X86_64 ; *nix x64 ;============================================= |
| 360 |
| 361 DECLARE_REG 0, rdi, edi, di, dil, edi |
| 362 DECLARE_REG 1, rsi, esi, si, sil, esi |
| 363 DECLARE_REG 2, rdx, edx, dx, dl, edx |
| 364 DECLARE_REG 3, rcx, ecx, cx, cl, ecx |
| 365 DECLARE_REG 4, r8, r8d, r8w, r8b, r8d |
| 366 DECLARE_REG 5, r9, r9d, r9w, r9b, r9d |
| 367 DECLARE_REG 6, rax, eax, ax, al, [rsp + stack_offset + 8] |
| 368 %define r7m [rsp + stack_offset + 16] |
| 369 %define r8m [rsp + stack_offset + 24] |
| 370 |
| 371 %macro LOAD_IF_USED 2 ; reg_id, number_of_args |
| 372 %if %1 < %2 |
| 373 mov r%1, [rsp - 40 + %1*8] |
| 374 %endif |
| 375 %endmacro |
| 376 |
| 377 %macro PROLOGUE 2-4+ ; #args, #regs, #xmm_regs, arg_names... |
| 378 ASSERT %2 >= %1 |
| 379 ASSERT %2 <= 7 |
| 380 LOAD_IF_USED 6, %1 |
| 381 DEFINE_ARGS %4 |
| 382 %endmacro |
| 383 |
| 384 %macro RET 0 |
| 385 ret |
| 386 %endmacro |
| 387 |
| 388 %macro REP_RET 0 |
| 389 rep ret |
| 390 %endmacro |
| 391 |
| 392 %else ; X86_32 ;============================================================== |
| 393 |
| 394 ; Begin chromium edits |
| 395 %ifdef CHROMIUM |
| 396 ; Change the order of registers so we can get the lower 8-bit or the 5th and 6th |
| 397 ; arguments. |
| 398 DECLARE_REG 0, esi, esi, si, null, [esp + stack_offset + 4] |
| 399 DECLARE_REG 1, edi, edi, di, null, [esp + stack_offset + 8] |
| 400 DECLARE_REG 2, ecx, ecx, cx, cl, [esp + stack_offset + 12] |
| 401 DECLARE_REG 3, edx, edx, dx, dl, [esp + stack_offset + 16] |
| 402 DECLARE_REG 4, eax, eax, ax, al, [esp + stack_offset + 20] |
| 403 DECLARE_REG 5, ebx, ebx, bx, bl, [esp + stack_offset + 24] |
| 404 %else |
| 405 DECLARE_REG 0, eax, eax, ax, al, [esp + stack_offset + 4] |
| 406 DECLARE_REG 1, ecx, ecx, cx, cl, [esp + stack_offset + 8] |
| 407 DECLARE_REG 2, edx, edx, dx, dl, [esp + stack_offset + 12] |
| 408 DECLARE_REG 3, ebx, ebx, bx, bl, [esp + stack_offset + 16] |
| 409 DECLARE_REG 4, esi, esi, si, null, [esp + stack_offset + 20] |
| 410 DECLARE_REG 5, edi, edi, di, null, [esp + stack_offset + 24] |
| 411 %endif |
| 412 ; End chromium edits |
| 413 DECLARE_REG 6, ebp, ebp, bp, null, [esp + stack_offset + 28] |
| 414 %define r7m [esp + stack_offset + 32] |
| 415 %define r8m [esp + stack_offset + 36] |
| 416 %define rsp esp |
| 417 |
| 418 %macro PUSH_IF_USED 1 ; reg_id |
| 419 %if %1 < regs_used |
| 420 push r%1 |
| 421 %assign stack_offset stack_offset+4 |
| 422 %endif |
| 423 %endmacro |
| 424 |
| 425 %macro POP_IF_USED 1 ; reg_id |
| 426 %if %1 < regs_used |
| 427 pop r%1 |
| 428 %endif |
| 429 %endmacro |
| 430 |
| 431 %macro LOAD_IF_USED 2 ; reg_id, number_of_args |
| 432 %if %1 < %2 |
| 433 mov r%1, [esp + stack_offset + 4 + %1*4] |
| 434 %endif |
| 435 %endmacro |
| 436 |
| 437 %macro PROLOGUE 2-4+ ; #args, #regs, #xmm_regs, arg_names... |
| 438 ASSERT %2 >= %1 |
| 439 %assign regs_used %2 |
| 440 ASSERT regs_used <= 7 |
| 441 %ifdef CHROMIUM |
| 442 PUSH_IF_USED 0 |
| 443 PUSH_IF_USED 1 |
| 444 PUSH_IF_USED 5 |
| 445 %else |
| 446 PUSH_IF_USED 3 |
| 447 PUSH_IF_USED 4 |
| 448 PUSH_IF_USED 5 |
| 449 %endif |
| 450 PUSH_IF_USED 6 |
| 451 LOAD_IF_USED 0, %1 |
| 452 LOAD_IF_USED 1, %1 |
| 453 LOAD_IF_USED 2, %1 |
| 454 LOAD_IF_USED 3, %1 |
| 455 LOAD_IF_USED 4, %1 |
| 456 LOAD_IF_USED 5, %1 |
| 457 LOAD_IF_USED 6, %1 |
| 458 DEFINE_ARGS %4 |
| 459 %endmacro |
| 460 |
| 461 %macro RET 0 |
| 462 POP_IF_USED 6 |
| 463 %ifdef CHROMIUM |
| 464 POP_IF_USED 5 |
| 465 POP_IF_USED 1 |
| 466 POP_IF_USED 0 |
| 467 %else |
| 468 POP_IF_USED 5 |
| 469 POP_IF_USED 4 |
| 470 POP_IF_USED 3 |
| 471 %endif |
| 472 ret |
| 473 %endmacro |
| 474 |
| 475 %macro REP_RET 0 |
| 476 %if regs_used > 3 |
| 477 RET |
| 478 %else |
| 479 rep ret |
| 480 %endif |
| 481 %endmacro |
| 482 |
| 483 %endif ;====================================================================== |
| 484 |
| 485 %ifndef WIN64 |
| 486 %macro WIN64_SPILL_XMM 1 |
| 487 %endmacro |
| 488 %macro WIN64_RESTORE_XMM 1 |
| 489 %endmacro |
| 490 %endif |
| 491 |
| 492 |
| 493 |
| 494 ;============================================================================= |
| 495 ; arch-independent part |
| 496 ;============================================================================= |
| 497 |
| 498 %assign function_align 16 |
| 499 |
| 500 ; Symbol prefix for C linkage |
| 501 %macro cglobal 1-2+ |
| 502 %xdefine %1 mangle(program_name %+ _ %+ %1) |
| 503 %xdefine %1.skip_prologue %1 %+ .skip_prologue |
| 504 %ifidn __OUTPUT_FORMAT__,elf |
| 505 global %1:function hidden |
| 506 %else |
| 507 global %1 |
| 508 %endif |
| 509 align function_align |
| 510 %1: |
| 511 RESET_MM_PERMUTATION ; not really needed, but makes disassembly somewhat nic
er |
| 512 %assign stack_offset 0 |
| 513 %if %0 > 1 |
| 514 PROLOGUE %2 |
| 515 %endif |
| 516 %endmacro |
| 517 |
| 518 %macro cextern 1 |
| 519 %xdefine %1 mangle(program_name %+ _ %+ %1) |
| 520 extern %1 |
| 521 %endmacro |
| 522 |
| 523 ;like cextern, but without the prefix |
| 524 %macro cextern_naked 1 |
| 525 %xdefine %1 mangle(%1) |
| 526 extern %1 |
| 527 %endmacro |
| 528 |
| 529 %macro const 2+ |
| 530 %xdefine %1 mangle(program_name %+ _ %+ %1) |
| 531 global %1 |
| 532 %1: %2 |
| 533 %endmacro |
| 534 |
| 535 ; This is needed for ELF, otherwise the GNU linker assumes the stack is |
| 536 ; executable by default. |
| 537 %ifidn __OUTPUT_FORMAT__,elf |
| 538 SECTION .note.GNU-stack noalloc noexec nowrite progbits |
| 539 %endif |
| 540 |
| 541 ; merge mmx and sse* |
| 542 |
| 543 %macro CAT_XDEFINE 3 |
| 544 %xdefine %1%2 %3 |
| 545 %endmacro |
| 546 |
| 547 %macro CAT_UNDEF 2 |
| 548 %undef %1%2 |
| 549 %endmacro |
| 550 |
| 551 %macro INIT_MMX 0 |
| 552 %assign avx_enabled 0 |
| 553 %define RESET_MM_PERMUTATION INIT_MMX |
| 554 %define mmsize 8 |
| 555 %define num_mmregs 8 |
| 556 %define mova movq |
| 557 %define movu movq |
| 558 %define movh movd |
| 559 %define movnta movntq |
| 560 %assign %%i 0 |
| 561 %rep 8 |
| 562 CAT_XDEFINE m, %%i, mm %+ %%i |
| 563 CAT_XDEFINE nmm, %%i, %%i |
| 564 %assign %%i %%i+1 |
| 565 %endrep |
| 566 %rep 8 |
| 567 CAT_UNDEF m, %%i |
| 568 CAT_UNDEF nmm, %%i |
| 569 %assign %%i %%i+1 |
| 570 %endrep |
| 571 %endmacro |
| 572 |
| 573 %macro INIT_XMM 0 |
| 574 %assign avx_enabled 0 |
| 575 %define RESET_MM_PERMUTATION INIT_XMM |
| 576 %define mmsize 16 |
| 577 %define num_mmregs 8 |
| 578 %ifdef ARCH_X86_64 |
| 579 %define num_mmregs 16 |
| 580 %endif |
| 581 %define mova movdqa |
| 582 %define movu movdqu |
| 583 %define movh movq |
| 584 %define movnta movntdq |
| 585 %assign %%i 0 |
| 586 %rep num_mmregs |
| 587 CAT_XDEFINE m, %%i, xmm %+ %%i |
| 588 CAT_XDEFINE nxmm, %%i, %%i |
| 589 %assign %%i %%i+1 |
| 590 %endrep |
| 591 %endmacro |
| 592 |
| 593 %macro INIT_AVX 0 |
| 594 INIT_XMM |
| 595 %assign avx_enabled 1 |
| 596 %define PALIGNR PALIGNR_SSSE3 |
| 597 %define RESET_MM_PERMUTATION INIT_AVX |
| 598 %endmacro |
| 599 |
| 600 %macro INIT_YMM 0 |
| 601 %assign avx_enabled 1 |
| 602 %define RESET_MM_PERMUTATION INIT_YMM |
| 603 %define mmsize 32 |
| 604 %define num_mmregs 8 |
| 605 %ifdef ARCH_X86_64 |
| 606 %define num_mmregs 16 |
| 607 %endif |
| 608 %define mova vmovaps |
| 609 %define movu vmovups |
| 610 %assign %%i 0 |
| 611 %rep num_mmregs |
| 612 CAT_XDEFINE m, %%i, ymm %+ %%i |
| 613 CAT_XDEFINE nymm, %%i, %%i |
| 614 %assign %%i %%i+1 |
| 615 %endrep |
| 616 %endmacro |
| 617 |
| 618 INIT_MMX |
| 619 |
| 620 ; I often want to use macros that permute their arguments. e.g. there's no |
| 621 ; efficient way to implement butterfly or transpose or dct without swapping some |
| 622 ; arguments. |
| 623 ; |
| 624 ; I would like to not have to manually keep track of the permutations: |
| 625 ; If I insert a permutation in the middle of a function, it should automatically |
| 626 ; change everything that follows. For more complex macros I may also have multip
le |
| 627 ; implementations, e.g. the SSE2 and SSSE3 versions may have different permutati
ons. |
| 628 ; |
| 629 ; Hence these macros. Insert a PERMUTE or some SWAPs at the end of a macro that |
| 630 ; permutes its arguments. It's equivalent to exchanging the contents of the |
| 631 ; registers, except that this way you exchange the register names instead, so it |
| 632 ; doesn't cost any cycles. |
| 633 |
| 634 %macro PERMUTE 2-* ; takes a list of pairs to swap |
| 635 %rep %0/2 |
| 636 %xdefine tmp%2 m%2 |
| 637 %xdefine ntmp%2 nm%2 |
| 638 %rotate 2 |
| 639 %endrep |
| 640 %rep %0/2 |
| 641 %xdefine m%1 tmp%2 |
| 642 %xdefine nm%1 ntmp%2 |
| 643 %undef tmp%2 |
| 644 %undef ntmp%2 |
| 645 %rotate 2 |
| 646 %endrep |
| 647 %endmacro |
| 648 |
| 649 %macro SWAP 2-* ; swaps a single chain (sometimes more concise than pairs) |
| 650 %rep %0-1 |
| 651 %ifdef m%1 |
| 652 %xdefine tmp m%1 |
| 653 %xdefine m%1 m%2 |
| 654 %xdefine m%2 tmp |
| 655 CAT_XDEFINE n, m%1, %1 |
| 656 CAT_XDEFINE n, m%2, %2 |
| 657 %else |
| 658 ; If we were called as "SWAP m0,m1" rather than "SWAP 0,1" infer the origina
l numbers here. |
| 659 ; Be careful using this mode in nested macros though, as in some cases there
may be |
| 660 ; other copies of m# that have already been dereferenced and don't get updat
ed correctly. |
| 661 %xdefine %%n1 n %+ %1 |
| 662 %xdefine %%n2 n %+ %2 |
| 663 %xdefine tmp m %+ %%n1 |
| 664 CAT_XDEFINE m, %%n1, m %+ %%n2 |
| 665 CAT_XDEFINE m, %%n2, tmp |
| 666 CAT_XDEFINE n, m %+ %%n1, %%n1 |
| 667 CAT_XDEFINE n, m %+ %%n2, %%n2 |
| 668 %endif |
| 669 %undef tmp |
| 670 %rotate 1 |
| 671 %endrep |
| 672 %endmacro |
| 673 |
| 674 ; If SAVE_MM_PERMUTATION is placed at the end of a function and given the |
| 675 ; function name, then any later calls to that function will automatically |
| 676 ; load the permutation, so values can be returned in mmregs. |
| 677 %macro SAVE_MM_PERMUTATION 1 ; name to save as |
| 678 %assign %%i 0 |
| 679 %rep num_mmregs |
| 680 CAT_XDEFINE %1_m, %%i, m %+ %%i |
| 681 %assign %%i %%i+1 |
| 682 %endrep |
| 683 %endmacro |
| 684 |
| 685 %macro LOAD_MM_PERMUTATION 1 ; name to load from |
| 686 %assign %%i 0 |
| 687 %rep num_mmregs |
| 688 CAT_XDEFINE m, %%i, %1_m %+ %%i |
| 689 CAT_XDEFINE n, m %+ %%i, %%i |
| 690 %assign %%i %%i+1 |
| 691 %endrep |
| 692 %endmacro |
| 693 |
| 694 %macro call 1 |
| 695 call %1 |
| 696 %ifdef %1_m0 |
| 697 LOAD_MM_PERMUTATION %1 |
| 698 %endif |
| 699 %endmacro |
| 700 |
| 701 ; Substitutions that reduce instruction size but are functionally equivalent |
| 702 %macro add 2 |
| 703 %ifnum %2 |
| 704 %if %2==128 |
| 705 sub %1, -128 |
| 706 %else |
| 707 add %1, %2 |
| 708 %endif |
| 709 %else |
| 710 add %1, %2 |
| 711 %endif |
| 712 %endmacro |
| 713 |
| 714 %macro sub 2 |
| 715 %ifnum %2 |
| 716 %if %2==128 |
| 717 add %1, -128 |
| 718 %else |
| 719 sub %1, %2 |
| 720 %endif |
| 721 %else |
| 722 sub %1, %2 |
| 723 %endif |
| 724 %endmacro |
| 725 |
| 726 ;============================================================================= |
| 727 ; AVX abstraction layer |
| 728 ;============================================================================= |
| 729 |
| 730 %assign i 0 |
| 731 %rep 16 |
| 732 %if i < 8 |
| 733 CAT_XDEFINE sizeofmm, i, 8 |
| 734 %endif |
| 735 CAT_XDEFINE sizeofxmm, i, 16 |
| 736 CAT_XDEFINE sizeofymm, i, 32 |
| 737 %assign i i+1 |
| 738 %endrep |
| 739 %undef i |
| 740 |
| 741 ;%1 == instruction |
| 742 ;%2 == 1 if float, 0 if int |
| 743 ;%3 == 0 if 3-operand (xmm, xmm, xmm), 1 if 4-operand (xmm, xmm, xmm, imm) |
| 744 ;%4 == number of operands given |
| 745 ;%5+: operands |
| 746 %macro RUN_AVX_INSTR 6-7+ |
| 747 %if sizeof%5==32 |
| 748 v%1 %5, %6, %7 |
| 749 %else |
| 750 %if sizeof%5==8 |
| 751 %define %%regmov movq |
| 752 %elif %2 |
| 753 %define %%regmov movaps |
| 754 %else |
| 755 %define %%regmov movdqa |
| 756 %endif |
| 757 |
| 758 %if %4>=3+%3 |
| 759 %ifnidn %5, %6 |
| 760 %if avx_enabled && sizeof%5==16 |
| 761 v%1 %5, %6, %7 |
| 762 %else |
| 763 %%regmov %5, %6 |
| 764 %1 %5, %7 |
| 765 %endif |
| 766 %else |
| 767 %1 %5, %7 |
| 768 %endif |
| 769 %elif %3 |
| 770 %1 %5, %6, %7 |
| 771 %else |
| 772 %1 %5, %6 |
| 773 %endif |
| 774 %endif |
| 775 %endmacro |
| 776 |
| 777 ;%1 == instruction |
| 778 ;%2 == 1 if float, 0 if int |
| 779 ;%3 == 0 if 3-operand (xmm, xmm, xmm), 1 if 4-operand (xmm, xmm, xmm, imm) |
| 780 %macro AVX_INSTR 3 |
| 781 %macro %1 2-8 fnord, fnord, fnord, %1, %2, %3 |
| 782 %ifidn %3, fnord |
| 783 RUN_AVX_INSTR %6, %7, %8, 2, %1, %2 |
| 784 %elifidn %4, fnord |
| 785 RUN_AVX_INSTR %6, %7, %8, 3, %1, %2, %3 |
| 786 %elifidn %5, fnord |
| 787 RUN_AVX_INSTR %6, %7, %8, 4, %1, %2, %3, %4 |
| 788 %else |
| 789 RUN_AVX_INSTR %6, %7, %8, 5, %1, %2, %3, %4, %5 |
| 790 %endif |
| 791 %endmacro |
| 792 %endmacro |
| 793 |
| 794 AVX_INSTR addpd, 1, 0 |
| 795 AVX_INSTR addps, 1, 0 |
| 796 AVX_INSTR addsd, 1, 0 |
| 797 AVX_INSTR addss, 1, 0 |
| 798 AVX_INSTR addsubpd, 1, 0 |
| 799 AVX_INSTR addsubps, 1, 0 |
| 800 AVX_INSTR andpd, 1, 0 |
| 801 AVX_INSTR andps, 1, 0 |
| 802 AVX_INSTR andnpd, 1, 0 |
| 803 AVX_INSTR andnps, 1, 0 |
| 804 AVX_INSTR blendpd, 1, 0 |
| 805 AVX_INSTR blendps, 1, 0 |
| 806 AVX_INSTR blendvpd, 1, 0 |
| 807 AVX_INSTR blendvps, 1, 0 |
| 808 AVX_INSTR cmppd, 1, 0 |
| 809 AVX_INSTR cmpps, 1, 0 |
| 810 AVX_INSTR cmpsd, 1, 0 |
| 811 AVX_INSTR cmpss, 1, 0 |
| 812 AVX_INSTR divpd, 1, 0 |
| 813 AVX_INSTR divps, 1, 0 |
| 814 AVX_INSTR divsd, 1, 0 |
| 815 AVX_INSTR divss, 1, 0 |
| 816 AVX_INSTR dppd, 1, 0 |
| 817 AVX_INSTR dpps, 1, 0 |
| 818 AVX_INSTR haddpd, 1, 0 |
| 819 AVX_INSTR haddps, 1, 0 |
| 820 AVX_INSTR hsubpd, 1, 0 |
| 821 AVX_INSTR hsubps, 1, 0 |
| 822 AVX_INSTR maxpd, 1, 0 |
| 823 AVX_INSTR maxps, 1, 0 |
| 824 AVX_INSTR maxsd, 1, 0 |
| 825 AVX_INSTR maxss, 1, 0 |
| 826 AVX_INSTR minpd, 1, 0 |
| 827 AVX_INSTR minps, 1, 0 |
| 828 AVX_INSTR minsd, 1, 0 |
| 829 AVX_INSTR minss, 1, 0 |
| 830 AVX_INSTR mpsadbw, 0, 1 |
| 831 AVX_INSTR mulpd, 1, 0 |
| 832 AVX_INSTR mulps, 1, 0 |
| 833 AVX_INSTR mulsd, 1, 0 |
| 834 AVX_INSTR mulss, 1, 0 |
| 835 AVX_INSTR orpd, 1, 0 |
| 836 AVX_INSTR orps, 1, 0 |
| 837 AVX_INSTR packsswb, 0, 0 |
| 838 AVX_INSTR packssdw, 0, 0 |
| 839 AVX_INSTR packuswb, 0, 0 |
| 840 AVX_INSTR packusdw, 0, 0 |
| 841 AVX_INSTR paddb, 0, 0 |
| 842 AVX_INSTR paddw, 0, 0 |
| 843 AVX_INSTR paddd, 0, 0 |
| 844 AVX_INSTR paddq, 0, 0 |
| 845 AVX_INSTR paddsb, 0, 0 |
| 846 AVX_INSTR paddsw, 0, 0 |
| 847 AVX_INSTR paddusb, 0, 0 |
| 848 AVX_INSTR paddusw, 0, 0 |
| 849 AVX_INSTR palignr, 0, 1 |
| 850 AVX_INSTR pand, 0, 0 |
| 851 AVX_INSTR pandn, 0, 0 |
| 852 AVX_INSTR pavgb, 0, 0 |
| 853 AVX_INSTR pavgw, 0, 0 |
| 854 AVX_INSTR pblendvb, 0, 0 |
| 855 AVX_INSTR pblendw, 0, 1 |
| 856 AVX_INSTR pcmpestri, 0, 0 |
| 857 AVX_INSTR pcmpestrm, 0, 0 |
| 858 AVX_INSTR pcmpistri, 0, 0 |
| 859 AVX_INSTR pcmpistrm, 0, 0 |
| 860 AVX_INSTR pcmpeqb, 0, 0 |
| 861 AVX_INSTR pcmpeqw, 0, 0 |
| 862 AVX_INSTR pcmpeqd, 0, 0 |
| 863 AVX_INSTR pcmpeqq, 0, 0 |
| 864 AVX_INSTR pcmpgtb, 0, 0 |
| 865 AVX_INSTR pcmpgtw, 0, 0 |
| 866 AVX_INSTR pcmpgtd, 0, 0 |
| 867 AVX_INSTR pcmpgtq, 0, 0 |
| 868 AVX_INSTR phaddw, 0, 0 |
| 869 AVX_INSTR phaddd, 0, 0 |
| 870 AVX_INSTR phaddsw, 0, 0 |
| 871 AVX_INSTR phsubw, 0, 0 |
| 872 AVX_INSTR phsubd, 0, 0 |
| 873 AVX_INSTR phsubsw, 0, 0 |
| 874 AVX_INSTR pmaddwd, 0, 0 |
| 875 AVX_INSTR pmaddubsw, 0, 0 |
| 876 AVX_INSTR pmaxsb, 0, 0 |
| 877 AVX_INSTR pmaxsw, 0, 0 |
| 878 AVX_INSTR pmaxsd, 0, 0 |
| 879 AVX_INSTR pmaxub, 0, 0 |
| 880 AVX_INSTR pmaxuw, 0, 0 |
| 881 AVX_INSTR pmaxud, 0, 0 |
| 882 AVX_INSTR pminsb, 0, 0 |
| 883 AVX_INSTR pminsw, 0, 0 |
| 884 AVX_INSTR pminsd, 0, 0 |
| 885 AVX_INSTR pminub, 0, 0 |
| 886 AVX_INSTR pminuw, 0, 0 |
| 887 AVX_INSTR pminud, 0, 0 |
| 888 AVX_INSTR pmulhuw, 0, 0 |
| 889 AVX_INSTR pmulhrsw, 0, 0 |
| 890 AVX_INSTR pmulhw, 0, 0 |
| 891 AVX_INSTR pmullw, 0, 0 |
| 892 AVX_INSTR pmulld, 0, 0 |
| 893 AVX_INSTR pmuludq, 0, 0 |
| 894 AVX_INSTR pmuldq, 0, 0 |
| 895 AVX_INSTR por, 0, 0 |
| 896 AVX_INSTR psadbw, 0, 0 |
| 897 AVX_INSTR pshufb, 0, 0 |
| 898 AVX_INSTR psignb, 0, 0 |
| 899 AVX_INSTR psignw, 0, 0 |
| 900 AVX_INSTR psignd, 0, 0 |
| 901 AVX_INSTR psllw, 0, 0 |
| 902 AVX_INSTR pslld, 0, 0 |
| 903 AVX_INSTR psllq, 0, 0 |
| 904 AVX_INSTR pslldq, 0, 0 |
| 905 AVX_INSTR psraw, 0, 0 |
| 906 AVX_INSTR psrad, 0, 0 |
| 907 AVX_INSTR psrlw, 0, 0 |
| 908 AVX_INSTR psrld, 0, 0 |
| 909 AVX_INSTR psrlq, 0, 0 |
| 910 AVX_INSTR psrldq, 0, 0 |
| 911 AVX_INSTR psubb, 0, 0 |
| 912 AVX_INSTR psubw, 0, 0 |
| 913 AVX_INSTR psubd, 0, 0 |
| 914 AVX_INSTR psubq, 0, 0 |
| 915 AVX_INSTR psubsb, 0, 0 |
| 916 AVX_INSTR psubsw, 0, 0 |
| 917 AVX_INSTR psubusb, 0, 0 |
| 918 AVX_INSTR psubusw, 0, 0 |
| 919 AVX_INSTR punpckhbw, 0, 0 |
| 920 AVX_INSTR punpckhwd, 0, 0 |
| 921 AVX_INSTR punpckhdq, 0, 0 |
| 922 AVX_INSTR punpckhqdq, 0, 0 |
| 923 AVX_INSTR punpcklbw, 0, 0 |
| 924 AVX_INSTR punpcklwd, 0, 0 |
| 925 AVX_INSTR punpckldq, 0, 0 |
| 926 AVX_INSTR punpcklqdq, 0, 0 |
| 927 AVX_INSTR pxor, 0, 0 |
| 928 AVX_INSTR shufps, 0, 1 |
| 929 AVX_INSTR subpd, 1, 0 |
| 930 AVX_INSTR subps, 1, 0 |
| 931 AVX_INSTR subsd, 1, 0 |
| 932 AVX_INSTR subss, 1, 0 |
| 933 AVX_INSTR unpckhpd, 1, 0 |
| 934 AVX_INSTR unpckhps, 1, 0 |
| 935 AVX_INSTR unpcklpd, 1, 0 |
| 936 AVX_INSTR unpcklps, 1, 0 |
| 937 AVX_INSTR xorpd, 1, 0 |
| 938 AVX_INSTR xorps, 1, 0 |
| 939 |
| 940 ; 3DNow instructions, for sharing code between AVX, SSE and 3DN |
| 941 AVX_INSTR pfadd, 1, 0 |
| 942 AVX_INSTR pfsub, 1, 0 |
| 943 AVX_INSTR pfmul, 1, 0 |
| 944 |
| 945 ;============================================================================= |
| 946 ; Chromium extensions |
| 947 ;============================================================================= |
| 948 |
| 949 %ifdef CHROMIUM |
| 950 ; |
| 951 ; LOAD_SYM %1 (reg), %2 (sym) |
| 952 ; Copies the address to a local symbol to the specified register. |
| 953 ; |
| 954 |
| 955 %macro LOAD_SYM 2 |
| 956 |
| 957 %ifdef MACHO |
| 958 call %%geteip |
| 959 add %1, %2 - $ |
| 960 jmp %%end |
| 961 %%geteip: |
| 962 mov %1, [rsp] |
| 963 ret |
| 964 %%end: |
| 965 |
| 966 %else |
| 967 lea %1, [%2] |
| 968 %endif |
| 969 |
| 970 %endmacro |
| 971 |
| 972 ; |
| 973 ; MOVq %1 (xmm), %2 (reg) |
| 974 ; MOVq %1 (reg), %2 (xmm) |
| 975 ; Copies a general-purpose register to an XMM register, and vice versa. |
| 976 ; |
| 977 %macro MOVq 2 |
| 978 %if gprsize == 8 |
| 979 movq %1, %2 |
| 980 %else |
| 981 movd %1, %2 |
| 982 %endif |
| 983 %endmacro |
| 984 |
| 985 ; |
| 986 ; PRIVATE |
| 987 ; A flag representing the specified symbol is a private symbol. This define adds |
| 988 ; a hidden flag on Linux and a private_extern flag on Mac. (We can use this |
| 989 ; private_extern flag only on the latest yasm.) |
| 990 ; |
| 991 %ifdef MACHO |
| 992 %define PRIVATE :private_extern |
| 993 %elifdef ELF |
| 994 %define PRIVATE :hidden |
| 995 %else |
| 996 %define PRIVATE |
| 997 %endif |
| 998 |
| 999 %endif ; CHROMIUM |
| 1000 |
| 1001 %endif ; MEDIA_BASE_SIMD_X86INC_ASM_ |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 7003082:
Implements RGB to YV12 conversion in YASM. (Closed)
Base URL: svn://chrome-svn/chrome/trunk/src/