Implements RGB to YV12 conversion in YASM.

media/base/simd/convert_row_yuv.inc

+; Copyright (c) 2011 The Chromium Authors. All rights reserved.
+; Use of this source code is governed by a BSD-style license that can be
+; found in the LICENSE file.
+
+;
+; void media::simd::ConvertYUVtoRGBARow(const uint8* y,

[fbarchard, 2011/06/10 17:29:45]

ConvertYUVtoRGBARow
suggest ConvertYUVtoARGBRow

For Talk/WebRTC we do use ARGB, BGRA, RGBA, RGB and BGR.

+;                                       const uint8* u,
+;                                       const uint8* v,
+;                                       uint8* rgba,
+;                                       int width);
+; void media::simd::ConvertYUVtoRGBRow(const uint8* y,
+;                                      const uint8* u,
+;                                      const uint8* v,
+;                                      uint8* rgb,
+;                                      int width);
+;
+; Converts a row of YUV (4:1:1) pixels to the RGB colorspace. This function
+; reads YUV pixels from right to left, convert their colorspace to RGB, and
+; write the converted pixels to the output buffers. The following code snippet
+; represents the rough structure of this fucntion.
+;
+;   if (width & 1) {
+;     --width;
+;     convert_one_pixel(y, u, v, rgba, width);
+;   }
+;   if (width & 2) {
+;     width -= 2;
+;     convert_two_pixels(y, u, v, rgba, width);
+;   }
+;   while (width) {
+;     width -= 4;
+;     convert_four_pixels(y, u, v, rgba, width);
+;   }
+;
+; This function has pseudo-code snippets to describe its behaviors. For
+; simplicity, they use the following notations.
+;
+;   #define Y(n)      y[n] - 16
+;   #define U(n)      u[n] - 128
+;   #define V(n)      v[n] - 128
+;   #define FIX(n)    (n * (1 << SHIFT_BITS) * 0.5)
+;   #define ToByte(n) ((n < 0) ? 0 : (n > 255 ? 255 : n))
+;
+;   union {
+;     int32 d[4];
+;     int16 w[8];
+;     int8 b[16];
+;   } xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7
+;
+;   int ToByte(int n) {
+;     if (n < 0) return n;
+;     else if (n > 255) return 255;
+;     return n;
+;   }
+
+; Input Parameters
+%ifdef __x86_64__
+%ifdef WIN64
+%define PARAM_Y(r)      rcx
+%define PARAM_U(r)      rdx
+%define PARAM_V(r)      r8
+%define PARAM_RGBA(r)   r9
+%define PARAM_WIDTH(r)  INT [(r) + SIZEOF_POINTER * 2]
+%else
+%define PARAM_Y(r)      rdi
+%define PARAM_U(r)      rsi
+%define PARAM_V(r)      rdx
+%define PARAM_RGBA(r)   rcx
+%define PARAM_WIDTH(r)  r8
+%endif
+%else
+%define PARAM_Y(r)      POINTER [(r) + SIZEOF_POINTER * 2 + SIZEOF_POINTER * 0]
+%define PARAM_U(r)      POINTER [(r) + SIZEOF_POINTER * 2 + SIZEOF_POINTER * 1]
+%define PARAM_V(r)      POINTER [(r) + SIZEOF_POINTER * 2 + SIZEOF_POINTER * 2]
+%define PARAM_RGBA(r)   POINTER [(r) + SIZEOF_POINTER * 2 + SIZEOF_POINTER * 3]
+%define PARAM_WIDTH(r)  INT [(r) + SIZEOF_POINTER * 2 + SIZEOF_POINTER * 4]
+%endif  ; __x86_64__
+
+; Local variables used in this function.
+%define INT_WIDTH32     eax
+%define INT_WIDTH8      al
+
+%define INT_PIXEL32     ebx
+%define INT_PIXEL16     bx
+%define INT_PIXEL8      bl
+
+%ifdef __x86_64__
+%define INT_WIDTH       rax
+%define INT_PIXEL       rbx
+%define PTR_Y           PARAM_Y(_EBP_)
+%define PTR_U           PARAM_U(_EBP_)
+%define PTR_V           PARAM_V(_EBP_)
+%define PTR_RGBA        PARAM_RGBA(_EBP_)
+%else
+%define INT_WIDTH       eax
+%define INT_PIXEL       ebx
+%define PTR_RGBA        ecx
+%define PTR_Y           edx
+%define PTR_U           esi
+%define PTR_V           edi
+%endif
+
+%define XMM_TABLE0      xmm7
+%define XMM_TABLE1      xmm6
+%define XMM_RGBA        xmm5
+%define XMM_ZERO        xmm4
+
+  align     16
+  global    EXTN(SYMBOL_NAME) PRIVATE
+
+EXTN(SYMBOL_NAME):
+  push      _EBP_
+  mov       _EBP_, _ESP_
+
+  ; Save the resisters used in this function. We do not need to save eax, ecx,
+  ; or edx even though this function uses them. (Callers save them.)
+  push      _EBX_
+  push      _ESI_
+  push      _EDI_
+
+  ; Copy the tables used in this function to xmm6 and xmm7.
+  lea       _EAX_, [SIMD_ConvertYUVtoRGBA_kTable]
+  movdqa    XMM_TABLE0, XMMWORD [_EAX_ + 0]
+  movdqa    XMM_TABLE1, XMMWORD [_EAX_ + 16]
+
+  ; Copy the input parameters to registers. (We do not need to copy the input
+  ; pointers on x64 because x64 uses registers to provide the first five
+  ; parameters and we can use them.)
+  mov       INT_WIDTH, PARAM_WIDTH(_EBP_)
+%ifndef __x86_64__
+  mov       PTR_RGBA, PARAM_RGBA(_EBP_)
+  mov       PTR_Y, PARAM_Y(_EBP_)
+  mov       PTR_U, PARAM_U(_EBP_)
+  mov       PTR_V, PARAM_V(_EBP_)
+%endif
+
+.convert_one_pixel:
+  ; Divide the input width by 2 so it represents the offset for u or v. When
+  ; this width is odd, we read the rightmost Y pixel, U pixel, and V pixel to
+  ; convert its colorspace to RGB.
+  sar       INT_WIDTH, 1
+  jnc       .convert_two_pixels
+
+  movzx     INT_PIXEL32, BYTE [PTR_Y + INT_WIDTH * 2]
+  movd      xmm0, INT_PIXEL32
+  movzx     INT_PIXEL32, BYTE [PTR_U + INT_WIDTH]
+  movd      xmm1, INT_PIXEL32
+  movzx     INT_PIXEL32, BYTE [PTR_V + INT_WIDTH]
+  movd      xmm2, INT_PIXEL32
+
+  ; xmm0.w[0] = Y(0); xmm0.w[1] = U(0); xmm0.w[2] = V(0); xmm0.w[3] = FIX(0.5);
+  ; xmm0.w[4] = Y(1); xmm0.w[5] = U(0); xmm0.w[6] = V(0); xmm0.w[7] = FIX(0.5);
+  pxor      XMM_ZERO, XMM_ZERO
+  punpcklbw xmm0, XMM_ZERO
+  punpcklwd xmm0, XMM_ZERO
+
+  punpcklbw xmm1, XMM_ZERO
+  punpcklwd xmm1, XMM_ZERO
+  pshufd    xmm1, xmm1, 01010000B
+  pslld     xmm1, 8
+  por       xmm0, xmm1
+
+  punpcklbw xmm2, XMM_ZERO
+  punpcklwd xmm2, XMM_ZERO
+  pshufd    xmm2, xmm2, 01010000B
+  pslld     xmm2, 16
+  por       xmm0, xmm2
+
+  punpcklbw xmm0, XMM_ZERO
+
+  pshufd    xmm2, XMM_TABLE0, 01000100B
+  paddsw    xmm0, xmm2
+
+  ; Set up the result pixels.
+%if ALPHA == 0 || PIXELSIZE == 3
+  ; xmm5.d[0] = xmm5.d[1] = xmm5.d[2] = xmm5.d[3] = 0;
+  pxor      XMM_RGBA, XMM_RGBA
+%else
+  ; xmm5.d[0] = xmm5.d[1] = xmm5.d[2] = xmm5.d[3] = 0xFF000000;
+  pcmpeqd   XMM_RGBA, XMM_RGBA
+  psrad     XMM_RGBA, 24
+  pslld     XMM_RGBA, 24
+%endif
+
+  ; Calculate r[0].
+  ; xmm2.d[0] = FIX(1.164) * Y(0) + 0 * U(0);
+  ; xmm2.d[1] = FIX(1.596) * V(0) + FIX(0.5);
+  movdqa    xmm2, xmm0
+  pshufd    xmm3, XMM_TABLE0, 11101110B
+  pmaddwd   xmm2, xmm3
+
+  ; xmm2.d[0] = (xmm2.d[0] + xmm2.d[1]) / FIX(1);
+  pshufd    xmm3, xmm2, 11011101B
+  pshufd    xmm2, xmm2, 10001000B
+  paddd     xmm2, xmm3
+  psrad     xmm2, SHIFT_BITS
+
+  ; xmm5.b[02] = ToByte(xmm2.d[0]);
+  packssdw  xmm2, xmm2
+  packuswb  xmm2, xmm2
+  punpckhbw xmm2, XMM_ZERO
+  punpcklbw xmm2, XMM_ZERO
+  pslld     xmm2, 16
+  por       XMM_RGBA, xmm2
+
+  ; Calculate g[0].
+  ; xmm2.d[0] =  FIX(1.164) * Y(0) + -FIX(0.391) * U(0);
+  ; xmm2.d[1] = -FIX(0.813) * V(0) + FIX(0.5);
+  movdqa    xmm2, xmm0
+  pshufd    xmm3, XMM_TABLE1, 01000100B
+  pmaddwd   xmm2, xmm3
+
+  ; xmm2.d[0] = (xmm2.d[0] + xmm2.d[1]) / FIX(1);
+  pshufd    xmm3, xmm2, 11011101B
+  shufps    xmm2, xmm2, 10001000B
+  paddd     xmm2, xmm3
+  psrad     xmm2, SHIFT_BITS
+
+  ; xmm5.b[01] = ToByte(xmm2.d[0]);
+  packssdw  xmm2, xmm2
+  packuswb  xmm2, xmm2
+  punpckhbw xmm2, XMM_ZERO
+  punpcklbw xmm2, XMM_ZERO
+  pslld     xmm2, 8
+  por       XMM_RGBA, xmm2
+
+  ; Calculate b[0].
+  ; xmm2.d[0] = FIX(1.164) * Y(0) + FIX(2.018) * U(0);
+  ; xmm2.d[1] = 0          * V(0) + FIX(0.5);
+  movdqa    xmm2, xmm0
+  pshufd    xmm3, XMM_TABLE1, 11101110B
+  pmaddwd   xmm2, xmm3
+
+  ; xmm2.d[0] = (xmm2.d[0] + xmm2.d[1]) / FIX(1);
+  pshufd    xmm3, xmm2, 11011101B
+  shufps    xmm2, xmm2, 10001000B
+  paddd     xmm2, xmm3
+  psrad     xmm2, SHIFT_BITS
+
+  ; xmm5.b[00] = ToByte(xmm2.d[0]);
+  packssdw  xmm2, xmm2
+  packuswb  xmm2, xmm2
+  punpckhbw xmm2, XMM_ZERO
+  punpcklbw xmm2, XMM_ZERO
+  por       XMM_RGBA, xmm2
+
+%if PIXELSIZE == 4
+  movd      DWORD [PTR_RGBA + INT_WIDTH * 4 * 2], XMM_RGBA
+%else
+  movd      xmm0, INT_WIDTH32
+  lea       INT_WIDTH, [INT_WIDTH + INT_WIDTH * 2]
+  movd      INT_PIXEL32, XMM_RGBA
+  mov       WORD [PTR_RGBA + INT_WIDTH * 2], INT_PIXEL16
+  sar       INT_PIXEL32, 16
+  mov       BYTE [PTR_RGBA + INT_WIDTH * 2 + 2], INT_PIXEL8
+  movd      INT_WIDTH32, xmm0
+%endif
+
+.convert_two_pixels:
+  ; When the input width is not a multiple of four, we read the rightmost two Y
+  ; pixels, one U pixel, and one V pixel to convert their colorspace to RGB.
+  ; This code stores two RGBA pixels.
+  test      INT_WIDTH8, 1
+  jz        .convert_four_pixels
+  sub       INT_WIDTH8, 2 / 2
+
+  movzx     INT_PIXEL32, WORD [PTR_Y + INT_WIDTH * 2]
+  movd      xmm0, INT_PIXEL32
+  movzx     INT_PIXEL32, BYTE [PTR_U + INT_WIDTH]
+  movd      xmm1, INT_PIXEL32
+  movzx     INT_PIXEL32, BYTE [PTR_V + INT_WIDTH]
+  movd      xmm2, INT_PIXEL32
+
+  ; xmm0.w[0] = Y(0); xmm0.w[1] = U(0); xmm0.w[2] = V(0); xmm0.w[3] = FIX(0.5);
+  ; xmm0.w[4] = Y(1); xmm0.w[5] = U(0); xmm0.w[6] = V(0); xmm0.w[7] = FIX(0.5);
+  pxor      XMM_ZERO, XMM_ZERO
+  punpcklbw xmm0, XMM_ZERO
+  punpcklwd xmm0, XMM_ZERO
+
+  punpcklbw xmm1, XMM_ZERO
+  punpcklwd xmm1, XMM_ZERO
+  pshufd    xmm1, xmm1, 01010000B
+  pslld     xmm1, 8
+  por       xmm0, xmm1
+
+  punpcklbw xmm2, XMM_ZERO
+  punpcklwd xmm2, XMM_ZERO
+  pshufd    xmm2, xmm2, 01010000B
+  pslld     xmm2, 16
+  por       xmm0, xmm2
+
+  punpcklbw xmm0, XMM_ZERO
+
+  pshufd    xmm2, XMM_TABLE0, 01000100B
+  paddsw    xmm0, xmm2
+
+  ; Set up the result pixels.
+%if ALPHA == 0 || PIXELSIZE == 3
+  ; xmm5.d[0] = xmm5.d[1] = xmm5.d[2] = xmm5.d[3] = 0;
+  pxor      XMM_RGBA, XMM_RGBA
+%else
+  ; xmm5.d[0] = xmm5.d[1] = xmm5.d[2] = xmm5.d[3] = 0xFF000000;
+  pcmpeqd   XMM_RGBA, XMM_RGBA
+  psrad     XMM_RGBA, 24
+  pslld     XMM_RGBA, 24
+%endif
+
+  ; Calculate r[0] and r[1].
+  ; xmm2.d[0] = FIX(1.164) * Y(0) + 0 * U(0);
+  ; xmm2.d[1] = FIX(1.596) * V(0) + FIX(0.5);
+  ; xmm2.d[2] = FIX(1.164) * Y(1) + 0 * U(0);
+  ; xmm2.d[3] = FIX(1.596) * V(0) + FIX(0.5);
+  movdqa    xmm2, xmm0
+  pshufd    xmm3, XMM_TABLE0, 11101110B
+  pmaddwd   xmm2, xmm3

[fbarchard, 2011/06/10 17:29:45]

would it make sense to do a pmaddubsw version?  Using doubles requires a lot of
pack/unpack overhead and does half as much per instruction.

+
+  ; xmm2.d[0] = (xmm2.d[0] + xmm2.d[1]) / FIX(1);
+  ; xmm2.d[1] = (xmm2.d[2] + xmm2.d[3]) / FIX(1);
+  pshufd    xmm3, xmm2, 11011101B
+  pshufd    xmm2, xmm2, 10001000B
+  paddd     xmm2, xmm3
+  psrad     xmm2, SHIFT_BITS
+
+  ; xmm5.b[02] = ToByte(xmm2.d[0]); xmm5.b[06] = ToByte(xmm2.d[1]);
+  packssdw  xmm2, xmm2
+  packuswb  xmm2, xmm2
+  punpckhbw xmm2, XMM_ZERO
+  punpcklbw xmm2, XMM_ZERO
+  pslld     xmm2, 16
+  por       XMM_RGBA, xmm2
+
+  ; Calculate g[0] and g[1].
+  ; xmm2.d[0] =  FIX(1.164) * Y(0) + -FIX(0.391) * U(0);
+  ; xmm2.d[1] = -FIX(0.813) * V(0) + FIX(0.5);
+  ; xmm2.d[2] =  FIX(1.164) * Y(1) + -FIX(0.391) * U(0);
+  ; xmm2.d[3] = -FIX(0.813) * V(0) + FIX(0.5);
+  movdqa    xmm2, xmm0
+  pshufd    xmm3, XMM_TABLE1, 01000100B
+  pmaddwd   xmm2, xmm3
+
+  ; xmm2.d[0] = (xmm2.d[0] + xmm2.d[1]) / FIX(1);
+  ; xmm2.d[1] = (xmm2.d[2] + xmm2.d[3]) / FIX(1);
+  pshufd    xmm3, xmm2, 11011101B
+  shufps    xmm2, xmm2, 10001000B
+  paddd     xmm2, xmm3
+  psrad     xmm2, SHIFT_BITS
+
+  ; xmm5.b[01] = ToByte(xmm2.d[0]); xmm5.b[05] = ToByte(xmm2.d[1]);
+  packssdw  xmm2, xmm2
+  packuswb  xmm2, xmm2
+  punpckhbw xmm2, XMM_ZERO
+  punpcklbw xmm2, XMM_ZERO
+  pslld     xmm2, 8
+  por       XMM_RGBA, xmm2
+
+  ; Calculate b[0] and b[1].
+  ; xmm2.d[0] = FIX(1.164) * Y(0) + FIX(2.018) * U(0);
+  ; xmm2.d[1] = FIX(0.000) * V(0) + FIX(0.5);
+  ; xmm2.d[2] = FIX(1.164) * Y(1) + FIX(2.018) * U(0);
+  ; xmm2.d[3] = FIX(0.000) * V(0) + FIX(0.5);
+  movdqa    xmm2, xmm0
+  pshufd    xmm3, XMM_TABLE1, 11101110B
+  pmaddwd   xmm2, xmm3
+
+  ; xmm2.d[0] = (xmm2.d[0] + xmm2.d[1]) / FIX(1);
+  ; xmm2.d[1] = (xmm2.d[2] + xmm2.d[3]) / FIX(1);
+  pshufd    xmm3, xmm2, 11011101B
+  shufps    xmm2, xmm2, 10001000B
+  paddd     xmm2, xmm3
+  psrad     xmm2, SHIFT_BITS
+
+  ; xmm5.b[00] = ToByte(xmm2.d[0]); xmm5.b[04] = ToByte(xmm2.d[1]);
+  packssdw  xmm2, xmm2
+  packuswb  xmm2, xmm2
+  punpckhbw xmm2, XMM_ZERO
+  punpcklbw xmm2, XMM_ZERO
+  por       XMM_RGBA, xmm2
+
+%if PIXELSIZE == 3
+  ; Pack two RGBA pixels to two RGB pixels.
+  movdqa    xmm0, XMM_RGBA
+  psrldq    xmm0, 4
+  pslldq    xmm0, 4
+  pxor      XMM_RGBA, xmm0
+
+  psrldq    xmm0, 1
+  por       XMM_RGBA, xmm0
+%endif
+
+%if PIXELSIZE == 4
+  movq      MMWORD [PTR_RGBA + INT_WIDTH * 4 * 2], XMM_RGBA
+%else
+  movd      xmm0, INT_WIDTH
+  lea       INT_WIDTH, [INT_WIDTH + INT_WIDTH * 2]
+  movd      DWORD [PTR_RGBA + INT_WIDTH * 2], XMM_RGBA
+  psrldq    XMM_RGBA, 4
+  movd      INT_PIXEL32, XMM_RGBA
+  mov       WORD [PTR_RGBA + INT_WIDTH * 2 + 4], INT_PIXEL16
+  movd      INT_WIDTH, xmm0
+%endif
+
+.convert_four_pixels:
+  ; Read four Y pixels, two U pixels, and two V pixels to convert their
+  ; colorspace to RGB. This code stores four RGBA (or RGB) pixels.
+  test      INT_WIDTH32, INT_WIDTH32
+  jz        .convert_finish
+  sub       INT_WIDTH32, 4 / 2
+
+  movd      xmm0, DWORD [PTR_Y + INT_WIDTH * 2]
+  movzx     INT_PIXEL32, WORD [PTR_U + INT_WIDTH]
+  movd      xmm1, INT_PIXEL32
+  movzx     INT_PIXEL32, WORD [PTR_V + INT_WIDTH]
+  movd      xmm2, INT_PIXEL32
+
+  ; xmm0.w[0] = Y(0); xmm0.w[1] = U(0); xmm0.w[2] = V(0); xmm0.w[3] = FIX(0.5);
+  ; xmm0.w[4] = Y(1); xmm0.w[5] = U(0); xmm0.w[6] = V(0); xmm0.w[7] = FIX(0.5);
+  ; xmm1.w[0] = Y(2); xmm1.w[1] = U(1); xmm1.w[2] = V(1); xmm1.w[3] = FIX(0.5);
+  ; xmm1.w[4] = Y(3); xmm1.w[5] = U(1); xmm1.w[6] = V(1); xmm1.w[7] = FIX(0.5);
+  pxor      XMM_ZERO, XMM_ZERO
+  punpcklbw xmm0, XMM_ZERO
+  punpcklwd xmm0, XMM_ZERO
+
+  punpcklbw xmm1, XMM_ZERO
+  punpcklwd xmm1, XMM_ZERO
+  pshufd    xmm1, xmm1, 01010000B
+  pslld     xmm1, 8
+  por       xmm0, xmm1
+
+  punpcklbw xmm2, XMM_ZERO
+  punpcklwd xmm2, XMM_ZERO
+  pshufd    xmm2, xmm2, 01010000B
+  pslld     xmm2, 16
+  por       xmm0, xmm2
+
+  movdqa    xmm1, xmm0
+  punpckhbw xmm1, XMM_ZERO
+  punpcklbw xmm0, XMM_ZERO
+
+  pshufd    xmm2, XMM_TABLE0, 01000100B
+  paddsw    xmm0, xmm2
+  paddsw    xmm1, xmm2
+
+  ; Set up the result pixels.
+%if ALPHA == 0 || PIXELSIZE == 3
+  ; xmm5.d[0] = xmm5.d[1] = xmm5.d[2] = xmm5.d[3] = 0;
+  pxor      XMM_RGBA, XMM_RGBA
+%else
+  ; xmm5.d[0] = xmm5.d[1] = xmm5.d[2] = xmm5.d[3] = 0xFF000000;
+  pcmpeqd   XMM_RGBA, XMM_RGBA
+  psrad     XMM_RGBA, 24
+  pslld     XMM_RGBA, 24
+%endif
+
+  ; Calculate r[0], r[1], r[2], and r[3].
+  ; xmm2.d[0] = FIX(1.164) * Y(0) + 0 * U(0);
+  ; xmm2.d[1] = FIX(1.596) * V(0) + FIX(0.5);
+  ; xmm2.d[2] = FIX(1.164) * Y(1) + 0 * U(0);
+  ; xmm2.d[3] = FIX(1.596) * V(0) + FIX(0.5);
+  ; xmm3.d[0] = FIX(1.164) * Y(2) + 0 * U(0);
+  ; xmm3.d[1] = FIX(1.596) * V(1) + FIX(0.5);
+  ; xmm3.d[2] = FIX(1.164) * Y(2) + 0 * U(0);
+  ; xmm3.d[3] = FIX(1.596) * V(1) + FIX(0.5);
+  movdqa    xmm2, xmm0
+  movdqa    xmm3, xmm1
+  pshufd    xmm4, XMM_TABLE0, 11101110B
+  pmaddwd   xmm2, xmm4
+  pmaddwd   xmm3, xmm4
+
+  ; xmm2.d[0] = (xmm2.d[0] + xmm2.d[1]) / FIX(1);
+  ; xmm2.d[1] = (xmm2.d[2] + xmm2.d[3]) / FIX(1);
+  ; xmm2.d[2] = (xmm3.d[0] + xmm3.d[1]) / FIX(1);
+  ; xmm2.d[3] = (xmm3.d[2] + xmm3.d[3]) / FIX(1);
+  movdqa    xmm4, xmm2
+  shufps    xmm4, xmm3, 10001000B
+  shufps    xmm2, xmm3, 11011101B
+  paddd     xmm2, xmm4
+  psrad     xmm2, SHIFT_BITS
+
+  ; xmm5.b[02] = ToByte(xmm2.d[0]); xmm5.b[06] = ToByte(xmm2.d[1]);
+  ; xmm5.b[10] = ToByte(xmm2.d[2]); xmm5.b[14] = ToByte(xmm2.d[3]);
+  packssdw  xmm2, xmm2
+  packuswb  xmm2, xmm2
+  pxor      XMM_ZERO, XMM_ZERO
+  punpckhbw xmm2, XMM_ZERO
+  punpcklbw xmm2, XMM_ZERO
+  pslld     xmm2, 16
+  por       XMM_RGBA, xmm2
+
+  ; Calculate g[0], g[1], g[2], and g[3].
+  ; xmm2.d[0] =  FIX(1.164) * Y(0) + -FIX(0.391) * U(0);
+  ; xmm2.d[1] = -FIX(0.813) * V(0) + FIX(0.5);
+  ; xmm2.d[2] =  FIX(1.164) * Y(1) + -FIX(0.391) * U(0);
+  ; xmm2.d[3] = -FIX(0.813) * V(0) + FIX(0.5);
+  ; xmm3.d[0] =  FIX(1.164) * Y(2) + -FIX(0.391) * U(0);
+  ; xmm3.d[1] = -FIX(0.813) * V(1) + FIX(0.5);
+  ; xmm3.d[2] =  FIX(1.164) * Y(2) + -FIX(0.391) * U(0);
+  ; xmm3.d[3] = -FIX(0.813) * V(1) + FIX(0.5);
+  movdqa    xmm2, xmm0
+  movdqa    xmm3, xmm1
+  pshufd    xmm4, XMM_TABLE1, 01000100B
+  pmaddwd   xmm2, xmm4
+  pmaddwd   xmm3, xmm4
+
+  ; xmm2.d[0] = (xmm2.d[0] + xmm2.d[1]) / FIX(1);
+  ; xmm2.d[1] = (xmm2.d[2] + xmm2.d[3]) / FIX(1);
+  ; xmm2.d[2] = (xmm3.d[0] + xmm3.d[1]) / FIX(1);
+  ; xmm2.d[3] = (xmm3.d[2] + xmm3.d[3]) / FIX(1);
+  movdqa    xmm4, xmm2
+  shufps    xmm4, xmm3, 10001000B
+  shufps    xmm2, xmm3, 11011101B
+  paddd     xmm2, xmm4
+  psrad     xmm2, SHIFT_BITS
+
+  ; xmm5.b[01] = ToByte(xmm2.d[0]); xmm5.b[05] = ToByte(xmm2.d[1]);
+  ; xmm5.b[09] = ToByte(xmm2.d[2]); xmm5.b[13] = ToByte(xmm2.d[3]);
+  packssdw  xmm2, xmm2
+  packuswb  xmm2, xmm2
+  pxor      XMM_ZERO, XMM_ZERO
+  punpckhbw xmm2, XMM_ZERO
+  punpcklbw xmm2, XMM_ZERO
+  pslld     xmm2, 8
+  por       XMM_RGBA, xmm2
+
+  ; Calculate b[0], b[1], b[2], and b[3].
+  ; xmm2.d[0] = FIX(1.164) * Y(0) + FIX(2.018) * U(0);
+  ; xmm2.d[1] = FIX(0.000) * V(0) + FIX(0.5);
+  ; xmm2.d[2] = FIX(1.164) * Y(1) + FIX(2.018) * U(0);
+  ; xmm2.d[3] = FIX(0.000) * V(0) + FIX(0.5);
+  ; xmm3.d[0] = FIX(1.164) * Y(2) + FIX(2.018) * U(0);
+  ; xmm3.d[1] = FIX(0.000) * V(1) + FIX(0.5);
+  ; xmm3.d[2] = FIX(1.164) * Y(2) + FIX(2.018) * U(0);
+  ; xmm3.d[3] = FIX(0.000) * V(1) + FIX(0.5);
+  movdqa    xmm2, xmm0
+  movdqa    xmm3, xmm1
+  pshufd    xmm4, XMM_TABLE1, 11101110B
+  pmaddwd   xmm2, xmm4
+  pmaddwd   xmm3, xmm4
+
+  ; xmm2.d[0] = (xmm2.d[0] + xmm2.d[1]) / FIX(1);
+  ; xmm2.d[1] = (xmm2.d[2] + xmm2.d[3]) / FIX(1);
+  ; xmm2.d[2] = (xmm3.d[0] + xmm3.d[1]) / FIX(1);
+  ; xmm2.d[3] = (xmm3.d[2] + xmm3.d[3]) / FIX(1);
+  movdqa    xmm4, xmm2
+  shufps    xmm4, xmm3, 10001000B
+  shufps    xmm2, xmm3, 11011101B
+  paddd     xmm2, xmm4
+  psrad     xmm2, SHIFT_BITS
+
+  ; xmm5.b[00] = ToByte(xmm2.d[0]); xmm5.b[04] = ToByte(xmm2.d[1]);
+  ; xmm5.b[08] = ToByte(xmm2.d[2]); xmm5.b[12] = ToByte(xmm2.d[3]);
+  packssdw  xmm2, xmm2
+  packuswb  xmm2, xmm2
+  pxor      XMM_ZERO, XMM_ZERO
+  punpckhbw xmm2, XMM_ZERO
+  punpcklbw xmm2, XMM_ZERO
+  por       XMM_RGBA, xmm2
+
+%if PIXELSIZE == 3
+  ; Pack four RGBA pixels to four RGB pixels.
+  movdqa    xmm0, XMM_RGBA
+  psrldq    xmm0, 4
+  pslldq    xmm0, 4
+  movdqa    xmm1, XMM_RGBA
+  psrldq    xmm1, 8
+  pslldq    xmm1, 8
+  movdqa    xmm2, XMM_RGBA
+  psrldq    xmm2, 12
+  pslldq    xmm2, 12
+
+  pxor      XMM_RGBA, xmm0
+  pxor      xmm0, xmm1
+  pxor      xmm1, xmm2
+
+  psrldq    xmm0, 1
+  por       XMM_RGBA, xmm0
+  psrldq    xmm1, 2
+  por       XMM_RGBA, xmm1
+  psrldq    xmm2, 3
+  por       XMM_RGBA, xmm2
+%endif
+
+%if PIXELSIZE == 4
+  movdqu    XMMWORD [PTR_RGBA + INT_WIDTH * 4 * 2], XMM_RGBA
+%else
+  mov       _EBX_, INT_WIDTH
+  lea       _EBX_, [_EBX_ + _EBX_ * 2]
+  movq      MMWORD [PTR_RGBA + _EBX_ * 2], XMM_RGBA
+  psrldq    XMM_RGBA, 8
+  movd      DWORD [PTR_RGBA + _EBX_ * 2 + 8], XMM_RGBA
+%endif
+  jmp       .convert_four_pixels

[fbarchard, 2011/06/10 17:29:45]

more efficient to put conditional loop at bottom.

+
+.convert_finish:
+  ; Restore the registers used in this function, the stack frame, and the stack
+  ; pointer.
+  pop       _EDI_
+  pop       _ESI_
+  pop       _EBX_
+
+  mov       _ESP_, _EBP_
+  pop       _EBP_
+
+  ret
+
+  ; For some reason, the OS X linker does not honor the request to align the
+  ; segment unless we do this.
+  align     16