Fix bugs in intel-gcm-x86-masm.asm and re-enable the Intel AES assembly

nss/lib/freebl/intel-gcm-x86-masm.asm

 ; LICENSE:
 ; This submission to NSS is to be made available under the terms of the
 ; Mozilla Public License, v. 2.0. You can obtain one at http:
 ; //mozilla.org/MPL/2.0/.
 ;###############################################################################
 ; Copyright(c) 2014, Intel Corp.
 ; Developers and authors:
 ; Shay Gueron and Vlad Krasnov
 ; Intel Corporation, Israel Development Centre, Haifa, Israel
 ; Please send feedback directly to crypto.feedback.alias@intel.com
@@ skipping 821 matching lines @@
     je  @f
     vaesenc TMP1, TMP1, XMMWORD PTR[12*16 + KS]
     vaesenc TMP1, TMP1, XMMWORD PTR[13*16 + KS]
     vmovdqu TMP2, XMMWORD PTR[14*16 + KS]
 @@:
     vaesenclast TMP1, TMP1, TMP2
 ; zero a temp location
     vpxor   TMP2, TMP2, TMP2
     vmovdqa XMMWORD PTR[esp], TMP2
 ; copy as many bytes as needed
+    mov edi, edx

[wtc, 2014/04/30 02:00:50]

The code below uses the 8-bit dl register (which is part of the edx register).
edx is used for both Htbl and Gctx, so its value needs to be saved. After this
point, the edi register is not used, so I save the value of edx in edi. Is there
a better way to save and restore the value of a register?

     xor KS, KS
 @@:
         cmp len, KS
         je  @f
-        mov di, [PT + KS]
-        mov [esp + KS], di

[wtc, 2014/04/30 02:00:50]

Here we want to copy one byte at a time. Because of a typo, the 16-bit di
register was used, so we ended up copying two bytes at a time. This bug caused
the read access violation crashes. The fix is to use an 8-bit register such as
dl.

[agl, 2014/04/30 17:38:11]

I think this can be fixed with just:

mov di, BYTE PTR [PT + KS]

that would avoid more register shuffling. I don't know MASM and it would be
worth looking at the disassembly of the generated object file, but that should
do. If not, it might need movzx rather than mov. (movzx = MOVe with Zero eXtend)

[wtc, 2014/04/30 21:27:21]

Thanks for the suggestion.  MASM doesn't allow

    mov di, BYTE PTR [PT + KS]

but allows

    movzx di, BYTE PTR [PT + KS]

However, it seems that I can't move the di register to a one-byte memory
location. I'm afraid that we'll need to use an 8-bit register and deal with the
register shuffling.

+        mov dl, BYTE PTR[PT + KS]
+        mov BYTE PTR[esp + KS], dl
         inc KS
         jmp @b
 @@:
     vpxor   TMP1, TMP1, XMMWORD PTR[esp]
     vmovdqa XMMWORD PTR[esp], TMP1
     xor KS, KS
 @@:
         cmp len, KS
         je  @f
-        mov di, [esp + KS]
-        mov [CT + KS], di
+        mov dl, BYTE PTR[esp + KS]
+        mov BYTE PTR[CT + KS], dl
         inc KS
         jmp @b
 @@:
         cmp KS, 16
         je  @f
         mov BYTE PTR[esp + KS], 0
         inc KS
         jmp @b
 @@:
+    mov edx, edi

[wtc, 2014/04/30 02:00:50]

Here we restore the value of edx because we will use Htbl shortly (on line 875).

     vmovdqa TMP1, XMMWORD PTR[esp]
 
     vpshufb TMP1, TMP1, XMMWORD PTR[Lbswap_mask]
     vpxor   TMP1, TMP1, T
 
     vmovdqu TMP0, XMMWORD PTR[Htbl]
     GFMUL   TMP1, TMP1, TMP0, TMP5, TMP2, TMP3, TMP4
     vmovdqu T, TMP1
 
 LEncDataEnd:
@@ skipping 264 matching lines @@
     vmovdqu TMP2, XMMWORD PTR[12*16 + KS]
     cmp NR, 12
     je  @f
     vaesenc TMP1, TMP1, XMMWORD PTR[12*16 + KS]
     vaesenc TMP1, TMP1, XMMWORD PTR[13*16 + KS]
     vmovdqu TMP2, XMMWORD PTR[14*16 + KS]
 @@:
     vaesenclast xmm7, TMP1, TMP2
 
 ; copy as many bytes as needed
+    mov edi, edx
     xor KS, KS
 @@:
         cmp len, KS
         je  @f
-        mov di, [CT + KS]
-        mov [esp + KS], di
+        mov dl, BYTE PTR[CT + KS]
+        mov BYTE PTR[esp + KS], dl
         inc KS
         jmp @b
 @@:
         cmp KS, 16
         je  @f
         mov BYTE PTR[esp + KS], 0
         inc KS
         jmp @b
 @@:
 
+    mov edx, edi

[wtc, 2014/04/30 02:00:50]

Here we restore the value of edx because we will use Htbl shortly (on line
1176).

     vmovdqa TMP1, XMMWORD PTR[esp]
     vpshufb TMP1, TMP1, XMMWORD PTR[Lbswap_mask]
     vpxor   TMP1, TMP1, T
 
     vmovdqu TMP0, XMMWORD PTR[Htbl]
     GFMUL   TMP1, TMP1, TMP0, TMP5, TMP2, TMP3, TMP4
     vmovdqu T, TMP1
 
 
     vpxor   xmm7, xmm7, XMMWORD PTR[esp]
     vmovdqa XMMWORD PTR[esp], xmm7
+    mov edi, edx
     xor     KS, KS
 @@:
         cmp len, KS
         je  @f
-        mov di, [esp + KS]
-        mov [PT + KS], di
+        mov dl, BYTE PTR[esp + KS]
+        mov BYTE PTR[PT + KS], dl
         inc KS
         jmp @b
 @@:
-        cmp KS, 16
-        je  @f
-        mov BYTE PTR[PT + KS], 0
-        inc KS
-        jmp @b

[wtc, 2014/04/30 02:00:50]

This block of code seems to be a copy and paste error. It overwrites the
plaintext buffer PT.

-@@:
+    mov edx, edi
 
 LDecDataEnd:
 
     bswap   aluCTR
     mov     [16*16 + 2*16 + 3*4 + Gctx], aluCTR
 
     mov esp, ebp
     pop edi
     pop esi
     pop ebx
     pop ebp
 
     vzeroupper
 
     ret
 intel_aes_gcmDEC ENDP
 
 
 END