Fix bugs in intel-gcm-x86-masm.asm and re-enable the Intel AES assembly

nss/lib/freebl/intel-gcm-x86-masm.asm

Index: nss/lib/freebl/intel-gcm-x86-masm.asm
diff --git a/nss/lib/freebl/intel-gcm-x86-masm.asm b/nss/lib/freebl/intel-gcm-x86-masm.asm
index d8ba9b4961cf26a073905d760111755efe6f7058..4767aa52d0a224b2c55b7740d7bbd3355d4aeecf 100644
--- a/nss/lib/freebl/intel-gcm-x86-masm.asm
+++ b/nss/lib/freebl/intel-gcm-x86-masm.asm
@@ -839,12 +839,13 @@ LEncDataTail:
     vpxor   TMP2, TMP2, TMP2
     vmovdqa XMMWORD PTR[esp], TMP2
 ; copy as many bytes as needed
+    mov edi, edx

[wtc, 2014/04/30 02:00:50]

The code below uses the 8-bit dl register (which is part of the edx register).
edx is used for both Htbl and Gctx, so its value needs to be saved. After this
point, the edi register is not used, so I save the value of edx in edi. Is there
a better way to save and restore the value of a register?

     xor KS, KS
 @@:
         cmp len, KS
         je  @f
-        mov di, [PT + KS]
-        mov [esp + KS], di

[wtc, 2014/04/30 02:00:50]

Here we want to copy one byte at a time. Because of a typo, the 16-bit di
register was used, so we ended up copying two bytes at a time. This bug caused
the read access violation crashes. The fix is to use an 8-bit register such as
dl.

[agl, 2014/04/30 17:38:11]

I think this can be fixed with just:

mov di, BYTE PTR [PT + KS]

that would avoid more register shuffling. I don't know MASM and it would be
worth looking at the disassembly of the generated object file, but that should
do. If not, it might need movzx rather than mov. (movzx = MOVe with Zero eXtend)

[wtc, 2014/04/30 21:27:21]

Thanks for the suggestion.  MASM doesn't allow

    mov di, BYTE PTR [PT + KS]

but allows

    movzx di, BYTE PTR [PT + KS]

However, it seems that I can't move the di register to a one-byte memory
location. I'm afraid that we'll need to use an 8-bit register and deal with the
register shuffling.

+        mov dl, BYTE PTR[PT + KS]
+        mov BYTE PTR[esp + KS], dl
         inc KS
         jmp @b
 @@:
@@ -854,8 +855,8 @@ LEncDataTail:
 @@:
         cmp len, KS
         je  @f
-        mov di, [esp + KS]
-        mov [CT + KS], di
+        mov dl, BYTE PTR[esp + KS]
+        mov BYTE PTR[CT + KS], dl
         inc KS
         jmp @b
 @@:
@@ -865,6 +866,7 @@ LEncDataTail:
         inc KS
         jmp @b
 @@:
+    mov edx, edi

[wtc, 2014/04/30 02:00:50]

Here we restore the value of edx because we will use Htbl shortly (on line 875).

     vmovdqa TMP1, XMMWORD PTR[esp]
 
     vpshufb TMP1, TMP1, XMMWORD PTR[Lbswap_mask]
@@ -1149,12 +1151,13 @@ LDecDataTail:
     vaesenclast xmm7, TMP1, TMP2
 
 ; copy as many bytes as needed
+    mov edi, edx
     xor KS, KS
 @@:
         cmp len, KS
         je  @f
-        mov di, [CT + KS]
-        mov [esp + KS], di
+        mov dl, BYTE PTR[CT + KS]
+        mov BYTE PTR[esp + KS], dl
         inc KS
         jmp @b
 @@:
@@ -1165,6 +1168,7 @@ LDecDataTail:
         jmp @b
 @@:
 
+    mov edx, edi

[wtc, 2014/04/30 02:00:50]

Here we restore the value of edx because we will use Htbl shortly (on line
1176).

     vmovdqa TMP1, XMMWORD PTR[esp]
     vpshufb TMP1, TMP1, XMMWORD PTR[Lbswap_mask]
     vpxor   TMP1, TMP1, T
@@ -1176,21 +1180,17 @@ LDecDataTail:
 
     vpxor   xmm7, xmm7, XMMWORD PTR[esp]
     vmovdqa XMMWORD PTR[esp], xmm7
+    mov edi, edx
     xor     KS, KS
 @@:
         cmp len, KS
         je  @f
-        mov di, [esp + KS]
-        mov [PT + KS], di
-        inc KS
-        jmp @b
-@@:
-        cmp KS, 16
-        je  @f
-        mov BYTE PTR[PT + KS], 0
+        mov dl, BYTE PTR[esp + KS]
+        mov BYTE PTR[PT + KS], dl
         inc KS
         jmp @b

[wtc, 2014/04/30 02:00:50]

This block of code seems to be a copy and paste error. It overwrites the
plaintext buffer PT.

 @@:
+    mov edx, edi
 
 LDecDataEnd: