Add RSA-OAEP support from upstream NSS bugs 1009794 and 1009785

patches/nss-rsa-oaep.patch

+diff --git a/nss/lib/pk11wrap/pk11obj.c b/nss/lib/pk11wrap/pk11obj.c
+index 84268ab..7080294 100644
+--- a/nss/lib/pk11wrap/pk11obj.c
++++ b/nss/lib/pk11wrap/pk11obj.c
+@@ -914,17 +914,11 @@ PK11_Encrypt(PK11SymKey *symKey,
+     return SECSuccess;
+ }
+ 
+-/*
+- * Now SSL 2.0 uses raw RSA stuff. These next to functions *must* use
+- * RSA keys, or they'll fail. We do the checks up front. If anyone comes
+- * up with a meaning for rawdecrypt for any other public key operation,
+- * then we need to move this check into some of PK11_PubDecrypt callers,
+- * (namely SSL 2.0).
+- */
+ static SECStatus
+-pk11_PrivDecryptRaw(SECKEYPrivateKey *key, unsigned char *data, 
+-       unsigned *outLen, unsigned int maxLen, unsigned char *enc,
+-                                   unsigned encLen, CK_MECHANISM_PTR mech)
++pk11_PrivDecryptRaw(SECKEYPrivateKey *key,
++                    unsigned char *data,  unsigned *outLen, unsigned int maxLen,
++                    const unsigned char *enc, unsigned encLen,
++                    CK_MECHANISM_PTR mech)
+ {
+     PK11SlotInfo *slot = key->pkcs11Slot;
+     CK_ULONG out = maxLen;
+@@ -960,11 +954,12 @@ pk11_PrivDecryptRaw(SECKEYPrivateKey *key, unsigned char *data,
+      * do C_Login with CKU_CONTEXT_SPECIFIC 
+      * between C_DecryptInit and C_Decrypt
+      * ... But see note above about servers */
+-     if (SECKEY_HAS_ATTRIBUTE_SET_LOCK(key, CKA_ALWAYS_AUTHENTICATE, haslock)) {
++    if (SECKEY_HAS_ATTRIBUTE_SET_LOCK(key, CKA_ALWAYS_AUTHENTICATE, haslock)) {
+        PK11_DoPassword(slot, session, PR_FALSE, key->wincx, haslock, PR_TRUE);
+     }
+ 
+-    crv = PK11_GETTAB(slot)->C_Decrypt(session,enc, encLen, data, &out);
++    crv = PK11_GETTAB(slot)->C_Decrypt(session, (unsigned char *)enc, encLen,
++                                      data, &out);
+     if (haslock) PK11_ExitSlotMonitor(slot);
+     pk11_CloseSession(slot,session,owner);
+     *outLen = out;
+@@ -976,41 +971,37 @@ pk11_PrivDecryptRaw(SECKEYPrivateKey *key, unsigned char *data,
+ }
+ 
+ SECStatus
+-PK11_PubDecryptRaw(SECKEYPrivateKey *key, unsigned char *data, 
+-       unsigned *outLen, unsigned int maxLen, unsigned char *enc,
+-                                   unsigned encLen)
++PK11_PubDecryptRaw(SECKEYPrivateKey *key,
++                   unsigned char *data, unsigned *outLen, unsigned int maxLen,
++                   const unsigned char *enc, unsigned encLen)
+ {
+     CK_MECHANISM mech = {CKM_RSA_X_509, NULL, 0 };
+     return pk11_PrivDecryptRaw(key, data, outLen, maxLen, enc, encLen, &mech);
+ }
+ 
+ SECStatus
+-PK11_PrivDecryptPKCS1(SECKEYPrivateKey *key, unsigned char *data, 
+-       unsigned *outLen, unsigned int maxLen, unsigned char *enc,
+-                                   unsigned encLen)
++PK11_PrivDecryptPKCS1(SECKEYPrivateKey *key,
++                      unsigned char *data, unsigned *outLen, unsigned int maxLen,
++                      const unsigned char *enc, unsigned encLen)
+ {
+     CK_MECHANISM mech = {CKM_RSA_PKCS, NULL, 0 };
+     return pk11_PrivDecryptRaw(key, data, outLen, maxLen, enc, encLen, &mech);
+ }
+ 
+ static SECStatus
+-pk11_PubEncryptRaw(SECKEYPublicKey *key, unsigned char *enc,
+-                   unsigned char *data, unsigned dataLen, 
+-                   CK_MECHANISM_PTR mech, void *wincx)
++pk11_PubEncryptRaw(SECKEYPublicKey *key,
++                   unsigned char *out, unsigned int *outLen,
++                   unsigned int maxLen,
++                   const unsigned char *data, unsigned dataLen,
++                   CK_MECHANISM_PTR mech, void *wincx)
+ {
+     PK11SlotInfo *slot;
+     CK_OBJECT_HANDLE id;
+-    CK_ULONG out;
++    CK_ULONG len = maxLen;
+     PRBool owner = PR_TRUE;
+     CK_SESSION_HANDLE session;
+     CK_RV crv;
+ 
+-    if (!key || key->keyType != rsaKey) {
+-       PORT_SetError( SEC_ERROR_BAD_KEY );
+-       return SECFailure;
+-    }
+-    out = SECKEY_PublicKeyStrength(key);
+-
+     slot = PK11_GetBestSlotWithAttributes(mech->mechanism,CKF_ENCRYPT,0,wincx);
+     if (slot == NULL) {
+        PORT_SetError( SEC_ERROR_NO_MODULE );
+@@ -1035,10 +1026,12 @@ pk11_PubEncryptRaw(SECKEYPublicKey *key, unsigned char *enc,
+        PORT_SetError( PK11_MapError(crv) );
+        return SECFailure;
+     }
+-    crv = PK11_GETTAB(slot)->C_Encrypt(session,data,dataLen,enc,&out);
++    crv = PK11_GETTAB(slot)->C_Encrypt(session,(unsigned char *)data,dataLen,
++                                      out,&len);
+     if (!owner || !(slot->isThreadSafe)) PK11_ExitSlotMonitor(slot);
+     pk11_CloseSession(slot,session,owner);
+     PK11_FreeSlot(slot);
++    *outLen = len;
+     if (crv != CKR_OK) {
+        PORT_SetError( PK11_MapError(crv) );
+        return SECFailure;
+@@ -1047,19 +1040,69 @@ pk11_PubEncryptRaw(SECKEYPublicKey *key, unsigned char *enc,
+ }
+ 
+ SECStatus
+-PK11_PubEncryptRaw(SECKEYPublicKey *key, unsigned char *enc,
+-               unsigned char *data, unsigned dataLen, void *wincx)
++PK11_PubEncryptRaw(SECKEYPublicKey *key,
++                   unsigned char *enc,
++                   const unsigned char *data, unsigned dataLen,
++                   void *wincx)
+ {
+     CK_MECHANISM mech = {CKM_RSA_X_509, NULL, 0 };
+-    return pk11_PubEncryptRaw(key, enc, data, dataLen, &mech, wincx);
++    unsigned int outLen;
++    if (!key || key->keyType != rsaKey) {
++       PORT_SetError(SEC_ERROR_BAD_KEY);
++       return SECFailure;
++    }
++    outLen = SECKEY_PublicKeyStrength(key);
++    return pk11_PubEncryptRaw(key, enc, &outLen, outLen, data, dataLen, &mech,
++                              wincx);
+ }
+ 
+ SECStatus
+-PK11_PubEncryptPKCS1(SECKEYPublicKey *key, unsigned char *enc,
+-               unsigned char *data, unsigned dataLen, void *wincx)
++PK11_PubEncryptPKCS1(SECKEYPublicKey *key,
++                     unsigned char *enc,
++                     const unsigned char *data, unsigned dataLen,
++                     void *wincx)
+ {
+     CK_MECHANISM mech = {CKM_RSA_PKCS, NULL, 0 };
+-    return pk11_PubEncryptRaw(key, enc, data, dataLen, &mech, wincx);
++    unsigned int outLen;
++    if (!key || key->keyType != rsaKey) {
++       PORT_SetError(SEC_ERROR_BAD_KEY);
++       return SECFailure;
++    }
++    outLen = SECKEY_PublicKeyStrength(key);
++    return pk11_PubEncryptRaw(key, enc, &outLen, outLen, data, dataLen, &mech,
++                              wincx);
++}
++
++SECStatus
++PK11_PrivDecrypt(SECKEYPrivateKey *key,
++                 CK_MECHANISM_TYPE mechanism, SECItem *param,
++                 unsigned char *out, unsigned int *outLen,
++                 unsigned int maxLen,
++                 const unsigned char *enc, unsigned encLen)
++{
++    CK_MECHANISM mech = { mechanism, NULL, 0 };
++    if (param) {
++        mech.pParameter = param->data;
++        mech.ulParameterLen = param->len;
++    }
++    return pk11_PrivDecryptRaw(key, out, outLen, maxLen, enc, encLen, &mech);
++}
++
++SECStatus
++PK11_PubEncrypt(SECKEYPublicKey *key,
++                CK_MECHANISM_TYPE mechanism, SECItem *param,
++                unsigned char *out, unsigned int *outLen,
++                unsigned int maxLen,
++                const unsigned char *data, unsigned dataLen,
++                void *wincx)
++{
++    CK_MECHANISM mech = { mechanism, NULL, 0 };
++    if (param) {
++        mech.pParameter = param->data;
++        mech.ulParameterLen = param->len;
++    }
++    return pk11_PubEncryptRaw(key, out, outLen, maxLen, data, dataLen, &mech,
++                              wincx);
+ }
+ 
+ SECKEYPrivateKey *
+diff --git a/nss/lib/pk11wrap/pk11pub.h b/nss/lib/pk11wrap/pk11pub.h
+index 2f004b1..c3beef9 100644
+--- a/nss/lib/pk11wrap/pk11pub.h
++++ b/nss/lib/pk11wrap/pk11pub.h
+@@ -520,18 +520,38 @@ SECStatus PK11_Encrypt(PK11SymKey *symKey,
+                       const unsigned char *data, unsigned int dataLen);
+ 
+ /* note: despite the name, this function takes a private key. */
+-SECStatus PK11_PubDecryptRaw(SECKEYPrivateKey *key, unsigned char *data,
+-   unsigned *outLen, unsigned int maxLen, unsigned char *enc, unsigned encLen);
++SECStatus PK11_PubDecryptRaw(SECKEYPrivateKey *key,
++                             unsigned char *data, unsigned *outLen,
++                             unsigned int maxLen,
++                             const unsigned char *enc, unsigned encLen);
+ #define PK11_PrivDecryptRaw PK11_PubDecryptRaw
+ /* The encrypt function that complements the above decrypt function. */
+-SECStatus PK11_PubEncryptRaw(SECKEYPublicKey *key, unsigned char *enc,
+-                unsigned char *data, unsigned dataLen, void *wincx);
+-
+-SECStatus PK11_PrivDecryptPKCS1(SECKEYPrivateKey *key, unsigned char *data,
+-   unsigned *outLen, unsigned int maxLen, unsigned char *enc, unsigned encLen);
++SECStatus PK11_PubEncryptRaw(SECKEYPublicKey *key,
++                             unsigned char *enc,
++                             const unsigned char *data, unsigned dataLen,
++                             void *wincx);
++
++SECStatus PK11_PrivDecryptPKCS1(SECKEYPrivateKey *key,
++                                unsigned char *data, unsigned *outLen,
++                                unsigned int maxLen,
++                                const unsigned char *enc, unsigned encLen);
+ /* The encrypt function that complements the above decrypt function. */
+-SECStatus PK11_PubEncryptPKCS1(SECKEYPublicKey *key, unsigned char *enc,
+-                unsigned char *data, unsigned dataLen, void *wincx);
++SECStatus PK11_PubEncryptPKCS1(SECKEYPublicKey *key,
++                               unsigned char *enc,
++                               const unsigned char *data, unsigned dataLen,
++                               void *wincx);
++
++SECStatus PK11_PrivDecrypt(SECKEYPrivateKey *key,
++                           CK_MECHANISM_TYPE mechanism, SECItem *param,
++                           unsigned char *out, unsigned int *outLen,
++                           unsigned int maxLen,
++                           const unsigned char *enc, unsigned int encLen);
++SECStatus PK11_PubEncrypt(SECKEYPublicKey *key,
++                          CK_MECHANISM_TYPE mechanism, SECItem *param,
++                          unsigned char *out, unsigned int *outLen,
++                          unsigned int maxLen,
++                          const unsigned char *data, unsigned int dataLen,
++                          void *wincx);
+ 
+ SECStatus PK11_ImportPrivateKeyInfo(PK11SlotInfo *slot, 
+                SECKEYPrivateKeyInfo *pki, SECItem *nickname,
+diff --git a/nss/lib/softoken/pkcs11.c b/nss/lib/softoken/pkcs11.c
+index da1e1b2..5a2b1e8 100644
+--- a/nss/lib/softoken/pkcs11.c
++++ b/nss/lib/softoken/pkcs11.c
+@@ -266,6 +266,8 @@ static const struct mechanismList mechanisms[] = {
+                                 CKF_DUZ_IT_ALL},       PR_TRUE},
+      {CKM_RSA_PKCS_PSS,         {RSA_MIN_MODULUS_BITS,CK_MAX,
+                                 CKF_SN_VR},            PR_TRUE},
++     {CKM_RSA_PKCS_OAEP,        {RSA_MIN_MODULUS_BITS,CK_MAX,
++                                 CKF_EN_DE_WR_UN},       PR_TRUE},
+ #ifdef SFTK_RSA9796_SUPPORTED
+      {CKM_RSA_9796,            {RSA_MIN_MODULUS_BITS,CK_MAX,
+                                 CKF_DUZ_IT_ALL},       PR_TRUE},
+diff --git a/nss/lib/softoken/pkcs11c.c b/nss/lib/softoken/pkcs11c.c
+index 472b71a..080dfda 100644
+--- a/nss/lib/softoken/pkcs11c.c
++++ b/nss/lib/softoken/pkcs11c.c
+@@ -302,6 +302,46 @@ GetHashTypeFromMechanism(CK_MECHANISM_TYPE mech)
+     }
+ }
+ 
++/*
++ * Returns true if "params" contains a valid set of PSS parameters
++ */
++static PRBool
++sftk_ValidatePssParams(const CK_RSA_PKCS_PSS_PARAMS *params)
++{
++    if (!params) {
++        return PR_FALSE;
++    }
++    if (GetHashTypeFromMechanism(params->hashAlg) == HASH_AlgNULL ||
++        GetHashTypeFromMechanism(params->mgf) == HASH_AlgNULL) {
++        return PR_FALSE;
++    }
++    return PR_TRUE;
++}
++
++/*
++ * Returns true if "params" contains a valid set of OAEP parameters
++ */
++static PRBool
++sftk_ValidateOaepParams(const CK_RSA_PKCS_OAEP_PARAMS *params)
++{
++    if (!params) {
++        return PR_FALSE;
++    }
++    /* The requirements of ulSourceLen/pSourceData come from PKCS #11, which
++     * state:
++     *   If the parameter is empty, pSourceData must be NULL and
++     *   ulSourceDataLen must be zero.
++     */
++    if (params->source != CKZ_DATA_SPECIFIED ||
++        (GetHashTypeFromMechanism(params->hashAlg) == HASH_AlgNULL) ||
++        (GetHashTypeFromMechanism(params->mgf) == HASH_AlgNULL) ||
++        (params->ulSourceDataLen == 0 && params->pSourceData != NULL) ||
++        (params->ulSourceDataLen != 0 && params->pSourceData == NULL)) {
++        return PR_FALSE;
++    }
++    return PR_TRUE;
++}
++
+ /* 
+  * return a context based on the SFTKContext type.
+  */
+@@ -588,11 +628,6 @@ sftk_RSAEncryptOAEP(SFTKOAEPEncryptInfo *info, unsigned char *output,
+     hashAlg = GetHashTypeFromMechanism(info->params->hashAlg);
+     maskHashAlg = GetHashTypeFromMechanism(info->params->mgf);
+ 
+-    if (info->params->source != CKZ_DATA_SPECIFIED) {
+-        PORT_SetError(SEC_ERROR_INVALID_ALGORITHM);
+-        return SECFailure;
+-    }
+-
+     return RSA_EncryptOAEP(&info->key->u.rsa, hashAlg, maskHashAlg,
+                            (const unsigned char*)info->params->pSourceData,
+                            info->params->ulSourceDataLen, NULL, 0,
+@@ -617,11 +652,6 @@ sftk_RSADecryptOAEP(SFTKOAEPDecryptInfo *info, unsigned char *output,
+     hashAlg = GetHashTypeFromMechanism(info->params->hashAlg);
+     maskHashAlg = GetHashTypeFromMechanism(info->params->mgf);
+ 
+-    if (info->params->source != CKZ_DATA_SPECIFIED) {
+-        PORT_SetError(SEC_ERROR_INVALID_ALGORITHM);
+-        return SECFailure;
+-    }
+-
+     rv = RSA_DecryptOAEP(&info->key->u.rsa, hashAlg, maskHashAlg,
+                          (const unsigned char*)info->params->pSourceData,
+                          info->params->ulSourceDataLen,
+@@ -801,19 +831,18 @@ sftk_CryptInit(CK_SESSION_HANDLE hSession, CK_MECHANISM_PTR pMechanism,
+        }
+        context->destroy = sftk_Null;
+        break;
+-/* XXX: Disabled until unit tests land.
+     case CKM_RSA_PKCS_OAEP:
+        if (key_type != CKK_RSA) {
+            crv = CKR_KEY_TYPE_INCONSISTENT;
+            break;
+        }
+-       context->multi = PR_FALSE;
+-       context->rsa = PR_TRUE;
+-       if (pMechanism->ulParameterLen != sizeof(CK_RSA_PKCS_OAEP_PARAMS)) {
++       if (pMechanism->ulParameterLen != sizeof(CK_RSA_PKCS_OAEP_PARAMS) ||
++           !sftk_ValidateOaepParams((CK_RSA_PKCS_OAEP_PARAMS*)pMechanism->pParameter)) {
+            crv = CKR_MECHANISM_PARAM_INVALID;
+            break;
+        }
+-       /\* XXX: Need Parameter validation here *\/
++       context->multi = PR_FALSE;
++       context->rsa = PR_TRUE;
+        if (isEncrypt) {
+            SFTKOAEPEncryptInfo *info = PORT_New(SFTKOAEPEncryptInfo);
+            if (info == NULL) {
+@@ -849,7 +878,6 @@ sftk_CryptInit(CK_SESSION_HANDLE hSession, CK_MECHANISM_PTR pMechanism,
+        }
+        context->destroy = (SFTKDestroy) sftk_Space;
+        break;
+-*/
+     case CKM_RC2_CBC_PAD:
+        context->doPad = PR_TRUE;
+        /* fall thru */
+@@ -2506,7 +2534,8 @@ finish_rsa:
+            break;
+        } 
+        context->rsa = PR_TRUE;
+-       if (pMechanism->ulParameterLen != sizeof(CK_RSA_PKCS_PSS_PARAMS)) {
++       if (pMechanism->ulParameterLen != sizeof(CK_RSA_PKCS_PSS_PARAMS) ||
++           !sftk_ValidatePssParams((const CK_RSA_PKCS_PSS_PARAMS*)pMechanism->pParameter)) {
+            crv = CKR_MECHANISM_PARAM_INVALID;
+            break;
+        }
+@@ -3143,7 +3172,8 @@ finish_rsa:
+            break;
+        } 
+        context->rsa = PR_TRUE;
+-       if (pMechanism->ulParameterLen != sizeof(CK_RSA_PKCS_PSS_PARAMS)) {
++       if (pMechanism->ulParameterLen != sizeof(CK_RSA_PKCS_PSS_PARAMS) ||
++           !sftk_ValidatePssParams((const CK_RSA_PKCS_PSS_PARAMS*)pMechanism->pParameter)) {
+            crv = CKR_MECHANISM_PARAM_INVALID;
+            break;
+        }