Update to NSS 3.16.2 Beta 3.

patches/nss-rsa-oaep.patch

-diff --git a/nss/lib/pk11wrap/pk11obj.c b/nss/lib/pk11wrap/pk11obj.c
-index 84268ab..7080294 100644
---- a/nss/lib/pk11wrap/pk11obj.c
-+++ b/nss/lib/pk11wrap/pk11obj.c
-@@ -914,17 +914,11 @@ PK11_Encrypt(PK11SymKey *symKey,
-     return SECSuccess;
- }
- 
--/*
-- * Now SSL 2.0 uses raw RSA stuff. These next to functions *must* use
-- * RSA keys, or they'll fail. We do the checks up front. If anyone comes
-- * up with a meaning for rawdecrypt for any other public key operation,
-- * then we need to move this check into some of PK11_PubDecrypt callers,
-- * (namely SSL 2.0).
-- */
- static SECStatus
--pk11_PrivDecryptRaw(SECKEYPrivateKey *key, unsigned char *data, 
--       unsigned *outLen, unsigned int maxLen, unsigned char *enc,
--                                   unsigned encLen, CK_MECHANISM_PTR mech)
-+pk11_PrivDecryptRaw(SECKEYPrivateKey *key,
-+                    unsigned char *data,  unsigned *outLen, unsigned int maxLen,
-+                    const unsigned char *enc, unsigned encLen,
-+                    CK_MECHANISM_PTR mech)
- {
-     PK11SlotInfo *slot = key->pkcs11Slot;
-     CK_ULONG out = maxLen;
-@@ -960,11 +954,12 @@ pk11_PrivDecryptRaw(SECKEYPrivateKey *key, unsigned char *data,
-      * do C_Login with CKU_CONTEXT_SPECIFIC 
-      * between C_DecryptInit and C_Decrypt
-      * ... But see note above about servers */
--     if (SECKEY_HAS_ATTRIBUTE_SET_LOCK(key, CKA_ALWAYS_AUTHENTICATE, haslock)) {
-+    if (SECKEY_HAS_ATTRIBUTE_SET_LOCK(key, CKA_ALWAYS_AUTHENTICATE, haslock)) {
-        PK11_DoPassword(slot, session, PR_FALSE, key->wincx, haslock, PR_TRUE);
-     }
- 
--    crv = PK11_GETTAB(slot)->C_Decrypt(session,enc, encLen, data, &out);
-+    crv = PK11_GETTAB(slot)->C_Decrypt(session, (unsigned char *)enc, encLen,
-+                                      data, &out);
-     if (haslock) PK11_ExitSlotMonitor(slot);
-     pk11_CloseSession(slot,session,owner);
-     *outLen = out;
-@@ -976,41 +971,37 @@ pk11_PrivDecryptRaw(SECKEYPrivateKey *key, unsigned char *data,
- }
- 
- SECStatus
--PK11_PubDecryptRaw(SECKEYPrivateKey *key, unsigned char *data, 
--       unsigned *outLen, unsigned int maxLen, unsigned char *enc,
--                                   unsigned encLen)
-+PK11_PubDecryptRaw(SECKEYPrivateKey *key,
-+                   unsigned char *data, unsigned *outLen, unsigned int maxLen,
-+                   const unsigned char *enc, unsigned encLen)
- {
-     CK_MECHANISM mech = {CKM_RSA_X_509, NULL, 0 };
-     return pk11_PrivDecryptRaw(key, data, outLen, maxLen, enc, encLen, &mech);
- }
- 
- SECStatus
--PK11_PrivDecryptPKCS1(SECKEYPrivateKey *key, unsigned char *data, 
--       unsigned *outLen, unsigned int maxLen, unsigned char *enc,
--                                   unsigned encLen)
-+PK11_PrivDecryptPKCS1(SECKEYPrivateKey *key,
-+                      unsigned char *data, unsigned *outLen, unsigned int maxLen,
-+                      const unsigned char *enc, unsigned encLen)
- {
-     CK_MECHANISM mech = {CKM_RSA_PKCS, NULL, 0 };
-     return pk11_PrivDecryptRaw(key, data, outLen, maxLen, enc, encLen, &mech);
- }
- 
- static SECStatus
--pk11_PubEncryptRaw(SECKEYPublicKey *key, unsigned char *enc,
--                   unsigned char *data, unsigned dataLen, 
--                   CK_MECHANISM_PTR mech, void *wincx)
-+pk11_PubEncryptRaw(SECKEYPublicKey *key,
-+                   unsigned char *out, unsigned int *outLen,
-+                   unsigned int maxLen,
-+                   const unsigned char *data, unsigned dataLen,
-+                   CK_MECHANISM_PTR mech, void *wincx)
- {
-     PK11SlotInfo *slot;
-     CK_OBJECT_HANDLE id;
--    CK_ULONG out;
-+    CK_ULONG len = maxLen;
-     PRBool owner = PR_TRUE;
-     CK_SESSION_HANDLE session;
-     CK_RV crv;
- 
--    if (!key || key->keyType != rsaKey) {
--       PORT_SetError( SEC_ERROR_BAD_KEY );
--       return SECFailure;
--    }
--    out = SECKEY_PublicKeyStrength(key);
--
-     slot = PK11_GetBestSlotWithAttributes(mech->mechanism,CKF_ENCRYPT,0,wincx);
-     if (slot == NULL) {
-        PORT_SetError( SEC_ERROR_NO_MODULE );
-@@ -1035,10 +1026,12 @@ pk11_PubEncryptRaw(SECKEYPublicKey *key, unsigned char *enc,
-        PORT_SetError( PK11_MapError(crv) );
-        return SECFailure;
-     }
--    crv = PK11_GETTAB(slot)->C_Encrypt(session,data,dataLen,enc,&out);
-+    crv = PK11_GETTAB(slot)->C_Encrypt(session,(unsigned char *)data,dataLen,
-+                                      out,&len);
-     if (!owner || !(slot->isThreadSafe)) PK11_ExitSlotMonitor(slot);
-     pk11_CloseSession(slot,session,owner);
-     PK11_FreeSlot(slot);
-+    *outLen = len;
-     if (crv != CKR_OK) {
-        PORT_SetError( PK11_MapError(crv) );
-        return SECFailure;
-@@ -1047,19 +1040,69 @@ pk11_PubEncryptRaw(SECKEYPublicKey *key, unsigned char *enc,
- }
- 
- SECStatus
--PK11_PubEncryptRaw(SECKEYPublicKey *key, unsigned char *enc,
--               unsigned char *data, unsigned dataLen, void *wincx)
-+PK11_PubEncryptRaw(SECKEYPublicKey *key,
-+                   unsigned char *enc,
-+                   const unsigned char *data, unsigned dataLen,
-+                   void *wincx)
- {
-     CK_MECHANISM mech = {CKM_RSA_X_509, NULL, 0 };
--    return pk11_PubEncryptRaw(key, enc, data, dataLen, &mech, wincx);
-+    unsigned int outLen;
-+    if (!key || key->keyType != rsaKey) {
-+       PORT_SetError(SEC_ERROR_BAD_KEY);
-+       return SECFailure;
-+    }
-+    outLen = SECKEY_PublicKeyStrength(key);
-+    return pk11_PubEncryptRaw(key, enc, &outLen, outLen, data, dataLen, &mech,
-+                              wincx);
- }
- 
- SECStatus
--PK11_PubEncryptPKCS1(SECKEYPublicKey *key, unsigned char *enc,
--               unsigned char *data, unsigned dataLen, void *wincx)
-+PK11_PubEncryptPKCS1(SECKEYPublicKey *key,
-+                     unsigned char *enc,
-+                     const unsigned char *data, unsigned dataLen,
-+                     void *wincx)
- {
-     CK_MECHANISM mech = {CKM_RSA_PKCS, NULL, 0 };
--    return pk11_PubEncryptRaw(key, enc, data, dataLen, &mech, wincx);
-+    unsigned int outLen;
-+    if (!key || key->keyType != rsaKey) {
-+       PORT_SetError(SEC_ERROR_BAD_KEY);
-+       return SECFailure;
-+    }
-+    outLen = SECKEY_PublicKeyStrength(key);
-+    return pk11_PubEncryptRaw(key, enc, &outLen, outLen, data, dataLen, &mech,
-+                              wincx);
-+}
-+
-+SECStatus
-+PK11_PrivDecrypt(SECKEYPrivateKey *key,
-+                 CK_MECHANISM_TYPE mechanism, SECItem *param,
-+                 unsigned char *out, unsigned int *outLen,
-+                 unsigned int maxLen,
-+                 const unsigned char *enc, unsigned encLen)
-+{
-+    CK_MECHANISM mech = { mechanism, NULL, 0 };
-+    if (param) {
-+        mech.pParameter = param->data;
-+        mech.ulParameterLen = param->len;
-+    }
-+    return pk11_PrivDecryptRaw(key, out, outLen, maxLen, enc, encLen, &mech);
-+}
-+
-+SECStatus
-+PK11_PubEncrypt(SECKEYPublicKey *key,
-+                CK_MECHANISM_TYPE mechanism, SECItem *param,
-+                unsigned char *out, unsigned int *outLen,
-+                unsigned int maxLen,
-+                const unsigned char *data, unsigned dataLen,
-+                void *wincx)
-+{
-+    CK_MECHANISM mech = { mechanism, NULL, 0 };
-+    if (param) {
-+        mech.pParameter = param->data;
-+        mech.ulParameterLen = param->len;
-+    }
-+    return pk11_PubEncryptRaw(key, out, outLen, maxLen, data, dataLen, &mech,
-+                              wincx);
- }
- 
- SECKEYPrivateKey *
-diff --git a/nss/lib/pk11wrap/pk11pub.h b/nss/lib/pk11wrap/pk11pub.h
-index 2f004b1..c3beef9 100644
---- a/nss/lib/pk11wrap/pk11pub.h
-+++ b/nss/lib/pk11wrap/pk11pub.h
-@@ -520,18 +520,38 @@ SECStatus PK11_Encrypt(PK11SymKey *symKey,
-                       const unsigned char *data, unsigned int dataLen);
- 
- /* note: despite the name, this function takes a private key. */
--SECStatus PK11_PubDecryptRaw(SECKEYPrivateKey *key, unsigned char *data,
--   unsigned *outLen, unsigned int maxLen, unsigned char *enc, unsigned encLen);
-+SECStatus PK11_PubDecryptRaw(SECKEYPrivateKey *key,
-+                             unsigned char *data, unsigned *outLen,
-+                             unsigned int maxLen,
-+                             const unsigned char *enc, unsigned encLen);
- #define PK11_PrivDecryptRaw PK11_PubDecryptRaw
- /* The encrypt function that complements the above decrypt function. */
--SECStatus PK11_PubEncryptRaw(SECKEYPublicKey *key, unsigned char *enc,
--                unsigned char *data, unsigned dataLen, void *wincx);
--
--SECStatus PK11_PrivDecryptPKCS1(SECKEYPrivateKey *key, unsigned char *data,
--   unsigned *outLen, unsigned int maxLen, unsigned char *enc, unsigned encLen);
-+SECStatus PK11_PubEncryptRaw(SECKEYPublicKey *key,
-+                             unsigned char *enc,
-+                             const unsigned char *data, unsigned dataLen,
-+                             void *wincx);
-+
-+SECStatus PK11_PrivDecryptPKCS1(SECKEYPrivateKey *key,
-+                                unsigned char *data, unsigned *outLen,
-+                                unsigned int maxLen,
-+                                const unsigned char *enc, unsigned encLen);
- /* The encrypt function that complements the above decrypt function. */
--SECStatus PK11_PubEncryptPKCS1(SECKEYPublicKey *key, unsigned char *enc,
--                unsigned char *data, unsigned dataLen, void *wincx);
-+SECStatus PK11_PubEncryptPKCS1(SECKEYPublicKey *key,
-+                               unsigned char *enc,
-+                               const unsigned char *data, unsigned dataLen,
-+                               void *wincx);
-+
-+SECStatus PK11_PrivDecrypt(SECKEYPrivateKey *key,
-+                           CK_MECHANISM_TYPE mechanism, SECItem *param,
-+                           unsigned char *out, unsigned int *outLen,
-+                           unsigned int maxLen,
-+                           const unsigned char *enc, unsigned int encLen);
-+SECStatus PK11_PubEncrypt(SECKEYPublicKey *key,
-+                          CK_MECHANISM_TYPE mechanism, SECItem *param,
-+                          unsigned char *out, unsigned int *outLen,
-+                          unsigned int maxLen,
-+                          const unsigned char *data, unsigned int dataLen,
-+                          void *wincx);
- 
- SECStatus PK11_ImportPrivateKeyInfo(PK11SlotInfo *slot, 
-                SECKEYPrivateKeyInfo *pki, SECItem *nickname,
-diff --git a/nss/lib/softoken/pkcs11.c b/nss/lib/softoken/pkcs11.c
-index da1e1b2..5a2b1e8 100644
---- a/nss/lib/softoken/pkcs11.c
-+++ b/nss/lib/softoken/pkcs11.c
-@@ -266,6 +266,8 @@ static const struct mechanismList mechanisms[] = {
-                                 CKF_DUZ_IT_ALL},       PR_TRUE},
-      {CKM_RSA_PKCS_PSS,         {RSA_MIN_MODULUS_BITS,CK_MAX,
-                                 CKF_SN_VR},            PR_TRUE},
-+     {CKM_RSA_PKCS_OAEP,        {RSA_MIN_MODULUS_BITS,CK_MAX,
-+                                 CKF_EN_DE_WR_UN},       PR_TRUE},
- #ifdef SFTK_RSA9796_SUPPORTED
-      {CKM_RSA_9796,            {RSA_MIN_MODULUS_BITS,CK_MAX,
-                                 CKF_DUZ_IT_ALL},       PR_TRUE},
-diff --git a/nss/lib/softoken/pkcs11c.c b/nss/lib/softoken/pkcs11c.c
-index 472b71a..080dfda 100644
---- a/nss/lib/softoken/pkcs11c.c
-+++ b/nss/lib/softoken/pkcs11c.c
-@@ -302,6 +302,46 @@ GetHashTypeFromMechanism(CK_MECHANISM_TYPE mech)
-     }
- }
- 
-+/*
-+ * Returns true if "params" contains a valid set of PSS parameters
-+ */
-+static PRBool
-+sftk_ValidatePssParams(const CK_RSA_PKCS_PSS_PARAMS *params)
-+{
-+    if (!params) {
-+        return PR_FALSE;
-+    }
-+    if (GetHashTypeFromMechanism(params->hashAlg) == HASH_AlgNULL ||
-+        GetHashTypeFromMechanism(params->mgf) == HASH_AlgNULL) {
-+        return PR_FALSE;
-+    }
-+    return PR_TRUE;
-+}
-+
-+/*
-+ * Returns true if "params" contains a valid set of OAEP parameters
-+ */
-+static PRBool
-+sftk_ValidateOaepParams(const CK_RSA_PKCS_OAEP_PARAMS *params)
-+{
-+    if (!params) {
-+        return PR_FALSE;
-+    }
-+    /* The requirements of ulSourceLen/pSourceData come from PKCS #11, which
-+     * state:
-+     *   If the parameter is empty, pSourceData must be NULL and
-+     *   ulSourceDataLen must be zero.
-+     */
-+    if (params->source != CKZ_DATA_SPECIFIED ||
-+        (GetHashTypeFromMechanism(params->hashAlg) == HASH_AlgNULL) ||
-+        (GetHashTypeFromMechanism(params->mgf) == HASH_AlgNULL) ||
-+        (params->ulSourceDataLen == 0 && params->pSourceData != NULL) ||
-+        (params->ulSourceDataLen != 0 && params->pSourceData == NULL)) {
-+        return PR_FALSE;
-+    }
-+    return PR_TRUE;
-+}
-+
- /* 
-  * return a context based on the SFTKContext type.
-  */
-@@ -588,11 +628,6 @@ sftk_RSAEncryptOAEP(SFTKOAEPEncryptInfo *info, unsigned char *output,
-     hashAlg = GetHashTypeFromMechanism(info->params->hashAlg);
-     maskHashAlg = GetHashTypeFromMechanism(info->params->mgf);
- 
--    if (info->params->source != CKZ_DATA_SPECIFIED) {
--        PORT_SetError(SEC_ERROR_INVALID_ALGORITHM);
--        return SECFailure;
--    }
--
-     return RSA_EncryptOAEP(&info->key->u.rsa, hashAlg, maskHashAlg,
-                            (const unsigned char*)info->params->pSourceData,
-                            info->params->ulSourceDataLen, NULL, 0,
-@@ -617,11 +652,6 @@ sftk_RSADecryptOAEP(SFTKOAEPDecryptInfo *info, unsigned char *output,
-     hashAlg = GetHashTypeFromMechanism(info->params->hashAlg);
-     maskHashAlg = GetHashTypeFromMechanism(info->params->mgf);
- 
--    if (info->params->source != CKZ_DATA_SPECIFIED) {
--        PORT_SetError(SEC_ERROR_INVALID_ALGORITHM);
--        return SECFailure;
--    }
--
-     rv = RSA_DecryptOAEP(&info->key->u.rsa, hashAlg, maskHashAlg,
-                          (const unsigned char*)info->params->pSourceData,
-                          info->params->ulSourceDataLen,
-@@ -801,19 +831,18 @@ sftk_CryptInit(CK_SESSION_HANDLE hSession, CK_MECHANISM_PTR pMechanism,
-        }
-        context->destroy = sftk_Null;
-        break;
--/* XXX: Disabled until unit tests land.
-     case CKM_RSA_PKCS_OAEP:
-        if (key_type != CKK_RSA) {
-            crv = CKR_KEY_TYPE_INCONSISTENT;
-            break;
-        }
--       context->multi = PR_FALSE;
--       context->rsa = PR_TRUE;
--       if (pMechanism->ulParameterLen != sizeof(CK_RSA_PKCS_OAEP_PARAMS)) {
-+       if (pMechanism->ulParameterLen != sizeof(CK_RSA_PKCS_OAEP_PARAMS) ||
-+           !sftk_ValidateOaepParams((CK_RSA_PKCS_OAEP_PARAMS*)pMechanism->pParameter)) {
-            crv = CKR_MECHANISM_PARAM_INVALID;
-            break;
-        }
--       /\* XXX: Need Parameter validation here *\/
-+       context->multi = PR_FALSE;
-+       context->rsa = PR_TRUE;
-        if (isEncrypt) {
-            SFTKOAEPEncryptInfo *info = PORT_New(SFTKOAEPEncryptInfo);
-            if (info == NULL) {
-@@ -849,7 +878,6 @@ sftk_CryptInit(CK_SESSION_HANDLE hSession, CK_MECHANISM_PTR pMechanism,
-        }
-        context->destroy = (SFTKDestroy) sftk_Space;
-        break;
--*/
-     case CKM_RC2_CBC_PAD:
-        context->doPad = PR_TRUE;
-        /* fall thru */
-@@ -2506,7 +2534,8 @@ finish_rsa:
-            break;
-        } 
-        context->rsa = PR_TRUE;
--       if (pMechanism->ulParameterLen != sizeof(CK_RSA_PKCS_PSS_PARAMS)) {
-+       if (pMechanism->ulParameterLen != sizeof(CK_RSA_PKCS_PSS_PARAMS) ||
-+           !sftk_ValidatePssParams((const CK_RSA_PKCS_PSS_PARAMS*)pMechanism->pParameter)) {
-            crv = CKR_MECHANISM_PARAM_INVALID;
-            break;
-        }
-@@ -3143,7 +3172,8 @@ finish_rsa:
-            break;
-        } 
-        context->rsa = PR_TRUE;
--       if (pMechanism->ulParameterLen != sizeof(CK_RSA_PKCS_PSS_PARAMS)) {
-+       if (pMechanism->ulParameterLen != sizeof(CK_RSA_PKCS_PSS_PARAMS) ||
-+           !sftk_ValidatePssParams((const CK_RSA_PKCS_PSS_PARAMS*)pMechanism->pParameter)) {
-            crv = CKR_MECHANISM_PARAM_INVALID;
-            break;
-        }