Add RSA-OAEP support from upstream NSS bugs 1009794 and 1009785

nss/lib/softoken/pkcs11c.c

 /* This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 /*
  * This file implements PKCS 11 on top of our existing security modules
  *
  * For more information about PKCS 11 See PKCS 11 Token Inteface Standard.
  *   This implementation has two slots:
  *      slot 1 is our generic crypto support. It does not require login.
  *   It supports Public Key ops, and all they bulk ciphers and hashes. 
@@ skipping 284 matching lines @@
         case CKG_MGF1_SHA384:
             return HASH_AlgSHA384;
         case CKM_SHA512:
         case CKG_MGF1_SHA512:
             return HASH_AlgSHA512;
         default:
             return HASH_AlgNULL;
     }
 }
 
+/*
+ * Returns true if "params" contains a valid set of PSS parameters
+ */
+static PRBool
+sftk_ValidatePssParams(const CK_RSA_PKCS_PSS_PARAMS *params)
+{
+    if (!params) {
+        return PR_FALSE;
+    }
+    if (GetHashTypeFromMechanism(params->hashAlg) == HASH_AlgNULL ||
+        GetHashTypeFromMechanism(params->mgf) == HASH_AlgNULL) {
+        return PR_FALSE;
+    }
+    return PR_TRUE;
+}
+
+/*
+ * Returns true if "params" contains a valid set of OAEP parameters
+ */
+static PRBool
+sftk_ValidateOaepParams(const CK_RSA_PKCS_OAEP_PARAMS *params)
+{
+    if (!params) {
+        return PR_FALSE;
+    }
+    /* The requirements of ulSourceLen/pSourceData come from PKCS #11, which
+     * state:
+     *   If the parameter is empty, pSourceData must be NULL and
+     *   ulSourceDataLen must be zero.
+     */
+    if (params->source != CKZ_DATA_SPECIFIED ||
+        (GetHashTypeFromMechanism(params->hashAlg) == HASH_AlgNULL) ||
+        (GetHashTypeFromMechanism(params->mgf) == HASH_AlgNULL) ||
+        (params->ulSourceDataLen == 0 && params->pSourceData != NULL) ||
+        (params->ulSourceDataLen != 0 && params->pSourceData == NULL)) {
+        return PR_FALSE;
+    }
+    return PR_TRUE;
+}
+
 /* 
  * return a context based on the SFTKContext type.
  */
 SFTKSessionContext *
 sftk_ReturnContextByType(SFTKSession *session, SFTKContextType type)
 {
     switch (type) {
         case SFTK_ENCRYPT:
         case SFTK_DECRYPT:
             return session->enc_context;
@@ skipping 266 matching lines @@
 
     PORT_Assert(info->key->keyType == NSSLOWKEYRSAKey);
     if (info->key->keyType != NSSLOWKEYRSAKey) {
         PORT_SetError(SEC_ERROR_INVALID_KEY);
         return SECFailure;
     }
 
     hashAlg = GetHashTypeFromMechanism(info->params->hashAlg);
     maskHashAlg = GetHashTypeFromMechanism(info->params->mgf);
 
-    if (info->params->source != CKZ_DATA_SPECIFIED) {
-        PORT_SetError(SEC_ERROR_INVALID_ALGORITHM);
-        return SECFailure;
-    }
-
     return RSA_EncryptOAEP(&info->key->u.rsa, hashAlg, maskHashAlg,
                            (const unsigned char*)info->params->pSourceData,
                            info->params->ulSourceDataLen, NULL, 0,
                            output, outputLen, maxLen, input, inputLen);
 }
 
 static SECStatus
 sftk_RSADecryptOAEP(SFTKOAEPDecryptInfo *info, unsigned char *output,
                     unsigned int *outputLen, unsigned int maxLen,
                     const unsigned char *input, unsigned int inputLen)
 {
     SECStatus rv = SECFailure;
     HASH_HashType hashAlg;
     HASH_HashType maskHashAlg;
 
     PORT_Assert(info->key->keyType == NSSLOWKEYRSAKey);
     if (info->key->keyType != NSSLOWKEYRSAKey) {
         PORT_SetError(SEC_ERROR_INVALID_KEY);
         return SECFailure;
     }
 
     hashAlg = GetHashTypeFromMechanism(info->params->hashAlg);
     maskHashAlg = GetHashTypeFromMechanism(info->params->mgf);
 
-    if (info->params->source != CKZ_DATA_SPECIFIED) {
-        PORT_SetError(SEC_ERROR_INVALID_ALGORITHM);
-        return SECFailure;
-    }
-
     rv = RSA_DecryptOAEP(&info->key->u.rsa, hashAlg, maskHashAlg,
                          (const unsigned char*)info->params->pSourceData,
                          info->params->ulSourceDataLen,
                          output, outputLen, maxLen, input, inputLen);
      if (rv != SECSuccess && PORT_GetError() == SEC_ERROR_LIBRARY_FAILURE) {
         sftk_fatalError = PR_TRUE;
     }
     return rv;
 }
 
@@ skipping 159 matching lines @@
                 break;
             }
             context->maxLen = nsslowkey_PrivateModulusLen(privKey);
             context->cipherInfo =  (void *)privKey;
             context->update = (SFTKCipher) 
                 (pMechanism->mechanism == CKM_RSA_X_509
                                         ? sftk_RSADecryptRaw : sftk_RSADecrypt);
         }
         context->destroy = sftk_Null;
         break;
-/* XXX: Disabled until unit tests land.
     case CKM_RSA_PKCS_OAEP:
         if (key_type != CKK_RSA) {
             crv = CKR_KEY_TYPE_INCONSISTENT;
             break;
         }
-        context->multi = PR_FALSE;
-        context->rsa = PR_TRUE;
-        if (pMechanism->ulParameterLen != sizeof(CK_RSA_PKCS_OAEP_PARAMS)) {
+        if (pMechanism->ulParameterLen != sizeof(CK_RSA_PKCS_OAEP_PARAMS) ||
+            !sftk_ValidateOaepParams((CK_RSA_PKCS_OAEP_PARAMS*)pMechanism->pParameter)) {
             crv = CKR_MECHANISM_PARAM_INVALID;
             break;
         }
-        /\* XXX: Need Parameter validation here *\/
+        context->multi = PR_FALSE;
+        context->rsa = PR_TRUE;
         if (isEncrypt) {
             SFTKOAEPEncryptInfo *info = PORT_New(SFTKOAEPEncryptInfo);
             if (info == NULL) {
                 crv = CKR_HOST_MEMORY;
                 break;
             }
             info->params = pMechanism->pParameter;
             info->key = sftk_GetPubKey(key, CKK_RSA, &crv);
             if (info->key == NULL) {
                 PORT_Free(info);
@@ skipping 15 matching lines @@
                 PORT_Free(info);
                 crv = CKR_KEY_HANDLE_INVALID;
                 break;
             }
             context->update = (SFTKCipher) sftk_RSADecryptOAEP;
             context->maxLen = nsslowkey_PrivateModulusLen(info->key);
             context->cipherInfo = info;
         }
         context->destroy = (SFTKDestroy) sftk_Space;
         break;
-*/
     case CKM_RC2_CBC_PAD:
         context->doPad = PR_TRUE;
         /* fall thru */
     case CKM_RC2_ECB:
     case CKM_RC2_CBC:
         context->blockSize = 8;
         if (key_type != CKK_RC2) {
             crv = CKR_KEY_TYPE_INCONSISTENT;
             break;
         }
@@ skipping 1636 matching lines @@
             context->destroy = (SFTKDestroy)sftk_Null;
         }
         context->maxLen = nsslowkey_PrivateModulusLen(privKey);
         break;
     case CKM_RSA_PKCS_PSS:
         if (key_type != CKK_RSA) {
             crv = CKR_KEY_TYPE_INCONSISTENT;
             break;
         } 
         context->rsa = PR_TRUE;
-        if (pMechanism->ulParameterLen != sizeof(CK_RSA_PKCS_PSS_PARAMS)) {
+        if (pMechanism->ulParameterLen != sizeof(CK_RSA_PKCS_PSS_PARAMS) ||
+            !sftk_ValidatePssParams((const CK_RSA_PKCS_PSS_PARAMS*)pMechanism->pParameter)) {
             crv = CKR_MECHANISM_PARAM_INVALID;
             break;
         }
         info = PORT_New(SFTKHashSignInfo);
         if (info == NULL) {
             crv = CKR_HOST_MEMORY;
             break;
         }
         info->params = pMechanism->pParameter;
         info->key = sftk_GetPrivKey(key,CKK_RSA,&crv);
@@ skipping 616 matching lines @@
             context->cipherInfo = pubKey;
             context->destroy = sftk_Null;
         }
         break;
     case CKM_RSA_PKCS_PSS:
         if (key_type != CKK_RSA) {
             crv = CKR_KEY_TYPE_INCONSISTENT;
             break;
         } 
         context->rsa = PR_TRUE;
-        if (pMechanism->ulParameterLen != sizeof(CK_RSA_PKCS_PSS_PARAMS)) {
+        if (pMechanism->ulParameterLen != sizeof(CK_RSA_PKCS_PSS_PARAMS) ||
+            !sftk_ValidatePssParams((const CK_RSA_PKCS_PSS_PARAMS*)pMechanism->pParameter)) {
             crv = CKR_MECHANISM_PARAM_INVALID;
             break;
         }
         info = PORT_New(SFTKHashVerifyInfo);
         if (info == NULL) {
             crv = CKR_HOST_MEMORY;
             break;
         }
         info->params = pMechanism->pParameter;
         info->key = sftk_GetPubKey(key,CKK_RSA,&crv);
@@ skipping 4252 matching lines @@
     att = sftk_FindAttribute(key,CKA_VALUE);
     sftk_FreeObject(key);
     if (!att) {
         return CKR_KEY_HANDLE_INVALID;        
     }
     crv = NSC_DigestUpdate(hSession,(CK_BYTE_PTR)att->attrib.pValue,
                            att->attrib.ulValueLen);
     sftk_FreeAttribute(att);
     return crv;
 }