Add RSA-OAEP support from upstream NSS bugs 1009794 and 1009785

nss/lib/softoken/pkcs11c.c

Index: nss/lib/softoken/pkcs11c.c
diff --git a/nss/lib/softoken/pkcs11c.c b/nss/lib/softoken/pkcs11c.c
index 472b71a77d1cdcedda1ac8808d3773c76d184ff7..080dfda0c22f9867321180b31cfc312de09da4ae 100644
--- a/nss/lib/softoken/pkcs11c.c
+++ b/nss/lib/softoken/pkcs11c.c
@@ -302,6 +302,46 @@ GetHashTypeFromMechanism(CK_MECHANISM_TYPE mech)
     }
 }
 
+/*
+ * Returns true if "params" contains a valid set of PSS parameters
+ */
+static PRBool
+sftk_ValidatePssParams(const CK_RSA_PKCS_PSS_PARAMS *params)
+{
+    if (!params) {
+        return PR_FALSE;
+    }
+    if (GetHashTypeFromMechanism(params->hashAlg) == HASH_AlgNULL ||
+        GetHashTypeFromMechanism(params->mgf) == HASH_AlgNULL) {
+        return PR_FALSE;
+    }
+    return PR_TRUE;
+}
+
+/*
+ * Returns true if "params" contains a valid set of OAEP parameters
+ */
+static PRBool
+sftk_ValidateOaepParams(const CK_RSA_PKCS_OAEP_PARAMS *params)
+{
+    if (!params) {
+        return PR_FALSE;
+    }
+    /* The requirements of ulSourceLen/pSourceData come from PKCS #11, which
+     * state:
+     *   If the parameter is empty, pSourceData must be NULL and
+     *   ulSourceDataLen must be zero.
+     */
+    if (params->source != CKZ_DATA_SPECIFIED ||
+        (GetHashTypeFromMechanism(params->hashAlg) == HASH_AlgNULL) ||
+        (GetHashTypeFromMechanism(params->mgf) == HASH_AlgNULL) ||
+        (params->ulSourceDataLen == 0 && params->pSourceData != NULL) ||
+        (params->ulSourceDataLen != 0 && params->pSourceData == NULL)) {
+        return PR_FALSE;
+    }
+    return PR_TRUE;
+}
+
 /* 
  * return a context based on the SFTKContext type.
  */
@@ -588,11 +628,6 @@ sftk_RSAEncryptOAEP(SFTKOAEPEncryptInfo *info, unsigned char *output,
     hashAlg = GetHashTypeFromMechanism(info->params->hashAlg);
     maskHashAlg = GetHashTypeFromMechanism(info->params->mgf);
 
-    if (info->params->source != CKZ_DATA_SPECIFIED) {
-        PORT_SetError(SEC_ERROR_INVALID_ALGORITHM);
-        return SECFailure;
-    }
-
     return RSA_EncryptOAEP(&info->key->u.rsa, hashAlg, maskHashAlg,
                            (const unsigned char*)info->params->pSourceData,
                            info->params->ulSourceDataLen, NULL, 0,
@@ -617,11 +652,6 @@ sftk_RSADecryptOAEP(SFTKOAEPDecryptInfo *info, unsigned char *output,
     hashAlg = GetHashTypeFromMechanism(info->params->hashAlg);
     maskHashAlg = GetHashTypeFromMechanism(info->params->mgf);
 
-    if (info->params->source != CKZ_DATA_SPECIFIED) {
-        PORT_SetError(SEC_ERROR_INVALID_ALGORITHM);
-        return SECFailure;
-    }
-
     rv = RSA_DecryptOAEP(&info->key->u.rsa, hashAlg, maskHashAlg,
                          (const unsigned char*)info->params->pSourceData,
                          info->params->ulSourceDataLen,
@@ -801,19 +831,18 @@ sftk_CryptInit(CK_SESSION_HANDLE hSession, CK_MECHANISM_PTR pMechanism,
 	}
 	context->destroy = sftk_Null;
 	break;
-/* XXX: Disabled until unit tests land.
     case CKM_RSA_PKCS_OAEP:
 	if (key_type != CKK_RSA) {
 	    crv = CKR_KEY_TYPE_INCONSISTENT;
 	    break;
 	}
-	context->multi = PR_FALSE;
-	context->rsa = PR_TRUE;
-	if (pMechanism->ulParameterLen != sizeof(CK_RSA_PKCS_OAEP_PARAMS)) {
+	if (pMechanism->ulParameterLen != sizeof(CK_RSA_PKCS_OAEP_PARAMS) ||
+	    !sftk_ValidateOaepParams((CK_RSA_PKCS_OAEP_PARAMS*)pMechanism->pParameter)) {
 	    crv = CKR_MECHANISM_PARAM_INVALID;
 	    break;
 	}
-	/\* XXX: Need Parameter validation here *\/
+	context->multi = PR_FALSE;
+	context->rsa = PR_TRUE;
 	if (isEncrypt) {
 	    SFTKOAEPEncryptInfo *info = PORT_New(SFTKOAEPEncryptInfo);
 	    if (info == NULL) {
@@ -849,7 +878,6 @@ sftk_CryptInit(CK_SESSION_HANDLE hSession, CK_MECHANISM_PTR pMechanism,
 	}
 	context->destroy = (SFTKDestroy) sftk_Space;
 	break;
-*/
     case CKM_RC2_CBC_PAD:
 	context->doPad = PR_TRUE;
 	/* fall thru */
@@ -2506,7 +2534,8 @@ finish_rsa:
 	    break;
 	} 
 	context->rsa = PR_TRUE;
-	if (pMechanism->ulParameterLen != sizeof(CK_RSA_PKCS_PSS_PARAMS)) {
+	if (pMechanism->ulParameterLen != sizeof(CK_RSA_PKCS_PSS_PARAMS) ||
+	    !sftk_ValidatePssParams((const CK_RSA_PKCS_PSS_PARAMS*)pMechanism->pParameter)) {
 	    crv = CKR_MECHANISM_PARAM_INVALID;
 	    break;
 	}
@@ -3143,7 +3172,8 @@ finish_rsa:
 	    break;
 	} 
 	context->rsa = PR_TRUE;
-	if (pMechanism->ulParameterLen != sizeof(CK_RSA_PKCS_PSS_PARAMS)) {
+	if (pMechanism->ulParameterLen != sizeof(CK_RSA_PKCS_PSS_PARAMS) ||
+	    !sftk_ValidatePssParams((const CK_RSA_PKCS_PSS_PARAMS*)pMechanism->pParameter)) {
 	    crv = CKR_MECHANISM_PARAM_INVALID;
 	    break;
 	}