Use CERT_GetCertKeyType to get KeyType for ssl3_PlatformSignHashes.

net/third_party/nss/ssl/sslplatf.c

 /*
  * Platform specific crypto wrappers
  *
  * ***** BEGIN LICENSE BLOCK *****
  * Version: MPL 1.1/GPL 2.0/LGPL 2.1
  *
  * The contents of this file are subject to the Mozilla Public License Version
  * 1.1 (the "License"); you may not use this file except in compliance with
  * the License. You may obtain a copy of the License at
  * http://www.mozilla.org/MPL/
@@ skipping 93 matching lines @@
     if (key) {
         if (key->dwKeySpec != CERT_NCRYPT_KEY_SPEC)
             CryptReleaseContext(key->hCryptProv, 0);
         /* FIXME(rsleevi): Close CNG keys. */
         PORT_Free(key);
     }
 }
 
 SECStatus
 ssl3_PlatformSignHashes(SSL3Hashes *hash, PlatformKey key, SECItem *buf, 
-                        PRBool isTLS)
+                        PRBool isTLS, KeyType keyType)
 {
     SECStatus    rv             = SECFailure;
     PRBool       doDerEncode    = PR_FALSE;
     SECItem      hashItem;
-    HCRYPTKEY    hKey           = 0;
     DWORD        argLen         = 0;
-    ALG_ID       keyAlg         = 0;
     DWORD        signatureLen   = 0;
     ALG_ID       hashAlg        = 0;
     HCRYPTHASH   hHash          = 0;
     DWORD        hashLen        = 0;
     unsigned int i              = 0;
 
     buf->data = NULL;
-    if (!CryptGetUserKey(key->hCryptProv, key->dwKeySpec, &hKey)) {
-        if (GetLastError() == NTE_NO_KEY) {
-            PORT_SetError(SEC_ERROR_NO_KEY);
-        } else {
-            PORT_SetError(SEC_ERROR_INVALID_KEY);
-        }
-        goto done;
-    }
 
-    argLen = sizeof(keyAlg);
-    if (!CryptGetKeyParam(hKey, KP_ALGID, (BYTE*)&keyAlg, &argLen, 0)) {
-        PORT_SetError(SEC_ERROR_INVALID_KEY);
-        goto done;
-    }
-
-    switch (keyAlg) {
-        case CALG_RSA_KEYX:
-        case CALG_RSA_SIGN:
+    switch (keyType) {
+        case rsaKey:
             hashAlg       = CALG_SSL3_SHAMD5;
             hashItem.data = hash->md5;
             hashItem.len  = sizeof(SSL3Hashes);
             break;
-        case CALG_DSS_SIGN:
-        case CALG_ECDSA:
-            if (keyAlg == CALG_ECDSA) {
+        case dsaKey:
+        case ecKey:
+            if (keyType == ecKey) {
                 doDerEncode = PR_TRUE;
             } else {
                 doDerEncode = isTLS;
             }
             hashAlg       = CALG_SHA1;
             hashItem.data = hash->sha;
             hashItem.len  = sizeof(hash->sha);
             break;
         default:
             PORT_SetError(SEC_ERROR_INVALID_KEY);
@@ skipping 52 matching lines @@
             PORT_Free(derSig.data);
         }
     } else {
         rv = SECSuccess;
     }
 
     PRINT_BUF(60, (NULL, "signed hashes", buf->data, buf->len));
 done:
     if (hHash)
         CryptDestroyHash(hHash);
-    if (hKey)
-        CryptDestroyKey(hKey);
     if (rv != SECSuccess && buf->data) {
         PORT_Free(buf->data);
         buf->data = NULL;
     }
     return rv;
 }
 
 #elif defined(XP_MACOSX)
 #include <Security/cssm.h>
 
 void
 ssl_FreePlatformKey(PlatformKey key)
 {
     CFRelease(key);
 }
 
 SECStatus
 ssl3_PlatformSignHashes(SSL3Hashes *hash, PlatformKey key, SECItem *buf, 
-                        PRBool isTLS)
+                        PRBool isTLS, KeyType keyType)
 {
     SECStatus       rv                  = SECFailure;
     PRBool          doDerEncode         = PR_FALSE;
     unsigned int    signatureLen;
     OSStatus        status              = noErr;
     CSSM_CSP_HANDLE cspHandle           = 0;
     const CSSM_KEY *cssmKey             = NULL;
     CSSM_ALGORITHMS sigAlg;
     const CSSM_ACCESS_CREDENTIALS * cssmCreds = NULL;
     CSSM_RETURN     cssmRv;
@@ skipping 23 matching lines @@
     if (signatureLen == 0) {
         PORT_SetError(SEC_ERROR_INVALID_KEY);
         goto done;
     }
 
     buf->data = (unsigned char *)PORT_Alloc(signatureLen);
     if (!buf->data)
         goto done;    /* error code was set. */
 
     sigAlg = cssmKey->KeyHeader.AlgorithmId;
     switch (sigAlg) {

[wtc, 2013/04/25 19:10:17]

You should also replace |sigAlg| with |keyType| in the
XP_MACOSX code.

[mef, 2013/04/25 19:31:15]

I'll be glad to do that.
On 2013/04/25 19:10:17, wtc wrote:

         case CSSM_ALGID_RSA:
             hashData.Data   = hash->md5;
             hashData.Length = sizeof(SSL3Hashes);
             break;
         case CSSM_ALGID_ECDSA:
         case CSSM_ALGID_DSA:
             if (sigAlg == CSSM_ALGID_ECDSA) {
                 doDerEncode = PR_TRUE;
             } else {
                 doDerEncode = isTLS;
@@ skipping 81 matching lines @@
     return rv;
 }
 #else
 void
 ssl_FreePlatformKey(PlatformKey key)
 {
 }
 
 SECStatus
 ssl3_PlatformSignHashes(SSL3Hashes *hash, PlatformKey key, SECItem *buf,
-                        PRBool isTLS)
+                        PRBool isTLS, KeyType keyType)
 {
     PORT_SetError(PR_NOT_IMPLEMENTED_ERROR);
     return SECFailure;
 }
 #endif
 
 #endif /* NSS_PLATFORM_CLIENT_AUTH */