Merge 222724 "The NSS client auth (as opposed to NSS_PLATFORM_CL..."

net/third_party/nss/ssl/ssl3con.c

 /* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
 /*
  * SSL3 Protocol
  *
  * This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 /* TODO(ekr): Implement HelloVerifyRequest on server side. OK for now. */
 
@@ skipping 6571 matching lines @@
 loser:
     PORT_SetError( errCode );
     return SECFailure;
 
 no_memory:      /* no-memory error has already been set. */
     ssl_MapLowLevelError(SSL_ERROR_SERVER_KEY_EXCHANGE_FAILURE);
     return SECFailure;
 }
 
 
+/*
+ * Returns true if the client authentication key is an RSA or DSA key that
+ * may be able to sign only SHA-1 hashes.
+ */
+static PRBool
+ssl3_ClientKeyPrefersSHA1(sslSocket *ss)
+{
+    SECKEYPublicKey *pubk;
+    PRBool prefer_sha1 = PR_FALSE;
+
+#if defined(NSS_PLATFORM_CLIENT_AUTH) && defined(_WIN32)
+    /* If the key is in CAPI, assume conservatively that the CAPI service
+     * provider may be unable to sign SHA-256 hashes.
+     */
+    if (ss->ssl3.platformClientKey->dwKeySpec != CERT_NCRYPT_KEY_SPEC) {
+        /* CAPI only supports RSA and DSA signatures, so we don't need to
+         * check the key type. */
+        return PR_TRUE;
+    }
+#endif  /* NSS_PLATFORM_CLIENT_AUTH && _WIN32 */
+
+    /* If the key is a 1024-bit RSA or DSA key, assume conservatively that
+     * it may be unable to sign SHA-256 hashes. This is the case for older
+     * Estonian ID cards that have 1024-bit RSA keys. In FIPS 186-2 and
+     * older, DSA key size is at most 1024 bits and the hash function must
+     * be SHA-1.
+     */
+    pubk = CERT_ExtractPublicKey(ss->ssl3.clientCertificate);
+    if (pubk == NULL) {
+        return PR_FALSE;
+    }
+    if (pubk->keyType == rsaKey || pubk->keyType == dsaKey) {
+        prefer_sha1 = SECKEY_PublicKeyStrength(pubk) <= 128;
+    }
+    SECKEY_DestroyPublicKey(pubk);
+    return prefer_sha1;
+}
+
+/* Destroys the backup handshake hash context if we don't need it. */
+static void
+ssl3_DestroyBackupHandshakeHashIfNotNeeded(sslSocket *ss,
+                                           const SECItem *algorithms)
+{
+    PRBool need_backup_hash = PR_FALSE;
+    unsigned int i;
+
+    PORT_Assert(ss->ssl3.hs.md5);
+    if (ssl3_ClientKeyPrefersSHA1(ss)) {
+        /* Use SHA-1 if the server supports it. */
+        for (i = 0; i < algorithms->len; i += 2) {
+            if (algorithms->data[i] == tls_hash_sha1 &&
+                (algorithms->data[i+1] == tls_sig_rsa ||
+                 algorithms->data[i+1] == tls_sig_dsa)) {
+                need_backup_hash = PR_TRUE;
+                break;
+            }
+        }
+    }
+    if (!need_backup_hash) {
+        PK11_DestroyContext(ss->ssl3.hs.md5, PR_TRUE);
+        ss->ssl3.hs.md5 = NULL;
+    }
+}
+
 typedef struct dnameNode {
     struct dnameNode *next;
     SECItem           name;
 } dnameNode;
 
 /* Called from ssl3_HandleHandshakeMessage() when it has deciphered a complete
  * ssl3 Certificate Request message.
  * Caller must hold Handshake and RecvBuf locks.
  */
 static SECStatus
@@ skipping 172 matching lines @@
                 if (ss->ssl3.clientCertificate != NULL) {
                     CERT_DestroyCertificate(ss->ssl3.clientCertificate);
                     ss->ssl3.clientCertificate = NULL;
                 }
                 if (ss->ssl3.platformClientKey) {
                     ssl_FreePlatformKey(ss->ssl3.platformClientKey);
                     ss->ssl3.platformClientKey = (PlatformKey)NULL;
                 }
                 goto send_no_certificate;
             }
-
-            if (isTLS12 && ss->ssl3.hs.md5) {
-                PRBool need_backup_hash = PR_FALSE;
-                PRBool prefer_sha1 = PR_FALSE;
-#ifdef _WIN32
-                /* If the key is in CAPI, assume conservatively that the CAPI
-                 * service provider may be unable to sign SHA-256 hashes.
-                 */
-                if (ss->ssl3.platformClientKey->dwKeySpec !=
-                    CERT_NCRYPT_KEY_SPEC) {
-                    /* CAPI only supports RSA and DSA signatures, so we don't
-                     * need to check the key type. */
-                    prefer_sha1 = PR_TRUE;
-                }
-#endif  /* _WIN32 */
-                /* If the key is a 1024-bit RSA or DSA key, assume
-                 * conservatively that it may be unable to sign SHA-256
-                 * hashes. This is the case for older Estonian ID cards that
-                 * have 1024-bit RSA keys. In FIPS 186-2 and older, DSA key
-                 * size is at most 1024 bits and the hash function must be
-                 * SHA-1.
-                 */
-                if (!prefer_sha1) {
-                    SECKEYPublicKey *pubk =
-                        CERT_ExtractPublicKey(ss->ssl3.clientCertificate);
-                    if (pubk == NULL) {
-                        errCode = SSL_ERROR_EXTRACT_PUBLIC_KEY_FAILURE;
-                        goto loser;
-                    }
-                    if (pubk->keyType == rsaKey || pubk->keyType == dsaKey) {
-                        prefer_sha1 = SECKEY_PublicKeyStrength(pubk) <= 128;
-                    }
-                    SECKEY_DestroyPublicKey(pubk);
-                }
-                /* Use SHA-1 if the server supports it. */
-                if (prefer_sha1) {
-                    for (i = 0; i < algorithms.len; i += 2) {
-                        if (algorithms.data[i] == tls_hash_sha1 &&
-                            (algorithms.data[i+1] == tls_sig_rsa ||
-                             algorithms.data[i+1] == tls_sig_dsa)) {
-                            need_backup_hash = PR_TRUE;
-                            break;
-                        }
-                    }
-                }
-                if (!need_backup_hash) {
-                    PK11_DestroyContext(ss->ssl3.hs.md5, PR_TRUE);
-                    ss->ssl3.hs.md5 = NULL;
-                }
+            if (isTLS12) {
+                ssl3_DestroyBackupHandshakeHashIfNotNeeded(ss, &algorithms);
             }
             break;  /* not an error */
         }
 #endif   /* NSS_PLATFORM_CLIENT_AUTH */
         /* check what the callback function returned */
         if ((!ss->ssl3.clientCertificate) || (!ss->ssl3.clientPrivateKey)) {
             /* we are missing either the key or cert */
             if (ss->ssl3.clientCertificate) {
                 /* got a cert, but no key - free it */
                 CERT_DestroyCertificate(ss->ssl3.clientCertificate);
@@ skipping 16 matching lines @@
             if (ss->ssl3.clientCertificate != NULL) {
                 CERT_DestroyCertificate(ss->ssl3.clientCertificate);
                 ss->ssl3.clientCertificate = NULL;
             }
             if (ss->ssl3.clientPrivateKey != NULL) {
                 SECKEY_DestroyPrivateKey(ss->ssl3.clientPrivateKey);
                 ss->ssl3.clientPrivateKey = NULL;
             }
             goto send_no_certificate;
         }
+        if (isTLS12) {
+            ssl3_DestroyBackupHandshakeHashIfNotNeeded(ss, &algorithms);
+        }
         break;  /* not an error */
 
     case SECFailure:
     default:
 send_no_certificate:
         if (isTLS) {
             ss->ssl3.sendEmptyCert = PR_TRUE;
         } else {
             (void)SSL3_SendAlert(ss, alert_warning, no_certificate);
         }
@@ skipping 5198 matching lines @@
             PORT_Free(ss->ssl3.hs.recvdFragments.buf);
         }
     }
 
     ss->ssl3.initialized = PR_FALSE;
 
     SECITEM_FreeItem(&ss->ssl3.nextProto, PR_FALSE);
 }
 
 /* End of ssl3con.c */