Merge 222724 "The NSS client auth (as opposed to NSS_PLATFORM_CL..."

net/third_party/nss/ssl/ssl3con.c

Index: net/third_party/nss/ssl/ssl3con.c
===================================================================
--- net/third_party/nss/ssl/ssl3con.c	(revision 223079)
+++ net/third_party/nss/ssl/ssl3con.c	(working copy)
@@ -6589,6 +6589,70 @@
 }
 
 
+/*
+ * Returns true if the client authentication key is an RSA or DSA key that
+ * may be able to sign only SHA-1 hashes.
+ */
+static PRBool
+ssl3_ClientKeyPrefersSHA1(sslSocket *ss)
+{
+    SECKEYPublicKey *pubk;
+    PRBool prefer_sha1 = PR_FALSE;
+
+#if defined(NSS_PLATFORM_CLIENT_AUTH) && defined(_WIN32)
+    /* If the key is in CAPI, assume conservatively that the CAPI service
+     * provider may be unable to sign SHA-256 hashes.
+     */
+    if (ss->ssl3.platformClientKey->dwKeySpec != CERT_NCRYPT_KEY_SPEC) {
+	/* CAPI only supports RSA and DSA signatures, so we don't need to
+	 * check the key type. */
+	return PR_TRUE;
+    }
+#endif  /* NSS_PLATFORM_CLIENT_AUTH && _WIN32 */
+
+    /* If the key is a 1024-bit RSA or DSA key, assume conservatively that
+     * it may be unable to sign SHA-256 hashes. This is the case for older
+     * Estonian ID cards that have 1024-bit RSA keys. In FIPS 186-2 and
+     * older, DSA key size is at most 1024 bits and the hash function must
+     * be SHA-1.
+     */
+    pubk = CERT_ExtractPublicKey(ss->ssl3.clientCertificate);
+    if (pubk == NULL) {
+	return PR_FALSE;
+    }
+    if (pubk->keyType == rsaKey || pubk->keyType == dsaKey) {
+	prefer_sha1 = SECKEY_PublicKeyStrength(pubk) <= 128;
+    }
+    SECKEY_DestroyPublicKey(pubk);
+    return prefer_sha1;
+}
+
+/* Destroys the backup handshake hash context if we don't need it. */
+static void
+ssl3_DestroyBackupHandshakeHashIfNotNeeded(sslSocket *ss,
+					   const SECItem *algorithms)
+{
+    PRBool need_backup_hash = PR_FALSE;
+    unsigned int i;
+
+    PORT_Assert(ss->ssl3.hs.md5);
+    if (ssl3_ClientKeyPrefersSHA1(ss)) {
+	/* Use SHA-1 if the server supports it. */
+	for (i = 0; i < algorithms->len; i += 2) {
+	    if (algorithms->data[i] == tls_hash_sha1 &&
+		(algorithms->data[i+1] == tls_sig_rsa ||
+		 algorithms->data[i+1] == tls_sig_dsa)) {
+		need_backup_hash = PR_TRUE;
+		break;
+	    }
+	}
+    }
+    if (!need_backup_hash) {
+	PK11_DestroyContext(ss->ssl3.hs.md5, PR_TRUE);
+	ss->ssl3.hs.md5 = NULL;
+    }
+}
+
 typedef struct dnameNode {
     struct dnameNode *next;
     SECItem           name;
@@ -6781,55 +6845,8 @@
 		}
 		goto send_no_certificate;
 	    }
-
-	    if (isTLS12 && ss->ssl3.hs.md5) {
-		PRBool need_backup_hash = PR_FALSE;
-		PRBool prefer_sha1 = PR_FALSE;
-#ifdef _WIN32
-		/* If the key is in CAPI, assume conservatively that the CAPI
-		 * service provider may be unable to sign SHA-256 hashes.
-		 */
-		if (ss->ssl3.platformClientKey->dwKeySpec !=
-		    CERT_NCRYPT_KEY_SPEC) {
-		    /* CAPI only supports RSA and DSA signatures, so we don't
-		     * need to check the key type. */
-		    prefer_sha1 = PR_TRUE;
-		}
-#endif  /* _WIN32 */
-		/* If the key is a 1024-bit RSA or DSA key, assume
-		 * conservatively that it may be unable to sign SHA-256
-		 * hashes. This is the case for older Estonian ID cards that
-		 * have 1024-bit RSA keys. In FIPS 186-2 and older, DSA key
-		 * size is at most 1024 bits and the hash function must be
-		 * SHA-1.
-		 */
-		if (!prefer_sha1) {
-		    SECKEYPublicKey *pubk =
-			CERT_ExtractPublicKey(ss->ssl3.clientCertificate);
-		    if (pubk == NULL) {
-			errCode = SSL_ERROR_EXTRACT_PUBLIC_KEY_FAILURE;
- 			goto loser;
-		    }
-		    if (pubk->keyType == rsaKey || pubk->keyType == dsaKey) {
-			prefer_sha1 = SECKEY_PublicKeyStrength(pubk) <= 128;
-		    }
-		    SECKEY_DestroyPublicKey(pubk);
-		}
-		/* Use SHA-1 if the server supports it. */
-		if (prefer_sha1) {
-		    for (i = 0; i < algorithms.len; i += 2) {
-			if (algorithms.data[i] == tls_hash_sha1 &&
-			    (algorithms.data[i+1] == tls_sig_rsa ||
-			     algorithms.data[i+1] == tls_sig_dsa)) {
-			    need_backup_hash = PR_TRUE;
-			    break;
-			}
-		    }
-		}
-		if (!need_backup_hash) {
-		    PK11_DestroyContext(ss->ssl3.hs.md5, PR_TRUE);
-		    ss->ssl3.hs.md5 = NULL;
-		}
+	    if (isTLS12) {
+		ssl3_DestroyBackupHandshakeHashIfNotNeeded(ss, &algorithms);
 	    }
 	    break;  /* not an error */
 	}
@@ -6866,6 +6883,9 @@
 	    }
 	    goto send_no_certificate;
 	}
+	if (isTLS12) {
+	    ssl3_DestroyBackupHandshakeHashIfNotNeeded(ss, &algorithms);
+	}
 	break;	/* not an error */
 
     case SECFailure: