Add DTLS support to NSS, contributed by Eric Rescorla.

net/third_party/nss/ssl/sslsock.c

Index: net/third_party/nss/ssl/sslsock.c
===================================================================
--- net/third_party/nss/ssl/sslsock.c	(revision 127709)
+++ net/third_party/nss/ssl/sslsock.c	(working copy)
@@ -194,11 +194,18 @@
 /*
  * default range of enabled SSL/TLS protocols
  */
-static SSLVersionRange versions_defaults = {
+static SSLVersionRange versions_defaults_stream = {
     SSL_LIBRARY_VERSION_3_0,
     SSL_LIBRARY_VERSION_TLS_1_0
 };
 
+static SSLVersionRange versions_defaults_datagram = {
+    SSL_LIBRARY_VERSION_TLS_1_1,
+    SSL_LIBRARY_VERSION_TLS_1_1
+};
+
+#define VERSIONS_DEFAULTS(variant) (variant == ssl_variant_stream ? &versions_defaults_stream : &versions_defaults_datagram)
+
 sslSessionIDLookupFunc  ssl_sid_lookup;
 sslSessionIDCacheFunc   ssl_sid_cache;
 sslSessionIDUncacheFunc ssl_sid_uncache;
@@ -217,7 +224,7 @@
 #define LOCKSTATUS_OFFSET 10 /* offset of ENABLED */
 
 /* forward declarations. */
-static sslSocket *ssl_NewSocket(PRBool makeLocks);
+static sslSocket *ssl_NewSocket(PRBool makeLocks, SSLProtocolVariant variant);
 static SECStatus  ssl_MakeLocks(sslSocket *ss);
 static void       ssl_SetDefaultsFromEnvironment(void);
 static PRStatus   ssl_PushIOLayer(sslSocket *ns, PRFileDesc *stack, 
@@ -281,7 +288,10 @@
     sslSocket *ss;
     SECStatus rv;
 
-    ss = ssl_NewSocket((PRBool)(!os->opt.noLocks));
+    /* Not implemented for datagram */
+    PORT_Assert(os->protocolVariant == ssl_variant_stream);
+
+    ss = ssl_NewSocket((PRBool)(!os->opt.noLocks), os->protocolVariant);
     if (ss) {
 	ss->opt                = os->opt;
 	ss->opt.useSocks       = PR_FALSE;
@@ -698,6 +708,16 @@
 	break;
 
       case SSL_ENABLE_TLS:
+        if (IS_DTLS(ss)) {
+	    if (on) {
+		PORT_SetError(SEC_ERROR_INVALID_ARGS);
+		rv = SECFailure; /* not allowed */
+	    }
+	    else {
+		rv = SECSuccess; /* no-op */
+	    }
+	    break;
+	}
 	ssl_EnableTLS(&ss->vrange, on);
 	ss->preferredCipher     = NULL;
 	if (ss->cipherSpecs) {
@@ -708,6 +728,16 @@
 	break;
 
       case SSL_ENABLE_SSL3:
+        if (IS_DTLS(ss)) {
+	    if (on) {
+		PORT_SetError(SEC_ERROR_INVALID_ARGS);
+		rv = SECFailure; /* not allowed */
+	    }
+	    else {
+		rv = SECSuccess; /* no-op */
+	    }
+	    break;
+	}
 	ssl_EnableSSL3(&ss->vrange, on);
 	ss->preferredCipher     = NULL;
 	if (ss->cipherSpecs) {
@@ -716,8 +746,17 @@
 	    ss->sizeCipherSpecs = 0;
 	}
 	break;
-
       case SSL_ENABLE_SSL2:
+        if (IS_DTLS(ss)) {
+	    if (on) {
+		PORT_SetError(SEC_ERROR_INVALID_ARGS);
+		rv = SECFailure; /* not allowed */
+	    }
+	    else {
+		rv = SECSuccess; /* no-op */
+	    }
+	    break;
+	}
 	ss->opt.enableSSL2       = on;
 	if (on) {
 	    ss->opt.v2CompatibleHello = on;
@@ -743,6 +782,16 @@
 	break;
 
       case SSL_V2_COMPATIBLE_HELLO:
+        if (IS_DTLS(ss)) {
+	    if (on) {
+		PORT_SetError(SEC_ERROR_INVALID_ARGS);
+		rv = SECFailure; /* not allowed */
+	    }
+	    else {
+		rv = SECSuccess; /* no-op */
+	    }
+	    break;
+	}
       	ss->opt.v2CompatibleHello = on;
 	if (!on) {
 	    ss->opt.enableSSL2    = on;
@@ -938,10 +987,10 @@
     case SSL_HANDSHAKE_AS_CLIENT: on = ssl_defaults.handshakeAsClient;  break;
     case SSL_HANDSHAKE_AS_SERVER: on = ssl_defaults.handshakeAsServer;  break;
     case SSL_ENABLE_TLS:
-	on = versions_defaults.max >= SSL_LIBRARY_VERSION_TLS_1_0;
+	on = versions_defaults_stream.max >= SSL_LIBRARY_VERSION_TLS_1_0;
 	break;
     case SSL_ENABLE_SSL3:
-	on = versions_defaults.min == SSL_LIBRARY_VERSION_3_0;
+	on = versions_defaults_stream.min == SSL_LIBRARY_VERSION_3_0;
 	break;
     case SSL_ENABLE_SSL2:         on = ssl_defaults.enableSSL2;         break;
     case SSL_NO_CACHE:            on = ssl_defaults.noCache;		break;
@@ -1034,11 +1083,11 @@
 	break;
 
       case SSL_ENABLE_TLS:
-	ssl_EnableTLS(&versions_defaults, on);
+	ssl_EnableTLS(&versions_defaults_stream, on);
 	break;
 
       case SSL_ENABLE_SSL3:
-	ssl_EnableSSL3(&versions_defaults, on);
+	ssl_EnableSSL3(&versions_defaults_stream, on);
 	break;
 
       case SSL_ENABLE_SSL2:
@@ -1361,7 +1410,7 @@
 
 /* LOCKS ??? XXX */
 PRFileDesc *
-SSL_ImportFD(PRFileDesc *model, PRFileDesc *fd)
+ssl_ImportFD(PRFileDesc *model, PRFileDesc *fd, SSLProtocolVariant variant)
 {
     sslSocket * ns = NULL;
     PRStatus    rv;
@@ -1374,10 +1423,10 @@
 
     if (model == NULL) {
 	/* Just create a default socket if we're given NULL for the model */
-	ns = ssl_NewSocket((PRBool)(!ssl_defaults.noLocks));
+	ns = ssl_NewSocket((PRBool)(!ssl_defaults.noLocks), variant);
     } else {
 	sslSocket * ss = ssl_FindSocket(model);
-	if (ss == NULL) {
+	if (ss == NULL || ss->protocolVariant != variant) {
 	    SSL_DBG(("%d: SSL[%d]: bad model socket in ssl_ImportFD", 
 	    	      SSL_GETPID(), model));
 	    return NULL;
@@ -1400,9 +1449,22 @@
     PORT_Assert(ns);
     if (ns)
 	ns->TCPconnected = (PR_SUCCESS == ssl_DefGetpeername(ns, &addr));
+
     return fd;
 }
 
+PRFileDesc *
+SSL_ImportFD(PRFileDesc *model, PRFileDesc *fd)
+{
+    return ssl_ImportFD(model, fd, ssl_variant_stream);
+}
+
+PRFileDesc *
+DTLS_ImportFD(PRFileDesc *model, PRFileDesc *fd)
+{
+    return ssl_ImportFD(model, fd, ssl_variant_datagram);
+}
+
 SECStatus
 SSL_SetNextProtoCallback(PRFileDesc *fd, SSLNextProtoCallback callback,
 			 void *arg)
@@ -1667,9 +1729,20 @@
 ssl3_VersionIsSupported(SSLProtocolVariant protocolVariant,
 			SSL3ProtocolVersion version)
 {
-    return protocolVariant == ssl_variant_stream &&
-	   version >= SSL_LIBRARY_VERSION_3_0 &&
-	   version <= SSL_LIBRARY_VERSION_MAX_SUPPORTED;
+    switch(protocolVariant) {
+    case ssl_variant_stream:
+	return(version >= SSL_LIBRARY_VERSION_3_0 &&
+	       version <= SSL_LIBRARY_VERSION_MAX_SUPPORTED);
+	break;  /* Belt and suspenders */

[ekr, 2012/03/26 21:49:18]

I guess this is house style, but I've gotten very paranoid about having any case
fall through even in situations where you are exiting the function. I find that
when you refactor you sometimes miss these. Your call though.

+    case ssl_variant_datagram:
+	return(version >= SSL_LIBRARY_VERSION_TLS_1_1 &&
+	       version <= SSL_LIBRARY_VERSION_MAX_SUPPORTED);
+	break;  /* Belt and suspenders */
+    }
+    
+    /* Can't get here */
+    PORT_Assert(PR_FALSE);
+    return PR_FALSE;
 }
 
 /* Returns PR_TRUE if the given version range is valid and
@@ -1689,14 +1762,21 @@
 SSL_VersionRangeGetSupported(SSLProtocolVariant protocolVariant,
 			     SSLVersionRange *vrange)
 {
-    if (protocolVariant != ssl_variant_stream || !vrange) {
-	PORT_SetError(SEC_ERROR_INVALID_ARGS);
+    switch(protocolVariant) {
+    case ssl_variant_stream:
+	vrange->min = SSL_LIBRARY_VERSION_3_0;
+	vrange->max = SSL_LIBRARY_VERSION_MAX_SUPPORTED;
+	break;
+    case ssl_variant_datagram:
+	vrange->min = SSL_LIBRARY_VERSION_TLS_1_1;
+	vrange->max = SSL_LIBRARY_VERSION_MAX_SUPPORTED;
+	break;
+    default:
+	/* Can't get here */
+	PORT_Assert(PR_FALSE);
 	return SECFailure;
     }
 
-    vrange->min = SSL_LIBRARY_VERSION_3_0;
-    vrange->max = SSL_LIBRARY_VERSION_MAX_SUPPORTED;
-
     return SECSuccess;
 }
 
@@ -1704,12 +1784,14 @@
 SSL_VersionRangeGetDefault(SSLProtocolVariant protocolVariant,
 			   SSLVersionRange *vrange)
 {
-    if (protocolVariant != ssl_variant_stream || !vrange) {
+    if ((protocolVariant != ssl_variant_stream 
+	 && protocolVariant != ssl_variant_datagram)
+	|| !vrange) {
 	PORT_SetError(SEC_ERROR_INVALID_ARGS);
 	return SECFailure;
     }
 
-    *vrange = versions_defaults;
+    *vrange = *VERSIONS_DEFAULTS(protocolVariant);
 
     return SECSuccess;
 }
@@ -1723,7 +1805,7 @@
 	return SECFailure;
     }
 
-    versions_defaults = *vrange;
+    *VERSIONS_DEFAULTS(protocolVariant) = *vrange;    
 
     return SECSuccess;
 }
@@ -2830,7 +2912,7 @@
 ** Create a newsocket structure for a file descriptor.
 */
 static sslSocket *
-ssl_NewSocket(PRBool makeLocks)
+ssl_NewSocket(PRBool makeLocks, SSLProtocolVariant protocolVariant)
 {
     sslSocket *ss;
 
@@ -2851,7 +2933,7 @@
 	ss->opt                = ssl_defaults;
 	ss->opt.useSocks       = PR_FALSE;
 	ss->opt.noLocks        = !makeLocks;
-	ss->vrange             = versions_defaults;
+	ss->vrange             = *VERSIONS_DEFAULTS(protocolVariant);
 
 	ss->peerID             = NULL;
 	ss->rTimeout	       = PR_INTERVAL_NO_TIMEOUT;
@@ -2890,7 +2972,7 @@
 	ssl_ChooseOps(ss);
 	ssl2_InitSocketPolicy(ss);
 	ssl3_InitSocketPolicy(ss);
-
+        
 	if (makeLocks) {
 	    status = ssl_MakeLocks(ss);
 	    if (status != SECSuccess)
@@ -2907,6 +2989,7 @@
 	    PORT_Free(ss);
 	    ss = NULL;
 	}
+	ss->protocolVariant = protocolVariant;
     }
     return ss;
 }