Add DTLS support to NSS, contributed by Eric Rescorla.

net/third_party/nss/ssl/sslsock.c

 /*
  * vtables (and methods that call through them) for the 4 types of 
  * SSLSockets supported.  Only one type is still supported.
  * Various other functions.
  *
  * ***** BEGIN LICENSE BLOCK *****
  * Version: MPL 1.1/GPL 2.0/LGPL 2.1
  *
  * The contents of this file are subject to the Mozilla Public License Version
  * 1.1 (the "License"); you may not use this file except in compliance with
@@ skipping 176 matching lines @@
     PR_FALSE,   /* enableFalseStart   */
     PR_TRUE,    /* cbcRandomIV        */
     PR_FALSE,   /* enableOCSPStapling */
     PR_FALSE,   /* enableOBCerts */
     PR_FALSE,   /* encryptClientCerts */
 };
 
 /*
  * default range of enabled SSL/TLS protocols
  */
-static SSLVersionRange versions_defaults = {
+static SSLVersionRange versions_defaults_stream = {
     SSL_LIBRARY_VERSION_3_0,
     SSL_LIBRARY_VERSION_TLS_1_0
 };
 
+static SSLVersionRange versions_defaults_datagram = {
+    SSL_LIBRARY_VERSION_TLS_1_1,
+    SSL_LIBRARY_VERSION_TLS_1_1
+};
+
+#define VERSIONS_DEFAULTS(variant) \
+    (variant == ssl_variant_stream ? &versions_defaults_stream : \
+                                     &versions_defaults_datagram)
+
 sslSessionIDLookupFunc  ssl_sid_lookup;
 sslSessionIDCacheFunc   ssl_sid_cache;
 sslSessionIDUncacheFunc ssl_sid_uncache;
 
 static PRBool ssl_inited = PR_FALSE;
 static PRDescIdentity ssl_layer_id;
 
 PRBool                  locksEverDisabled;      /* implicitly PR_FALSE */
 PRBool                  ssl_force_locks;        /* implicitly PR_FALSE */
 int                     ssl_lock_readers        = 1;    /* default true. */
 char                    ssl_debug;
 char                    ssl_trace;
 FILE *                  ssl_trace_iob;
 FILE *                  ssl_keylog_iob;
 char lockStatus[] = "Locks are ENABLED.  ";
 #define LOCKSTATUS_OFFSET 10 /* offset of ENABLED */
 
 /* forward declarations. */
-static sslSocket *ssl_NewSocket(PRBool makeLocks);
+static sslSocket *ssl_NewSocket(PRBool makeLocks, SSLProtocolVariant variant);
 static SECStatus  ssl_MakeLocks(sslSocket *ss);
 static void       ssl_SetDefaultsFromEnvironment(void);
 static PRStatus   ssl_PushIOLayer(sslSocket *ns, PRFileDesc *stack, 
                                   PRDescIdentity id);
 
 /************************************************************************/
 
 /*
 ** Lookup a socket structure from a file descriptor.
 ** Only functions called through the PRIOMethods table should use this.
@@ skipping 43 matching lines @@
     ss->fd = layer;
     return ss;
 }
 
 sslSocket *
 ssl_DupSocket(sslSocket *os)
 {
     sslSocket *ss;
     SECStatus rv;
 
-    ss = ssl_NewSocket((PRBool)(!os->opt.noLocks));
+    /* Not implemented for datagram */
+    if (IS_DTLS(os)) {
+        PORT_SetError(PR_NOT_IMPLEMENTED_ERROR);
+        return NULL;
+    }
+
+    ss = ssl_NewSocket((PRBool)(!os->opt.noLocks), os->protocolVariant);
     if (ss) {
         ss->opt                = os->opt;
         ss->opt.useSocks       = PR_FALSE;
         ss->vrange             = os->vrange;
 
         ss->peerID             = !os->peerID ? NULL : PORT_Strdup(os->peerID);
         ss->url                = !os->url    ? NULL : PORT_Strdup(os->url);
 
         ss->ops      = os->ops;
         ss->rTimeout = os->rTimeout;
@@ skipping 396 matching lines @@
       case SSL_HANDSHAKE_AS_SERVER:
         if ( ss->opt.handshakeAsClient && on ) {
             PORT_SetError(SEC_ERROR_INVALID_ARGS);
             rv = SECFailure;
             break;
         }
         ss->opt.handshakeAsServer = on;
         break;
 
       case SSL_ENABLE_TLS:
+        if (IS_DTLS(ss)) {
+            if (on) {
+                PORT_SetError(SEC_ERROR_INVALID_ARGS);
+                rv = SECFailure; /* not allowed */
+            }
+            break;
+        }
         ssl_EnableTLS(&ss->vrange, on);
         ss->preferredCipher     = NULL;
         if (ss->cipherSpecs) {
             PORT_Free(ss->cipherSpecs);
             ss->cipherSpecs     = NULL;
             ss->sizeCipherSpecs = 0;
         }
         break;
 
       case SSL_ENABLE_SSL3:
+        if (IS_DTLS(ss)) {
+            if (on) {
+                PORT_SetError(SEC_ERROR_INVALID_ARGS);
+                rv = SECFailure; /* not allowed */
+            }
+            break;
+        }
         ssl_EnableSSL3(&ss->vrange, on);
         ss->preferredCipher     = NULL;
         if (ss->cipherSpecs) {
             PORT_Free(ss->cipherSpecs);
             ss->cipherSpecs     = NULL;
             ss->sizeCipherSpecs = 0;
         }
         break;
 
       case SSL_ENABLE_SSL2:
+        if (IS_DTLS(ss)) {
+            if (on) {
+                PORT_SetError(SEC_ERROR_INVALID_ARGS);
+                rv = SECFailure; /* not allowed */
+            }
+            break;
+        }
         ss->opt.enableSSL2       = on;
         if (on) {
             ss->opt.v2CompatibleHello = on;
         }
         ss->preferredCipher     = NULL;
         if (ss->cipherSpecs) {
             PORT_Free(ss->cipherSpecs);
             ss->cipherSpecs     = NULL;
             ss->sizeCipherSpecs = 0;
         }
         break;
 
       case SSL_NO_CACHE:
         ss->opt.noCache = on;
         break;
 
       case SSL_ENABLE_FDX:
         if (on && ss->opt.noLocks) {
             PORT_SetError(SEC_ERROR_INVALID_ARGS);
             rv = SECFailure;
         }
         ss->opt.fdx = on;
         break;
 
       case SSL_V2_COMPATIBLE_HELLO:
+        if (IS_DTLS(ss)) {
+            if (on) {
+                PORT_SetError(SEC_ERROR_INVALID_ARGS);
+                rv = SECFailure; /* not allowed */
+            }
+            break;
+        }
         ss->opt.v2CompatibleHello = on;
         if (!on) {
             ss->opt.enableSSL2    = on;
         }
         break;
 
       case SSL_ROLLBACK_DETECTION:  
         ss->opt.detectRollBack = on;
         break;
 
@@ skipping 175 matching lines @@
     ssl_SetDefaultsFromEnvironment();
 
     switch (which) {
     case SSL_SOCKS:               on = PR_FALSE;                        break;
     case SSL_SECURITY:            on = ssl_defaults.useSecurity;        break;
     case SSL_REQUEST_CERTIFICATE: on = ssl_defaults.requestCertificate; break;
     case SSL_REQUIRE_CERTIFICATE: on = ssl_defaults.requireCertificate; break;
     case SSL_HANDSHAKE_AS_CLIENT: on = ssl_defaults.handshakeAsClient;  break;
     case SSL_HANDSHAKE_AS_SERVER: on = ssl_defaults.handshakeAsServer;  break;
     case SSL_ENABLE_TLS:
-        on = versions_defaults.max >= SSL_LIBRARY_VERSION_TLS_1_0;
+        on = versions_defaults_stream.max >= SSL_LIBRARY_VERSION_TLS_1_0;
         break;
     case SSL_ENABLE_SSL3:
-        on = versions_defaults.min == SSL_LIBRARY_VERSION_3_0;
+        on = versions_defaults_stream.min == SSL_LIBRARY_VERSION_3_0;
         break;
     case SSL_ENABLE_SSL2:         on = ssl_defaults.enableSSL2;         break;
     case SSL_NO_CACHE:            on = ssl_defaults.noCache;            break;
     case SSL_ENABLE_FDX:          on = ssl_defaults.fdx;                break;
     case SSL_V2_COMPATIBLE_HELLO: on = ssl_defaults.v2CompatibleHello;  break;
     case SSL_ROLLBACK_DETECTION:  on = ssl_defaults.detectRollBack;     break;
     case SSL_NO_STEP_DOWN:        on = ssl_defaults.noStepDown;         break;
     case SSL_BYPASS_PKCS11:       on = ssl_defaults.bypassPKCS11;       break;
     case SSL_NO_LOCKS:            on = ssl_defaults.noLocks;            break;
     case SSL_ENABLE_SESSION_TICKETS:
@@ skipping 72 matching lines @@
 
       case SSL_HANDSHAKE_AS_SERVER:
         if ( ssl_defaults.handshakeAsClient && on ) {
             PORT_SetError(SEC_ERROR_INVALID_ARGS);
             return SECFailure;
         }
         ssl_defaults.handshakeAsServer = on;
         break;
 
       case SSL_ENABLE_TLS:
-        ssl_EnableTLS(&versions_defaults, on);
+        ssl_EnableTLS(&versions_defaults_stream, on);
         break;
 
       case SSL_ENABLE_SSL3:
-        ssl_EnableSSL3(&versions_defaults, on);
+        ssl_EnableSSL3(&versions_defaults_stream, on);
         break;
 
       case SSL_ENABLE_SSL2:
         ssl_defaults.enableSSL2 = on;
         if (on) {
             ssl_defaults.v2CompatibleHello = on;
         }
         break;
 
       case SSL_NO_CACHE:
@@ skipping 301 matching lines @@
 
 SECStatus
 NSS_SetFrancePolicy(void)
 {
     return NSS_SetDomesticPolicy();
 }
 
 
 
 /* LOCKS ??? XXX */
-PRFileDesc *
-SSL_ImportFD(PRFileDesc *model, PRFileDesc *fd)
+static PRFileDesc *
+ssl_ImportFD(PRFileDesc *model, PRFileDesc *fd, SSLProtocolVariant variant)
 {
     sslSocket * ns = NULL;
     PRStatus    rv;
     PRNetAddr   addr;
     SECStatus   status = ssl_Init();
 
     if (status != SECSuccess) {
         return NULL;
     }
 
     if (model == NULL) {
         /* Just create a default socket if we're given NULL for the model */
-        ns = ssl_NewSocket((PRBool)(!ssl_defaults.noLocks));
+        ns = ssl_NewSocket((PRBool)(!ssl_defaults.noLocks), variant);
     } else {
         sslSocket * ss = ssl_FindSocket(model);
-        if (ss == NULL) {
+        if (ss == NULL || ss->protocolVariant != variant) {
             SSL_DBG(("%d: SSL[%d]: bad model socket in ssl_ImportFD", 
                       SSL_GETPID(), model));
             return NULL;
         }
         ns = ssl_DupSocket(ss);
     }
     if (ns == NULL)
         return NULL;
 
     rv = ssl_PushIOLayer(ns, fd, PR_TOP_IO_LAYER);
     if (rv != PR_SUCCESS) {
         ssl_FreeSocket(ns);
         SET_ERROR_CODE
         return NULL;
     }
 #ifdef _WIN32
     PR_Sleep(PR_INTERVAL_NO_WAIT);     /* workaround NT winsock connect bug. */
 #endif
     ns = ssl_FindSocket(fd);
     PORT_Assert(ns);
     if (ns)
         ns->TCPconnected = (PR_SUCCESS == ssl_DefGetpeername(ns, &addr));
     return fd;
 }
 
+PRFileDesc *
+SSL_ImportFD(PRFileDesc *model, PRFileDesc *fd)
+{
+    return ssl_ImportFD(model, fd, ssl_variant_stream);
+}
+
+PRFileDesc *
+DTLS_ImportFD(PRFileDesc *model, PRFileDesc *fd)
+{
+    return ssl_ImportFD(model, fd, ssl_variant_datagram);
+}
+
 SECStatus
 SSL_SetNextProtoCallback(PRFileDesc *fd, SSLNextProtoCallback callback,
                          void *arg)
 {
     sslSocket *ss = ssl_FindSocket(fd);
 
     if (!ss) {
         SSL_DBG(("%d: SSL[%d]: bad socket in SSL_SetNextProtoCallback", SSL_GETPID(),
                  fd));
         return SECFailure;
@@ skipping 244 matching lines @@
     return fd;
 loser:
     return NULL;
 #endif
 }
 
 PRBool
 ssl3_VersionIsSupported(SSLProtocolVariant protocolVariant,
                         SSL3ProtocolVersion version)
 {
-    return protocolVariant == ssl_variant_stream &&
-           version >= SSL_LIBRARY_VERSION_3_0 &&
-           version <= SSL_LIBRARY_VERSION_MAX_SUPPORTED;
+    switch (protocolVariant) {
+    case ssl_variant_stream:
+        return (version >= SSL_LIBRARY_VERSION_3_0 &&
+                version <= SSL_LIBRARY_VERSION_MAX_SUPPORTED);
+    case ssl_variant_datagram:
+        return (version >= SSL_LIBRARY_VERSION_TLS_1_1 &&
+                version <= SSL_LIBRARY_VERSION_MAX_SUPPORTED);
+    default:
+        /* Can't get here */
+        PORT_Assert(PR_FALSE);
+        return PR_FALSE;
+    }
 }
 
 /* Returns PR_TRUE if the given version range is valid and
 ** fully supported; otherwise, returns PR_FALSE.
 */
 static PRBool
 ssl3_VersionRangeIsValid(SSLProtocolVariant protocolVariant,
                          const SSLVersionRange *vrange)
 {
     return vrange &&
            vrange->min <= vrange->max &&
            ssl3_VersionIsSupported(protocolVariant, vrange->min) &&
            ssl3_VersionIsSupported(protocolVariant, vrange->max);
 }
 
 SECStatus
 SSL_VersionRangeGetSupported(SSLProtocolVariant protocolVariant,
                              SSLVersionRange *vrange)
 {
-    if (protocolVariant != ssl_variant_stream || !vrange) {
+    if (!vrange) {
         PORT_SetError(SEC_ERROR_INVALID_ARGS);
         return SECFailure;
     }
 
-    vrange->min = SSL_LIBRARY_VERSION_3_0;
-    vrange->max = SSL_LIBRARY_VERSION_MAX_SUPPORTED;
+    switch (protocolVariant) {
+    case ssl_variant_stream:
+        vrange->min = SSL_LIBRARY_VERSION_3_0;
+        vrange->max = SSL_LIBRARY_VERSION_MAX_SUPPORTED;
+        break;
+    case ssl_variant_datagram:
+        vrange->min = SSL_LIBRARY_VERSION_TLS_1_1;
+        vrange->max = SSL_LIBRARY_VERSION_MAX_SUPPORTED;
+        break;
+    default:
+        PORT_SetError(SEC_ERROR_INVALID_ARGS);
+        return SECFailure;
+    }
 
     return SECSuccess;
 }
 
 SECStatus
 SSL_VersionRangeGetDefault(SSLProtocolVariant protocolVariant,
                            SSLVersionRange *vrange)
 {
-    if (protocolVariant != ssl_variant_stream || !vrange) {
+    if ((protocolVariant != ssl_variant_stream &&
+         protocolVariant != ssl_variant_datagram) || !vrange) {
         PORT_SetError(SEC_ERROR_INVALID_ARGS);
         return SECFailure;
     }
 
-    *vrange = versions_defaults;
+    *vrange = *VERSIONS_DEFAULTS(protocolVariant);
 
     return SECSuccess;
 }
 
 SECStatus
 SSL_VersionRangeSetDefault(SSLProtocolVariant protocolVariant,
                            const SSLVersionRange *vrange)
 {
     if (!ssl3_VersionRangeIsValid(protocolVariant, vrange)) {
         PORT_SetError(SSL_ERROR_INVALID_VERSION_RANGE);
         return SECFailure;
     }
 
-    versions_defaults = *vrange;
+    *VERSIONS_DEFAULTS(protocolVariant) = *vrange;
 
     return SECSuccess;
 }
 
 SECStatus
 SSL_VersionRangeGet(PRFileDesc *fd, SSLVersionRange *vrange)
 {
     sslSocket *ss = ssl_FindSocket(fd);
 
     if (!ss) {
@@ skipping 1086 matching lines @@
             SSL_TRACE(("SSL: cbcRandomIV set to 0"));
         }
     }
 #endif /* NSS_HAVE_GETENV */
 }
 
 /*
 ** Create a newsocket structure for a file descriptor.
 */
 static sslSocket *
-ssl_NewSocket(PRBool makeLocks)
+ssl_NewSocket(PRBool makeLocks, SSLProtocolVariant protocolVariant)
 {
     sslSocket *ss;
 
     ssl_SetDefaultsFromEnvironment();
 
     if (ssl_force_locks)
         makeLocks = PR_TRUE;
 
     /* Make a new socket and get it ready */
     ss = (sslSocket*) PORT_ZAlloc(sizeof(sslSocket));
     if (ss) {
         /* This should be of type SSLKEAType, but CC on IRIX
          * complains during the for loop.
          */
         int i;
         SECStatus status;
  
         ss->opt                = ssl_defaults;
         ss->opt.useSocks       = PR_FALSE;
         ss->opt.noLocks        = !makeLocks;
-        ss->vrange             = versions_defaults;
+        ss->vrange             = *VERSIONS_DEFAULTS(protocolVariant);
 
         ss->peerID             = NULL;
         ss->rTimeout           = PR_INTERVAL_NO_TIMEOUT;
         ss->wTimeout           = PR_INTERVAL_NO_TIMEOUT;
         ss->cTimeout           = PR_INTERVAL_NO_TIMEOUT;
         ss->cipherSpecs        = NULL;
         ss->sizeCipherSpecs    = 0;  /* produced lazily */
         ss->preferredCipher    = NULL;
         ss->url                = NULL;
 
@@ skipping 35 matching lines @@
         if (status != SECSuccess) 
             goto loser;
         status = ssl_InitGather(&ss->gs);
         if (status != SECSuccess) {
 loser:
             ssl_DestroySocketContents(ss);
             ssl_DestroyLocks(ss);
             PORT_Free(ss);
             ss = NULL;
         }
+        ss->protocolVariant = protocolVariant;
     }
     return ss;
 }