Add DTLS support to NSS, contributed by Eric Rescorla.

net/third_party/nss/ssl/ssl3gthr.c

Index: net/third_party/nss/ssl/ssl3gthr.c
===================================================================
--- net/third_party/nss/ssl/ssl3gthr.c	(revision 127709)
+++ net/third_party/nss/ssl/ssl3gthr.c	(working copy)
@@ -167,6 +167,119 @@
     return rv;
 }
 
+/*
+ * Read in an entire DTLS record.
+ *
+ * Blocks here for blocking sockets, otherwise returns -1 with
+ * 	PR_WOULD_BLOCK_ERROR when socket would block.
+ *
+ * This is simpler than SSL because we are reading on a datagram socket
+ * and datagrams must contain >=1 complete records.
+ *
+ * returns  1 if received a complete DTLS record.
+ * returns  0 if recv returns EOF
+ * returns -1 if recv returns <0
+ *	(The error value may have already been set to PR_WOULD_BLOCK_ERROR)
+ *
+ * Caller must hold the recv buf lock.
+ *
+ * This loop returns when either (a) an error or EOF occurs,

[Ryan Sleevi, 2012/03/22 22:26:37]

newline before (a) ?

+ *	(b) PR_WOULD_BLOCK_ERROR,
+ * 	(c) data (entire DTLS record) has been received.
+ */
+static int
+dtls_GatherData(sslSocket *ss, sslGather *gs, int flags)
+{
+    int            nb;
+    int            err;
+    int            rv		= 1;
+
+    SSL_TRC(30, ("dtls_GatherData"));
+
+    PORT_Assert( ss->opt.noLocks || ssl_HaveRecvBufLock(ss) );
+ 
+    gs->state = GS_HEADER;
+    gs->offset = 0;
+
+    if (gs->dtlsPacketOffset == gs->dtlsPacket.len) {  /* No data left */
+        gs->dtlsPacketOffset = 0;
+        gs->dtlsPacket.len = 0;
+
+        /* Resize to the maximum possible size so we can fit a full datagram */
+        if (gs->dtlsPacket.space < MAX_FRAGMENT_LENGTH + 2048 + 5) {

[Ryan Sleevi, 2012/03/22 22:26:37]

What's the magic 2048 + 5? Should that be documented in the comment on line 208?

[wtc, 2012/03/23 00:21:18]

The magic constant is copied from ssl3_GatherData on lines
134-137.  I will also copy the comment here and note that
5 should be changed to 13 (the size of the record header).

[ekr, 2012/03/23 12:46:41]

I just noticed that as well. We could change that in the code if you wanted
(Note that in any sane network this is really way too big anyway), and the
effect is simply to reduce the effective MTU if it somehow is magically as big
as 16k.

+            err = sslBuffer_Grow(&gs->dtlsPacket,
+				 MAX_FRAGMENT_LENGTH + 2048 + 5);
+            if (err) {	/* realloc has set error code to no mem. */
+                return err;
+            }
+        }
+
+        /* recv() needs to read a full datagram at a time */
+        nb = ssl_DefRecv(ss, gs->dtlsPacket.buf, gs->dtlsPacket.space, flags);
+
+        if (nb > 0) {
+            PRINT_BUF(60, (ss, "raw gather data:", gs->dtlsPacket.buf, nb));
+        } else if (nb == 0) {
+            /* EOF */
+            SSL_TRC(30, ("%d: SSL3[%d]: EOF", SSL_GETPID(), ss->fd));
+            rv = 0;
+            return rv;
+        } else /* if (nb < 0) */ {
+            SSL_DBG(("%d: SSL3[%d]: recv error %d", SSL_GETPID(), ss->fd,
+                     PR_GetError()));
+            rv = SECFailure;
+            return rv;
+        }
+
+        gs->dtlsPacket.len = nb;
+    }
+
+    /* At this point we should have >=1 complete records lined up in
+     * dtlsPacket. Read off the header.
+     */
+    if ((gs->dtlsPacket.len - gs->dtlsPacketOffset) < 13) {

[Ryan Sleevi, 2012/03/22 22:26:37]

nit:

Use DTLS_RECORD_HEADER_LENGTH instead of 13 throughout here, for cleanliness.

[wtc, 2012/03/23 00:21:18]

This will make line 253 hard to read
    gs->remainder = (gs->hdr[11] << 8) | gs->hdr[12];
because 11 and 12 will need to become
DTLS_RECORD_HEADER_LENGTH - 2 and
DTLS_RECORD_HEADER_LENGTH - 1.  Not sure if that's the
right trade-off.

+        SSL_DBG(("%d: SSL3[%d]: rest of DTLS packet "
+		 "too short to contain header", SSL_GETPID(), ss->fd));
+        PR_SetError(PR_WOULD_BLOCK_ERROR, 0);
+        gs->dtlsPacketOffset = 0;
+        gs->dtlsPacket.len = 0;
+        rv = SECFailure;
+        return rv;
+    }
+    memcpy(gs->hdr, gs->dtlsPacket.buf + gs->dtlsPacketOffset, 13);
+    gs->dtlsPacketOffset += 13;
+
+    /* Have received SSL3 record header in gs->hdr. */
+    gs->remainder = (gs->hdr[11] << 8) | gs->hdr[12];
+
+    if ((gs->dtlsPacket.len - gs->dtlsPacketOffset) < gs->remainder) {
+        SSL_DBG(("%d: SSL3[%d]: rest of DTLS packet too short "
+		 "to contain rest of body", SSL_GETPID(), ss->fd));
+        PR_SetError(PR_WOULD_BLOCK_ERROR, 0);
+        gs->dtlsPacketOffset = 0;
+        gs->dtlsPacket.len = 0;
+        rv = SECFailure;
+        return rv;
+    }
+
+    /* OK, we have at least one complete packet, copy into inbuf */
+    if (gs->remainder > gs->inbuf.space) {
+	err = sslBuffer_Grow(&gs->inbuf, gs->remainder);
+	if (err) {	/* realloc has set error code to no mem. */
+	    return err;
+	}
+    }
+
+    memcpy(gs->inbuf.buf, gs->dtlsPacket.buf + gs->dtlsPacketOffset,
+	   gs->remainder);
+    gs->inbuf.len = gs->remainder;
+    gs->offset = gs->remainder;
+    gs->dtlsPacketOffset += gs->remainder;
+    gs->state = GS_INIT;
+
+    return 1;
+}
+
 /* Gather in a record and when complete, Handle that record.
  * Repeat this until the handshake is complete, 
  * or until application data is available.
@@ -190,6 +303,8 @@
     int            rv;
     PRBool         canFalseStart = PR_FALSE;
 
+    SSL_TRC(30, ("ssl3_GatherCompleteHandshake"));
+
     PORT_Assert( ss->opt.noLocks || ssl_HaveRecvBufLock(ss) );
     do {
 	/* Without this, we may end up wrongly reporting
@@ -224,7 +339,24 @@
 	    rv = ssl3_HandleRecord(ss, NULL, &ss->gs.buf);
 	} else {
 	    /* bring in the next sslv3 record. */
-	    rv = ssl3_GatherData(ss, &ss->gs, flags);
+	    if (IS_DTLS(ss)) {

[Ryan Sleevi, 2012/03/22 22:26:37]

Your normal ordering has been !IS_DTLS then handle DTLS in the else when there
is an SSL-or-DTLS block, and IS_DTLS when there's just a DTLS block.

Swap the order?

[wtc, 2012/03/23 00:21:18]

I actually prefer
    if (IS_DTLS(ss)) {
        ....
    } else {
        ....
    }

What's the general recommendation when the two blocks
are lopsided?  Do the simple block first?

[Ryan Sleevi, 2012/03/23 00:38:28]

I had assumed so - since often the 'simple block' is the error handling.

Here, my thought was more about 'common' block - !IS_DTLS() is how I think about
the code when reading (eg: I want to see the TLS implementation first). But
that's just my implementation bias, being more familiar with TLS rather than
DTLS :)

+		rv = dtls_GatherData(ss, &ss->gs, flags);
+		
+		/* If we got a would block error, that means that no data was
+		 * available, so we check the timer to see if it's time to
+		 * retransmit */
+		if (rv == SECFailure &&
+		    (PORT_GetError() == PR_WOULD_BLOCK_ERROR)) {
+		    ssl_GetSSL3HandshakeLock(ss);

[Ryan Sleevi, 2012/03/22 22:26:37]

XXX - Check the layering of locks (line 308, line 350) to ensure there is no
deadlock opportunity.

I thoguht Handshake came before Recv.

[wtc, 2012/03/23 00:21:18]

The lock order is recvbuf -> ssl3Handshake, so this is fine.

+		    dtls_CheckTimer(ss);
+		    ssl_ReleaseSSL3HandshakeLock(ss);
+		    /* Restore the error in case something succeeded */
+		    PORT_SetError(PR_WOULD_BLOCK_ERROR);
+		}
+	    } else {
+		rv = ssl3_GatherData(ss, &ss->gs, flags);
+	    }
+
 	    if (rv <= 0) {
 		return rv;
 	    }
@@ -236,6 +368,21 @@
 	     */
 	    cText.type    = (SSL3ContentType)ss->gs.hdr[0];
 	    cText.version = (ss->gs.hdr[1] << 8) | ss->gs.hdr[2];
+
+	    /* As above, we need to test datagram mode */
+	    if (IS_DTLS(ss)) {
+		int i;
+
+		cText.version = dtls_DTLSVersionToTLSVersion(cText.version);
+		/* DTLS sequence number */
+		cText.seq_num.high = 0; cText.seq_num.low = 0;
+		for (i = 0; i < 4; i++) {
+		    cText.seq_num.high <<= 8; cText.seq_num.low <<= 8;
+		    cText.seq_num.high |= ss->gs.hdr[3 + i];
+		    cText.seq_num.low |= ss->gs.hdr[7 + i];
+		}
+	    }
+
 	    cText.buf     = &ss->gs.inbuf;
 	    rv = ssl3_HandleRecord(ss, &cText, &ss->gs.buf);
 	}