Add DTLS support to NSS, contributed by Eric Rescorla.

net/third_party/nss/ssl/ssl3gthr.c

 /*
  * Gather (Read) entire SSL3 records from socket into buffer.  
  *
  * ***** BEGIN LICENSE BLOCK *****
  * Version: MPL 1.1/GPL 2.0/LGPL 2.1
  *
  * The contents of this file are subject to the Mozilla Public License Version
  * 1.1 (the "License"); you may not use this file except in compliance with
  * the License. You may obtain a copy of the License at
  * http://www.mozilla.org/MPL/
@@ skipping 149 matching lines @@
             ** SSL3 record has been completely received.
             */
             gs->state = GS_INIT;
             return 1;
         }
     }
 
     return rv;
 }
 
+/*
+ * Read in an entire DTLS record.
+ *
+ * Blocks here for blocking sockets, otherwise returns -1 with
+ *      PR_WOULD_BLOCK_ERROR when socket would block.
+ *
+ * This is simpler than SSL because we are reading on a datagram socket
+ * and datagrams must contain >=1 complete records.
+ *
+ * returns  1 if received a complete DTLS record.
+ * returns  0 if recv returns EOF
+ * returns -1 if recv returns <0
+ *      (The error value may have already been set to PR_WOULD_BLOCK_ERROR)
+ *
+ * Caller must hold the recv buf lock.
+ *
+ * This loop returns when either (a) an error or EOF occurs,

[Ryan Sleevi, 2012/03/22 22:26:37]

newline before (a) ?

+ *      (b) PR_WOULD_BLOCK_ERROR,
+ *      (c) data (entire DTLS record) has been received.
+ */
+static int
+dtls_GatherData(sslSocket *ss, sslGather *gs, int flags)
+{
+    int            nb;
+    int            err;
+    int            rv           = 1;
+
+    SSL_TRC(30, ("dtls_GatherData"));
+
+    PORT_Assert( ss->opt.noLocks || ssl_HaveRecvBufLock(ss) );
+ 
+    gs->state = GS_HEADER;
+    gs->offset = 0;
+
+    if (gs->dtlsPacketOffset == gs->dtlsPacket.len) {  /* No data left */
+        gs->dtlsPacketOffset = 0;
+        gs->dtlsPacket.len = 0;
+
+        /* Resize to the maximum possible size so we can fit a full datagram */
+        if (gs->dtlsPacket.space < MAX_FRAGMENT_LENGTH + 2048 + 5) {

[Ryan Sleevi, 2012/03/22 22:26:37]

What's the magic 2048 + 5? Should that be documented in the comment on line 208?

[wtc, 2012/03/23 00:21:18]

The magic constant is copied from ssl3_GatherData on lines
134-137.  I will also copy the comment here and note that
5 should be changed to 13 (the size of the record header).

[ekr, 2012/03/23 12:46:41]

I just noticed that as well. We could change that in the code if you wanted
(Note that in any sane network this is really way too big anyway), and the
effect is simply to reduce the effective MTU if it somehow is magically as big
as 16k.

+            err = sslBuffer_Grow(&gs->dtlsPacket,
+                                 MAX_FRAGMENT_LENGTH + 2048 + 5);
+            if (err) {  /* realloc has set error code to no mem. */
+                return err;
+            }
+        }
+
+        /* recv() needs to read a full datagram at a time */
+        nb = ssl_DefRecv(ss, gs->dtlsPacket.buf, gs->dtlsPacket.space, flags);
+
+        if (nb > 0) {
+            PRINT_BUF(60, (ss, "raw gather data:", gs->dtlsPacket.buf, nb));
+        } else if (nb == 0) {
+            /* EOF */
+            SSL_TRC(30, ("%d: SSL3[%d]: EOF", SSL_GETPID(), ss->fd));
+            rv = 0;
+            return rv;
+        } else /* if (nb < 0) */ {
+            SSL_DBG(("%d: SSL3[%d]: recv error %d", SSL_GETPID(), ss->fd,
+                     PR_GetError()));
+            rv = SECFailure;
+            return rv;
+        }
+
+        gs->dtlsPacket.len = nb;
+    }
+
+    /* At this point we should have >=1 complete records lined up in
+     * dtlsPacket. Read off the header.
+     */
+    if ((gs->dtlsPacket.len - gs->dtlsPacketOffset) < 13) {

[Ryan Sleevi, 2012/03/22 22:26:37]

nit:

Use DTLS_RECORD_HEADER_LENGTH instead of 13 throughout here, for cleanliness.

[wtc, 2012/03/23 00:21:18]

This will make line 253 hard to read
    gs->remainder = (gs->hdr[11] << 8) | gs->hdr[12];
because 11 and 12 will need to become
DTLS_RECORD_HEADER_LENGTH - 2 and
DTLS_RECORD_HEADER_LENGTH - 1.  Not sure if that's the
right trade-off.

+        SSL_DBG(("%d: SSL3[%d]: rest of DTLS packet "
+                 "too short to contain header", SSL_GETPID(), ss->fd));
+        PR_SetError(PR_WOULD_BLOCK_ERROR, 0);
+        gs->dtlsPacketOffset = 0;
+        gs->dtlsPacket.len = 0;
+        rv = SECFailure;
+        return rv;
+    }
+    memcpy(gs->hdr, gs->dtlsPacket.buf + gs->dtlsPacketOffset, 13);
+    gs->dtlsPacketOffset += 13;
+
+    /* Have received SSL3 record header in gs->hdr. */
+    gs->remainder = (gs->hdr[11] << 8) | gs->hdr[12];
+
+    if ((gs->dtlsPacket.len - gs->dtlsPacketOffset) < gs->remainder) {
+        SSL_DBG(("%d: SSL3[%d]: rest of DTLS packet too short "
+                 "to contain rest of body", SSL_GETPID(), ss->fd));
+        PR_SetError(PR_WOULD_BLOCK_ERROR, 0);
+        gs->dtlsPacketOffset = 0;
+        gs->dtlsPacket.len = 0;
+        rv = SECFailure;
+        return rv;
+    }
+
+    /* OK, we have at least one complete packet, copy into inbuf */
+    if (gs->remainder > gs->inbuf.space) {
+        err = sslBuffer_Grow(&gs->inbuf, gs->remainder);
+        if (err) {      /* realloc has set error code to no mem. */
+            return err;
+        }
+    }
+
+    memcpy(gs->inbuf.buf, gs->dtlsPacket.buf + gs->dtlsPacketOffset,
+           gs->remainder);
+    gs->inbuf.len = gs->remainder;
+    gs->offset = gs->remainder;
+    gs->dtlsPacketOffset += gs->remainder;
+    gs->state = GS_INIT;
+
+    return 1;
+}
+
 /* Gather in a record and when complete, Handle that record.
  * Repeat this until the handshake is complete, 
  * or until application data is available.
  *
  * Returns  1 when the handshake is completed without error, or 
  *                 application data is available.
  * Returns  0 if ssl3_GatherData hits EOF.
  * Returns -1 on read error, or PR_WOULD_BLOCK_ERROR, or handleRecord error.
  * Returns -2 on SECWouldBlock return from ssl3_HandleRecord.
  *
  * Called from ssl_GatherRecord1stHandshake       in sslcon.c, 
  *    and from SSL_ForceHandshake in sslsecur.c
  *    and from ssl3_GatherAppDataRecord below (<- DoRecv in sslsecur.c).
  *
  * Caller must hold the recv buf lock.
  */
 int
 ssl3_GatherCompleteHandshake(sslSocket *ss, int flags)
 {
     SSL3Ciphertext cText;
     int            rv;
     PRBool         canFalseStart = PR_FALSE;
 
+    SSL_TRC(30, ("ssl3_GatherCompleteHandshake"));
+
     PORT_Assert( ss->opt.noLocks || ssl_HaveRecvBufLock(ss) );
     do {
         /* Without this, we may end up wrongly reporting
          * SSL_ERROR_RX_UNEXPECTED_* errors if we receive any records from the
          * peer while we are waiting to be restarted.
          */
         ssl_GetSSL3HandshakeLock(ss);
         rv = ss->ssl3.hs.restartTarget == NULL ? SECSuccess : SECFailure;
         ssl_ReleaseSSL3HandshakeLock(ss);
         if (rv != SECSuccess) {
@@ skipping 14 matching lines @@
 
         if (ss->ssl3.hs.msgState.buf != NULL) {
             /* ssl3_HandleHandshake previously returned SECWouldBlock and the
              * as-yet-unprocessed plaintext of that previous handshake record.
              * We need to process it now before we overwrite it with the next
              * handshake record.
              */
             rv = ssl3_HandleRecord(ss, NULL, &ss->gs.buf);
         } else {
             /* bring in the next sslv3 record. */
-            rv = ssl3_GatherData(ss, &ss->gs, flags);
+            if (IS_DTLS(ss)) {

[Ryan Sleevi, 2012/03/22 22:26:37]

Your normal ordering has been !IS_DTLS then handle DTLS in the else when there
is an SSL-or-DTLS block, and IS_DTLS when there's just a DTLS block.

Swap the order?

[wtc, 2012/03/23 00:21:18]

I actually prefer
    if (IS_DTLS(ss)) {
        ....
    } else {
        ....
    }

What's the general recommendation when the two blocks
are lopsided?  Do the simple block first?

[Ryan Sleevi, 2012/03/23 00:38:28]

I had assumed so - since often the 'simple block' is the error handling.

Here, my thought was more about 'common' block - !IS_DTLS() is how I think about
the code when reading (eg: I want to see the TLS implementation first). But
that's just my implementation bias, being more familiar with TLS rather than
DTLS :)

+                rv = dtls_GatherData(ss, &ss->gs, flags);
+                
+                /* If we got a would block error, that means that no data was
+                 * available, so we check the timer to see if it's time to
+                 * retransmit */
+                if (rv == SECFailure &&
+                    (PORT_GetError() == PR_WOULD_BLOCK_ERROR)) {
+                    ssl_GetSSL3HandshakeLock(ss);

[Ryan Sleevi, 2012/03/22 22:26:37]

XXX - Check the layering of locks (line 308, line 350) to ensure there is no
deadlock opportunity.

I thoguht Handshake came before Recv.

[wtc, 2012/03/23 00:21:18]

The lock order is recvbuf -> ssl3Handshake, so this is fine.

+                    dtls_CheckTimer(ss);
+                    ssl_ReleaseSSL3HandshakeLock(ss);
+                    /* Restore the error in case something succeeded */
+                    PORT_SetError(PR_WOULD_BLOCK_ERROR);
+                }
+            } else {
+                rv = ssl3_GatherData(ss, &ss->gs, flags);
+            }
+
             if (rv <= 0) {
                 return rv;
             }
 
             /* decipher it, and handle it if it's a handshake.
              * If it's application data, ss->gs.buf will not be empty upon return.
              * If it's a change cipher spec, alert, or handshake message,
              * ss->gs.buf.len will be 0 when ssl3_HandleRecord returns SECSuccess.
              */
             cText.type    = (SSL3ContentType)ss->gs.hdr[0];
             cText.version = (ss->gs.hdr[1] << 8) | ss->gs.hdr[2];
+
+            /* As above, we need to test datagram mode */
+            if (IS_DTLS(ss)) {
+                int i;
+
+                cText.version = dtls_DTLSVersionToTLSVersion(cText.version);
+                /* DTLS sequence number */
+                cText.seq_num.high = 0; cText.seq_num.low = 0;
+                for (i = 0; i < 4; i++) {
+                    cText.seq_num.high <<= 8; cText.seq_num.low <<= 8;
+                    cText.seq_num.high |= ss->gs.hdr[3 + i];
+                    cText.seq_num.low |= ss->gs.hdr[7 + i];
+                }
+            }
+
             cText.buf     = &ss->gs.inbuf;
             rv = ssl3_HandleRecord(ss, &cText, &ss->gs.buf);
         }
         if (rv < 0) {
             return ss->recvdCloseNotify ? 0 : rv;
         }
 
         /* If we kicked off a false start in ssl3_HandleServerHelloDone, break
          * out of this loop early without finishing the handshake.
          */
@@ skipping 29 matching lines @@
 {
     int            rv;
 
     PORT_Assert( ss->opt.noLocks || ssl_HaveRecvBufLock(ss) );
     do {
         rv = ssl3_GatherCompleteHandshake(ss, flags);
     } while (rv > 0 && ss->gs.buf.len == 0);
 
     return rv;
 }