Add DTLS support to NSS, contributed by Eric Rescorla.

net/third_party/nss/ssl/sslimpl.h

Index: net/third_party/nss/ssl/sslimpl.h
===================================================================
--- net/third_party/nss/ssl/sslimpl.h	(revision 127709)
+++ net/third_party/nss/ssl/sslimpl.h	(working copy)
@@ -62,6 +62,7 @@
 #endif
 #include "nssrwlk.h"
 #include "prthread.h"
+#include "prclist.h"
 
 #include "sslt.h" /* for some formerly private types, now public */
 
@@ -195,6 +196,10 @@
 
 #define EXPORT_RSA_KEY_LENGTH 64	/* bytes */
 
+#define INITIAL_DTLS_TIMEOUT_MS   1000  /* Default value from RFC 4347 = 1s*/
+#define MAX_DTLS_TIMEOUT_MS      60000  /* 1 minute */
+#define DTLS_FINISHED_TIMER     120000  /* Time to wait in FINISHED state */

[wtc, 2012/03/21 01:22:07]

DTLS_FINISHED_TIMER is larger than MAX_DTLS_TIMEOUT_MS.
Will that be a problem?

[wtc, 2012/03/21 01:22:07]

Nit: it would be nice to standardize on one of "TIMEOUT" and
"TIMER".  Also, include _MS in the macro's name.

[ekr, 2012/03/21 01:36:40]

No. MAX_DTLS_TIMEOUT_MS is the biggest timer we back-off to for retransmission,
but this is a different case. Wonder if we should relable or comment these to
make clear. Honestly, MAX_DTLS_TIMEOUT_MS is probably too big...

[ekr, 2012/03/21 01:36:40]

Good point.

+
 typedef struct sslBufferStr             sslBuffer;
 typedef struct sslConnectInfoStr        sslConnectInfo;
 typedef struct sslGatherStr             sslGather;
@@ -287,6 +292,8 @@
 /* Flags interpreted by ssl send functions. */
 #define ssl_SEND_FLAG_FORCE_INTO_BUFFER	0x40000000
 #define ssl_SEND_FLAG_NO_BUFFER		0x20000000
+#define ssl_SEND_FLAG_USE_EPOCH         0x10000000 /* DTLS only */
+#define ssl_SEND_FLAG_NO_RETRANSMIT     0x08000000 /* DTLS only */
 #define ssl_SEND_FLAG_MASK		0x7f000000
 
 /*
@@ -382,6 +389,7 @@
 #define ssl_SHUTDOWN_SEND	2	/* PR_SHUTDOWN_SEND +1 */
 #define ssl_SHUTDOWN_BOTH	3	/* PR_SHUTDOWN_BOTH +1 */
 
+
 /*
 ** A gather object. Used to read some data until a count has been
 ** satisfied. Primarily for support of async sockets.
@@ -448,8 +456,15 @@
     ** The portion of the SSL record header put here always comes off the wire 
     ** as plaintext, never ciphertext.
     ** For SSL2, the plaintext portion is two bytes long.  For SSl3 it is 5.
-    */
-    unsigned char hdr[5];					/* ssl 2 & 3 */
+    ** For DTLS it is 13 
+   */
+    unsigned char hdr[13];					/* ssl 2 & 3, or dtls */
+
+    /* Buffer for DTLS data read off the wire as a single datagram */
+    sslBuffer      dtlsPacket;
+    
+    /* the start of the buffered DTLS record in dtlsPacket */
+    unsigned int   dtlsPacketOffset;
 };
 
 /* sslGather.state */
@@ -521,6 +536,10 @@
     PRUint32         low;
 } SSL3SequenceNumber;
 
+typedef PRUint16 DTLSEpoch;
+
+typedef void (*DtlsTimerCb)(sslSocket *);
+
 #define MAX_MAC_CONTEXT_BYTES 400
 #define MAX_MAC_CONTEXT_LLONGS (MAX_MAC_CONTEXT_BYTES / 8)
 
@@ -547,6 +566,21 @@
     PRUint64    cipher_context[MAX_CIPHER_CONTEXT_LLONGS];
 } ssl3KeyMaterial;
 
+
+/* The DTLS anti-replay window. Defined here because we need it in
+   the cipher spec. Note that this is a ring buffer but left and
+   right represent the true window, with modular arithmetic 
+   used to mape them onto the buffer.
+*/
+#define DTLS_RECVD_RECORDS_WINDOW 1024 /* Packets; approximate
+				       * Must be divisible by 8 
+				       **/
+typedef struct DTLSRecvdRecordsStr {
+    unsigned char data[DTLS_RECVD_RECORDS_WINDOW/8];
+    PRUint64 left;
+    PRUint64 right;
+} DTLSRecvdRecords;
+
 /*
 ** These are the "specs" in the "ssl3" struct.
 ** Access to the pointers to these specs, and all the specs' contents
@@ -582,6 +616,8 @@
     SECItem            srvVirtName;    /* for server: name that was negotiated
                                         * with a client. For client - is
                                         * always set to NULL.*/
+    DTLSEpoch          epoch;
+    DTLSRecvdRecords   recvdRecords;
 } ssl3CipherSpec;
 
 typedef enum {	never_cached, 
@@ -777,6 +813,17 @@
 typedef SECStatus (*sslRestartTarget)(sslSocket *);
 
 /*
+** A DTLS Queued message (potentially to be retransmitted)
+*/
+typedef struct DTLSQueuedMessageStr {
+    PRCList  link;        /* The linked list link */
+    DTLSEpoch epoch;      /* The epoch to use */
+    SSL3ContentType type; /* The message type */
+    unsigned char *data;  /* The data */
+    PRUint16 len;         /* The data length */
+} DTLSQueuedMessage;
+
+/*
 ** This is the "hs" member of the "ssl3" struct.
 ** This entire struct is protected by ssl3HandshakeLock
 */
@@ -831,6 +878,25 @@
     sslRestartTarget      restartTarget;
     /* Shared state between ssl3_HandleFinished and ssl3_FinishHandshake */
     PRBool                cacheSID;
+
+    /* This group of values is used for DTLS */
+    PRUint16              sendMessageSeq;  /* The sending message sequence number*/
+    PRCList *             lastMessageFlight; /* The last message flight we sent. This
+					      * is a pointer because ssl_FreeSocket
+					      *	relocates the structure in DEBUG mode
+					      *	which messes up the list macros */
+    PRUint16              maxMessageSent;    /* The largest message we sent */
+    PRUint16              recvMessageSeq;  /* The receiving message sequence number*/
+    sslBuffer             recvdFragments;  /* The fragments we have received in a bitmask */
+    PRInt32               recvdHighWater;  /* The high water mark for fragments received
+					    * -1 means no reassembly in progress. */
+    unsigned char         cookie[32];      /* The cookie */
+    unsigned char         cookieLen;       /* The length of the cookie*/
+    PRIntervalTime        rtTimerStarted;  /* When the timer was started */
+    DtlsTimerCb           rtTimerCb;       /* The timer to call on expiry */

[wtc, 2012/03/21 01:22:07]

Nit: this comment should read "The function to call on expiry",
right?

[ekr, 2012/03/21 01:36:40]

Agreed.

+    PRUint32              rtTimeoutMs;     /* The length of the current timeout
+					    * used for backoff (in ms)*/
+    PRUint32              rtRetries;       /* The retry counter */

[wtc, 2012/03/21 01:22:07]

In the names of these members, "rt" stands for "retransmission",
right?

[ekr, 2012/03/21 01:36:40]

Correct.

 } SSL3HandshakeState;
 
 
@@ -882,11 +948,18 @@
      */
     SECItem		 nextProto;
     SSLNextProtoState    nextProtoState;
+    PRUint16             mtu;     /* Our estimate of the MTU */
 };
 
+#define DTLS_MAX_MTU  (1500)      /* Ethernet MTU but without subtracting the headers,
+				   * so slightly larger than expected */
+#define IS_DTLS(ss) (ss->protocolVariant == ssl_variant_datagram)
+
+
 typedef struct {
     SSL3ContentType      type;
     SSL3ProtocolVersion  version;
+    SSL3SequenceNumber   seq_num;  /* DTLS only */
     sslBuffer *          buf;
 } SSL3Ciphertext;
 
@@ -1083,7 +1156,6 @@
     /* version of the protocol to use */
     SSL3ProtocolVersion version;
     SSL3ProtocolVersion clientHelloVersion; /* version sent in client hello. */
-
     sslSecurityInfo  sec;		/* not a pointer any more */
 
     /* protected by firstHandshakeLock AND (in ssl3) ssl3HandshakeLock. */
@@ -1188,6 +1260,9 @@
     /* True when the current session is a stateless resume. */
     PRBool               statelessResume;
     TLSExtensionData     xtnData;
+
+    /* Whether we are doing stream or datagram mode */
+    SSLProtocolVariant   protocolVariant;
 };
 
 
@@ -1321,7 +1396,28 @@
 extern SECStatus ssl_EnableNagleDelay(sslSocket *ss, PRBool enabled);
 
 extern PRBool    ssl3_CanFalseStart(sslSocket *ss);
+extern PRInt32    ssl3_SendRecord(sslSocket *ss, DTLSEpoch epoch,
+				  SSL3ContentType type,
+                                  const SSL3Opaque* pIn, PRInt32 nIn,
+                                  PRInt32 flags);
 
+#ifdef NSS_ENABLE_ZLIB
+/*
+ * The DEFLATE algorithm can result in an expansion of 0.1% + 12 bytes. For a
+ * maximum TLS record payload of 2**14 bytes, that's 29 bytes.
+ */
+#define SSL3_COMPRESSION_MAX_EXPANSION 29
+#else  /* !NSS_ENABLE_ZLIB */
+#define SSL3_COMPRESSION_MAX_EXPANSION 0
+#endif
+
+/*
+ * make sure there is room in the write buffer for padding and
+ * other compression and cryptographic expansions.
+ */
+#define SSL3_BUFFER_FUDGE     100 + SSL3_COMPRESSION_MAX_EXPANSION
+
+
 #define SSL_LOCK_READER(ss)		if (ss->recvLock) PZ_Lock(ss->recvLock)
 #define SSL_UNLOCK_READER(ss)		if (ss->recvLock) PZ_Unlock(ss->recvLock)
 #define SSL_LOCK_WRITER(ss)		if (ss->sendLock) PZ_Lock(ss->sendLock)
@@ -1436,7 +1532,7 @@
 /*
  * SSL3 specific routines
  */
-SECStatus ssl3_SendClientHello(sslSocket *ss);
+SECStatus ssl3_SendClientHello(sslSocket *ss, PRBool resending);
 
 /*
  * input into the SSL3 machinery from the actualy network reading code
@@ -1531,6 +1627,8 @@
 						 unsigned char *cs, int *size);
 
 extern SECStatus ssl3_RedoHandshake(sslSocket *ss, PRBool flushCache);
+extern SECStatus ssl3_HandleHandshakeMessage(sslSocket *ss, SSL3Opaque *b, 
+    PRUint32 length);
 
 extern void ssl3_DestroySSL3Info(sslSocket *ss);
 
@@ -1680,6 +1778,9 @@
                     SSL3KEAType               exchKeyType, 
 		    SSLWrappedSymWrappingKey *wswk);
 
+/* Generate an error */
+extern SECStatus ssl3_DecodeError(sslSocket *ss);
+
 /* The caller passes in the new value it wants
  * to set.  This code tests the wrapped sym key entry in the file on disk.  
  * If it is uninitialized, this function writes the caller's value into 
@@ -1724,6 +1825,41 @@
         CERTCertList* list);
 #endif  /* NSS_PLATFORM_CLIENT_AUTH */
 
+/**************** DTLS-specific functions **************/
+extern void dtls_FreeQueuedMessage(DTLSQueuedMessage *msg);
+extern void dtls_FreeQueuedMessages(PRCList *lst);
+extern void dtls_FreeHandshakeMessages(PRCList *lst);
+
+extern SECStatus dtls_HandleHandshake(sslSocket *ss, 
+    sslBuffer *origBuf);
+extern SECStatus dtls_HandleHelloVerifyRequest(sslSocket *ss,
+					       SSL3Opaque *b, PRUint32 length);
+extern SECStatus dtls_StageHandshakeMessage(sslSocket *ss);
+extern SECStatus dtls_QueueMessage(sslSocket *ss, SSL3ContentType type,
+    const SSL3Opaque *pIn, PRInt32 nIn);
+extern SECStatus dtls_FlushHandshakeMessages(sslSocket *ss, 
+    PRInt32 flags);
+extern SECStatus dtls_CompressMACEncryptRecord(sslSocket *ss,
+					       DTLSEpoch epoch, PRBool use_epoch,
+					       SSL3ContentType type, const SSL3Opaque * pIn,
+					       PRUint32 contentLen,
+					       sslBuffer * wrBuf);
+SECStatus ssl3_DisableNonDTLSSuites(sslSocket * ss);
+extern SECStatus dtls_StartTimer(sslSocket *ss, DtlsTimerCb cb);
+extern SECStatus dtls_RestartTimer(sslSocket *ss, PRBool backoff,
+				   DtlsTimerCb cb);
+extern void dtls_CheckTimer(sslSocket *ss);
+extern void dtls_CancelTimer(sslSocket *ss);
+extern void dtls_FinishedTimerCb(sslSocket *ss);
+extern void dtls_SetMTU(sslSocket *ss, PRUint16 advertised);
+extern void dtls_InitRecvdRecords(DTLSRecvdRecords *records);
+extern int dtls_RecordGetRecvd(DTLSRecvdRecords *records, PRUint64 seq);
+extern void dtls_RecordSetRecvd(DTLSRecvdRecords *records, PRUint64 seq);
+extern SSL3ProtocolVersion
+dtls_TLSVersionToDTLSVersion(SSL3ProtocolVersion ttlsv);
+extern SSL3ProtocolVersion
+dtls_DTLSVersionToTLSVersion(SSL3ProtocolVersion dtlsv);
+
 /********************** misc calls *********************/
 
 extern int ssl_MapLowLevelError(int hiLevelError);
@@ -1735,13 +1871,28 @@
 SECStatus SSL_DisableDefaultExportCipherSuites(void);
 SECStatus SSL_DisableExportCipherSuites(PRFileDesc * fd);
 PRBool    SSL_IsExportCipherSuite(PRUint16 cipherSuite);
-
 extern SECStatus
 ssl3_TLSPRFWithMasterSecret(ssl3CipherSpec *spec,
                             const char *label, unsigned int labelLen,
                             const unsigned char *val, unsigned int valLen,
                             unsigned char *out, unsigned int outLen);
 
+
+/****************** Exposed for DTLS ********************/
+extern SECStatus
+ssl3_CompressMACEncryptRecord(ssl3CipherSpec *   cwSpec,
+		              PRBool             isServer,
+			      PRBool             isDtls,
+                              SSL3ContentType    type,
+		              const SSL3Opaque * pIn,
+		              PRUint32           contentLen,
+		              sslBuffer *        wrBuf);
+extern void ssl3_DestroyCipherSpec(ssl3CipherSpec *spec, PRBool freeSrvName);
+extern const ssl3CipherSuiteDef *ssl_LookupCipherSuiteDef(ssl3CipherSuite
+							  suite);
+extern void dtls_RehandshakeCleanup(sslSocket *ss);
+
+
 #ifdef TRACE
 #define SSL_TRACE(msg) ssl_Trace msg
 #else