Add DTLS support to NSS, contributed by Eric Rescorla.

net/third_party/nss/ssl/sslimpl.h

 /*
  * This file is PRIVATE to SSL and should be the first thing included by
  * any SSL implementation file.
  *
  * ***** BEGIN LICENSE BLOCK *****
  * Version: MPL 1.1/GPL 2.0/LGPL 2.1
  *
  * The contents of this file are subject to the Mozilla Public License Version
  * 1.1 (the "License"); you may not use this file except in compliance with
  * the License. You may obtain a copy of the License at
@@ skipping 44 matching lines @@
 #include "sslerr.h"
 #include "ssl3prot.h"
 #include "hasht.h"
 #include "nssilock.h"
 #include "pkcs11t.h"
 #if defined(XP_UNIX) || defined(XP_BEOS)
 #include "unistd.h"
 #endif
 #include "nssrwlk.h"
 #include "prthread.h"
+#include "prclist.h"
 
 #include "sslt.h" /* for some formerly private types, now public */
 
 #ifdef NSS_PLATFORM_CLIENT_AUTH
 #if defined(XP_WIN32)
 #include <windows.h>
 #include <wincrypt.h>
 #elif defined(XP_MACOSX)
 #include <Security/Security.h>
 #endif
@@ skipping 113 matching lines @@
 #else
 #define SSL3_SUPPORTED_CURVES_MASK 0x3fffffe
 #endif
 
 #ifndef BPB
 #define BPB 8 /* Bits Per Byte */
 #endif
 
 #define EXPORT_RSA_KEY_LENGTH 64        /* bytes */
 
+#define INITIAL_DTLS_TIMEOUT_MS   1000  /* Default value from RFC 4347 = 1s*/
+#define MAX_DTLS_TIMEOUT_MS      60000  /* 1 minute */
+#define DTLS_FINISHED_TIMER_MS  120000  /* Time to wait in FINISHED state */
+
 typedef struct sslBufferStr             sslBuffer;
 typedef struct sslConnectInfoStr        sslConnectInfo;
 typedef struct sslGatherStr             sslGather;
 typedef struct sslSecurityInfoStr       sslSecurityInfo;
 typedef struct sslSessionIDStr          sslSessionID;
 typedef struct sslSocketStr             sslSocket;
 typedef struct sslSocketOpsStr          sslSocketOps;
 
 typedef struct ssl3StateStr             ssl3State;
 typedef struct ssl3CertNodeStr          ssl3CertNode;
@@ skipping 72 matching lines @@
     int         (*read)    (sslSocket *, unsigned char *, int);
     int         (*write)   (sslSocket *, const unsigned char *, int);
 
     int         (*getpeername)(sslSocket *, PRNetAddr *);
     int         (*getsockname)(sslSocket *, PRNetAddr *);
 };
 
 /* Flags interpreted by ssl send functions. */
 #define ssl_SEND_FLAG_FORCE_INTO_BUFFER 0x40000000
 #define ssl_SEND_FLAG_NO_BUFFER         0x20000000
+#define ssl_SEND_FLAG_USE_EPOCH         0x10000000 /* DTLS only */
+#define ssl_SEND_FLAG_NO_RETRANSMIT     0x08000000 /* DTLS only */
 #define ssl_SEND_FLAG_MASK              0x7f000000
 
 /*
 ** A buffer object.
 */
 struct sslBufferStr {
     unsigned char *     buf;
     unsigned int        len;
     unsigned int        space;
 };
@@ skipping 141 matching lines @@
     unsigned int  writeOffset; 
 
     /* Buffer for ssl3 to read (encrypted) data from the socket */
     sslBuffer     inbuf;                        /*recvBufLock*/ /* ssl3 only */
 
     /* The ssl[23]_GatherData functions read data into this buffer, rather
     ** than into buf or inbuf, while in the GS_HEADER state.  
     ** The portion of the SSL record header put here always comes off the wire 
     ** as plaintext, never ciphertext.
     ** For SSL2, the plaintext portion is two bytes long.  For SSl3 it is 5.
+    ** For DTLS it is 13.
     */
-    unsigned char hdr[5];                                       /* ssl 2 & 3 */
+    unsigned char hdr[13];                              /* ssl 2 & 3 or dtls */
+
+    /* Buffer for DTLS data read off the wire as a single datagram */
+    sslBuffer     dtlsPacket;
+
+    /* the start of the buffered DTLS record in dtlsPacket */
+    unsigned int  dtlsPacketOffset;
 };
 
 /* sslGather.state */
 #define GS_INIT         0
 #define GS_HEADER       1
 #define GS_MAC          2
 #define GS_DATA         3
 #define GS_PAD          4
 
 typedef SECStatus (*SSLCipher)(void *               context, 
@@ skipping 51 matching lines @@
 #define MAX_IV_LENGTH 24
 
 /*
  * Do not depend upon 64 bit arithmetic in the underlying machine. 
  */
 typedef struct {
     PRUint32         high;
     PRUint32         low;
 } SSL3SequenceNumber;
 
+typedef PRUint16 DTLSEpoch;
+
+typedef void (*DTLSTimerCb)(sslSocket *);
+
 #define MAX_MAC_CONTEXT_BYTES 400
 #define MAX_MAC_CONTEXT_LLONGS (MAX_MAC_CONTEXT_BYTES / 8)
 
 #define MAX_CIPHER_CONTEXT_BYTES 2080
 #define MAX_CIPHER_CONTEXT_LLONGS (MAX_CIPHER_CONTEXT_BYTES / 8)
 
 typedef struct {
     SSL3Opaque        client_write_iv         [24];
     SSL3Opaque        server_write_iv         [24];
     SSL3Opaque        wrapped_master_secret   [48];
     PRUint16          wrapped_master_secret_len;
     PRUint8           msIsWrapped;
     PRUint8           resumable;
 } ssl3SidKeys; /* 100 bytes */
 
 typedef struct {
     PK11SymKey  *write_key;
     PK11SymKey  *write_mac_key;
     PK11Context *write_mac_context;
     SECItem     write_key_item;
     SECItem     write_iv_item;
     SECItem     write_mac_key_item;
     SSL3Opaque  write_iv[MAX_IV_LENGTH];
     PRUint64    cipher_context[MAX_CIPHER_CONTEXT_LLONGS];
 } ssl3KeyMaterial;
 
+/* The DTLS anti-replay window. Defined here because we need it in
+ * the cipher spec. Note that this is a ring buffer but left and
+ * right represent the true window, with modular arithmetic used to
+ * map them onto the buffer.
+ */
+#define DTLS_RECVD_RECORDS_WINDOW 1024 /* Packets; approximate
+                                        * Must be divisible by 8
+                                        */
+typedef struct DTLSRecvdRecordsStr {
+    unsigned char data[DTLS_RECVD_RECORDS_WINDOW/8];
+    PRUint64 left;
+    PRUint64 right;
+} DTLSRecvdRecords;
+
 /*
 ** These are the "specs" in the "ssl3" struct.
 ** Access to the pointers to these specs, and all the specs' contents
 ** (direct and indirect) is protected by the reader/writer lock ss->specLock.
 */
 typedef struct {
     const ssl3BulkCipherDef *cipher_def;
     const ssl3MACDef * mac_def;
     SSLCompressionMethod compression_method;
     int                mac_size;
@@ skipping 15 matching lines @@
     SSL3SequenceNumber read_seq_num;
     SSL3ProtocolVersion version;
     ssl3KeyMaterial    client;
     ssl3KeyMaterial    server;
     SECItem            msItem;
     unsigned char      key_block[NUM_MIXERS * MD5_LENGTH];
     unsigned char      raw_master_secret[56];
     SECItem            srvVirtName;    /* for server: name that was negotiated
                                         * with a client. For client - is
                                         * always set to NULL.*/
+    DTLSEpoch          epoch;
+    DTLSRecvdRecords   recvdRecords;
 } ssl3CipherSpec;
 
 typedef enum {  never_cached, 
                 in_client_cache, 
                 in_server_cache, 
                 invalid_cache           /* no longer in any cache. */
 } Cached;
 
 #define MAX_PEER_CERT_CHAIN_SIZE 8
 
@@ skipping 175 matching lines @@
      * Names data is not coppied from the input buffer. It can not be
      * used outside the scope where input buffer is defined and that
      * is beyond ssl3_HandleClientHello function. */
     SECItem *sniNameArr;
     PRUint32 sniNameArrSize;
 };
 
 typedef SECStatus (*sslRestartTarget)(sslSocket *);
 
 /*
+** A DTLS queued message (potentially to be retransmitted)
+*/
+typedef struct DTLSQueuedMessageStr {
+    PRCList link;         /* The linked list link */
+    DTLSEpoch epoch;      /* The epoch to use */
+    SSL3ContentType type; /* The message type */
+    unsigned char *data;  /* The data */
+    PRUint16 len;         /* The data length */
+} DTLSQueuedMessage;
+
+/*
 ** This is the "hs" member of the "ssl3" struct.
 ** This entire struct is protected by ssl3HandshakeLock
 */
 typedef struct SSL3HandshakeStateStr {
     SSL3Random            server_random;
     SSL3Random            client_random;
     SSL3WaitState         ws;
     PRUint64              md5_cx[MAX_MAC_CONTEXT_LLONGS];
     PRUint64              sha_cx[MAX_MAC_CONTEXT_LLONGS];
     PK11Context *         md5;            /* handshake running hashes */
@@ skipping 34 matching lines @@
     PRUint32              negotiatedECCurves; /* bit mask */
 #endif /* NSS_ENABLE_ECC */
 
     PRBool                authCertificatePending;
     /* Which function should SSL_RestartHandshake* call if we're blocked?
      * One of NULL, ssl3_SendClientSecondRound, ssl3_FinishHandshake,
      * or ssl3_AlwaysFail */
     sslRestartTarget      restartTarget;
     /* Shared state between ssl3_HandleFinished and ssl3_FinishHandshake */
     PRBool                cacheSID;
+
+    /* This group of values is used for DTLS */
+    PRUint16              sendMessageSeq;  /* The sending message sequence
+                                            * number*/
+    PRCList *             lastMessageFlight; /* The last message flight we sent.
+                                              * This is a pointer because
+                                              * ssl_FreeSocket relocates the
+                                              * structure in DEBUG mode, which
+                                              * messes up the list macros */
+    PRUint16              maxMessageSent;    /* The largest message we sent */
+    PRUint16              recvMessageSeq;  /* The receiving message sequence
+                                            * number*/
+    sslBuffer             recvdFragments;  /* The fragments we have received in
+                                            * a bitmask */
+    PRInt32               recvdHighWater;  /* The high water mark for fragments
+                                            * received. -1 means no reassembly
+                                            * in progress. */
+    unsigned char         cookie[32];      /* The cookie */
+    unsigned char         cookieLen;       /* The length of the cookie*/
+    PRIntervalTime        rtTimerStarted;  /* When the timer was started */
+    DTLSTimerCb           rtTimerCb;       /* The function to call on expiry */
+    PRUint32              rtTimeoutMs;     /* The length of the current timeout
+                                            * used for backoff (in ms)*/
+    PRUint32              rtRetries;       /* The retry counter */
 } SSL3HandshakeState;
 
 
 
 /*
 ** This is the "ssl3" struct, as in "ss->ssl3".
 ** note:
 ** usually,   crSpec == cwSpec and prSpec == pwSpec.  
 ** Sometimes, crSpec == pwSpec and prSpec == cwSpec.
 ** But there are never more than 2 actual specs.  
@@ skipping 31 matching lines @@
                             /* used by server.  trusted CAs for this socket. */
     PRBool               initialized;
     SSL3HandshakeState   hs;
     ssl3CipherSpec       specs[2];      /* one is current, one is pending. */
 
     /* In a client: if the server supports Next Protocol Negotiation, then
      * this is the protocol that was negotiated.
      */
     SECItem              nextProto;
     SSLNextProtoState    nextProtoState;
+
+    PRUint16             mtu;   /* Our estimate of the MTU */
 };
 
+#define DTLS_MAX_MTU  1500      /* Ethernet MTU but without subtracting the
+                                 * headers, so slightly larger than expected */
+#define IS_DTLS(ss) (ss->protocolVariant == ssl_variant_datagram)
+
 typedef struct {
     SSL3ContentType      type;
     SSL3ProtocolVersion  version;
+    SSL3SequenceNumber   seq_num;  /* DTLS only */
     sslBuffer *          buf;
 } SSL3Ciphertext;
 
 struct ssl3KeyPairStr {
     SECKEYPrivateKey *    privKey;
     SECKEYPublicKey *     pubKey;
     PRInt32               refCount;     /* use PR_Atomic calls for this. */
 };
 
 typedef struct SSLWrappedSymWrappingKeyStr {
@@ skipping 281 matching lines @@
 
     /* SSL3 state info.  Formerly was a pointer */
     ssl3State        ssl3;
 
     /*
      * TLS extension related data.
      */
     /* True when the current session is a stateless resume. */
     PRBool               statelessResume;
     TLSExtensionData     xtnData;
+
+    /* Whether we are doing stream or datagram mode */
+    SSLProtocolVariant   protocolVariant;
 };
 
 
 
 /* All the global data items declared here should be protected using the 
 ** ssl_global_data_lock, which is a reader/writer lock.
 */
 extern NSSRWLock *             ssl_global_data_lock;
 extern char                    ssl_debug;
 extern char                    ssl_trace;
@@ skipping 113 matching lines @@
 
 extern PRBool    ssl_FdIsBlocking(PRFileDesc *fd);
 
 extern PRBool    ssl_SocketIsBlocking(sslSocket *ss);
 
 extern void      ssl3_SetAlwaysBlock(sslSocket *ss);
 
 extern SECStatus ssl_EnableNagleDelay(sslSocket *ss, PRBool enabled);
 
 extern PRBool    ssl3_CanFalseStart(sslSocket *ss);
+extern SECStatus
+ssl3_CompressMACEncryptRecord(ssl3CipherSpec *   cwSpec,
+                              PRBool             isServer,
+                              PRBool             isDTLS,
+                              SSL3ContentType    type,
+                              const SSL3Opaque * pIn,
+                              PRUint32           contentLen,
+                              sslBuffer *        wrBuf);
+extern PRInt32   ssl3_SendRecord(sslSocket *ss, DTLSEpoch epoch,
+                                 SSL3ContentType type,
+                                 const SSL3Opaque* pIn, PRInt32 nIn,
+                                 PRInt32 flags);
+
+#ifdef NSS_ENABLE_ZLIB
+/*
+ * The DEFLATE algorithm can result in an expansion of 0.1% + 12 bytes. For a
+ * maximum TLS record payload of 2**14 bytes, that's 29 bytes.
+ */
+#define SSL3_COMPRESSION_MAX_EXPANSION 29
+#else  /* !NSS_ENABLE_ZLIB */
+#define SSL3_COMPRESSION_MAX_EXPANSION 0
+#endif
+
+/*
+ * make sure there is room in the write buffer for padding and
+ * other compression and cryptographic expansions.
+ */
+#define SSL3_BUFFER_FUDGE     100 + SSL3_COMPRESSION_MAX_EXPANSION
 
 #define SSL_LOCK_READER(ss)             if (ss->recvLock) PZ_Lock(ss->recvLock)
 #define SSL_UNLOCK_READER(ss)           if (ss->recvLock) PZ_Unlock(ss->recvLock)
 #define SSL_LOCK_WRITER(ss)             if (ss->sendLock) PZ_Lock(ss->sendLock)
 #define SSL_UNLOCK_WRITER(ss)           if (ss->sendLock) PZ_Unlock(ss->sendLock)
 
 /* firstHandshakeLock -> recvBufLock */
 #define ssl_Get1stHandshakeLock(ss)     \
     { if (!ss->opt.noLocks) { \
           PORT_Assert(PZ_InMonitor((ss)->firstHandshakeLock) || \
@@ skipping 76 matching lines @@
                     const unsigned char * cr, const unsigned char * sr,
                     const SECItem * pms, PRBool isTLS, PRBool isRSA);
 
 /* These functions are called from secnav, even though they're "private". */
 
 extern int ssl2_SendErrorMessage(struct sslSocketStr *ss, int error);
 extern sslSocket *ssl_FindSocket(PRFileDesc *fd);
 extern void ssl_FreeSocket(struct sslSocketStr *ssl);
 extern SECStatus SSL3_SendAlert(sslSocket *ss, SSL3AlertLevel level,
                                 SSL3AlertDescription desc);
+extern SECStatus ssl3_DecodeError(sslSocket *ss);
 
 extern SECStatus ssl3_RestartHandshakeAfterCertReq(sslSocket *    ss,
                                              CERTCertificate *    cert, 
                                              SECKEYPrivateKey *   key,
                                              CERTCertificateList *certChain);
 
 extern SECStatus ssl3_AuthCertificateComplete(sslSocket *ss, PRErrorCode error);
 
 /*
  * for dealing with SSL 3.0 clients sending SSL 2.0 format hellos
  */
 extern SECStatus ssl3_HandleV2ClientHello(
     sslSocket *ss, unsigned char *buffer, int length);
 extern SECStatus ssl3_StartHandshakeHash(
     sslSocket *ss, unsigned char *buf, int length);
 
 /*
  * SSL3 specific routines
  */
-SECStatus ssl3_SendClientHello(sslSocket *ss);
+SECStatus ssl3_SendClientHello(sslSocket *ss, PRBool resending);
 
 /*
  * input into the SSL3 machinery from the actualy network reading code
  */
 SECStatus ssl3_HandleRecord(
     sslSocket *ss, SSL3Ciphertext *cipher, sslBuffer *out);
 
 int ssl3_GatherAppDataRecord(sslSocket *ss, int flags);
 int ssl3_GatherCompleteHandshake(sslSocket *ss, int flags);
 /*
@@ skipping 74 matching lines @@
 extern SECStatus ssl2_SetPolicy(PRInt32 which, PRInt32 policy);
 extern SECStatus ssl2_GetPolicy(PRInt32 which, PRInt32 *policy);
 
 extern void      ssl2_InitSocketPolicy(sslSocket *ss);
 extern void      ssl3_InitSocketPolicy(sslSocket *ss);
 
 extern SECStatus ssl3_ConstructV2CipherSpecsHack(sslSocket *ss,
                                                  unsigned char *cs, int *size);
 
 extern SECStatus ssl3_RedoHandshake(sslSocket *ss, PRBool flushCache);
+extern SECStatus ssl3_HandleHandshakeMessage(sslSocket *ss, SSL3Opaque *b, 
+                                             PRUint32 length);
 
 extern void ssl3_DestroySSL3Info(sslSocket *ss);
 
 extern SECStatus ssl3_NegotiateVersion(sslSocket *ss, 
                                        SSL3ProtocolVersion peerVersion,
                                        PRBool allowLargerPeerVersion);
 
 extern SECStatus ssl_GetPeerInfo(sslSocket *ss);
 
 #ifdef NSS_ENABLE_ECC
 /* ECDH functions */
 extern SECStatus ssl3_SendECDHClientKeyExchange(sslSocket * ss, 
                              SECKEYPublicKey * svrPubKey);
 extern SECStatus ssl3_HandleECDHServerKeyExchange(sslSocket *ss, 
                                         SSL3Opaque *b, PRUint32 length);
 extern SECStatus ssl3_HandleECDHClientKeyExchange(sslSocket *ss, 
                                      SSL3Opaque *b, PRUint32 length,
                                      SECKEYPublicKey *srvrPubKey,
                                      SECKEYPrivateKey *srvrPrivKey);
 extern SECStatus ssl3_SendECDHServerKeyExchange(sslSocket *ss);
 #endif
 
 extern SECStatus ssl3_ComputeCommonKeyHash(PRUint8 * hashBuf, 
                                 unsigned int bufLen, SSL3Hashes *hashes, 
                                 PRBool bypassPKCS11);
+extern void ssl3_DestroyCipherSpec(ssl3CipherSpec *spec, PRBool freeSrvName);
 extern SECStatus ssl3_InitPendingCipherSpec(sslSocket *ss, PK11SymKey *pms);
 extern SECStatus ssl3_AppendHandshake(sslSocket *ss, const void *void_src, 
                         PRInt32 bytes);
 extern SECStatus ssl3_AppendHandshakeHeader(sslSocket *ss, 
                         SSL3HandshakeType t, PRUint32 length);
 extern SECStatus ssl3_AppendHandshakeNumber(sslSocket *ss, PRInt32 num, 
                         PRInt32 lenSize);
 extern SECStatus ssl3_AppendHandshakeVariable( sslSocket *ss, 
                         const SSL3Opaque *src, PRInt32 bytes, PRInt32 lenSize);
 extern SECStatus ssl3_ConsumeHandshake(sslSocket *ss, void *v, PRInt32 bytes, 
@@ skipping 148 matching lines @@
 // Converts a CERTCertList* (A collection of CERTCertificates) into a
 // CERTCertificateList* (A collection of SECItems), or returns NULL if
 // it cannot be converted.
 // This is to allow the platform-supplied chain to be created with purely
 // public API functions, using the preferred CERTCertList mutators, rather
 // pushing this hack to clients.
 extern CERTCertificateList* hack_NewCertificateListFromCertList(
         CERTCertList* list);
 #endif  /* NSS_PLATFORM_CLIENT_AUTH */
 
+/**************** DTLS-specific functions **************/
+extern void dtls_FreeQueuedMessage(DTLSQueuedMessage *msg);
+extern void dtls_FreeQueuedMessages(PRCList *lst);
+extern void dtls_FreeHandshakeMessages(PRCList *lst);
+
+extern SECStatus dtls_HandleHandshake(sslSocket *ss, sslBuffer *origBuf);
+extern SECStatus dtls_HandleHelloVerifyRequest(sslSocket *ss,
+                                               SSL3Opaque *b, PRUint32 length);
+extern SECStatus dtls_StageHandshakeMessage(sslSocket *ss);
+extern SECStatus dtls_QueueMessage(sslSocket *ss, SSL3ContentType type,
+                                   const SSL3Opaque *pIn, PRInt32 nIn);
+extern SECStatus dtls_FlushHandshakeMessages(sslSocket *ss, PRInt32 flags);
+extern SECStatus dtls_CompressMACEncryptRecord(sslSocket *ss,
+                                               DTLSEpoch epoch,
+                                               PRBool use_epoch,
+                                               SSL3ContentType type,
+                                               const SSL3Opaque *pIn,
+                                               PRUint32 contentLen,
+                                               sslBuffer *wrBuf);
+SECStatus ssl3_DisableNonDTLSSuites(sslSocket * ss);
+extern SECStatus dtls_StartTimer(sslSocket *ss, DTLSTimerCb cb);
+extern SECStatus dtls_RestartTimer(sslSocket *ss, PRBool backoff,
+                                   DTLSTimerCb cb);
+extern void dtls_CheckTimer(sslSocket *ss);
+extern void dtls_CancelTimer(sslSocket *ss);
+extern void dtls_FinishedTimerCb(sslSocket *ss);
+extern void dtls_SetMTU(sslSocket *ss, PRUint16 advertised);
+extern void dtls_InitRecvdRecords(DTLSRecvdRecords *records);
+extern int dtls_RecordGetRecvd(DTLSRecvdRecords *records, PRUint64 seq);
+extern void dtls_RecordSetRecvd(DTLSRecvdRecords *records, PRUint64 seq);
+extern void dtls_RehandshakeCleanup(sslSocket *ss);
+extern SSL3ProtocolVersion
+dtls_TLSVersionToDTLSVersion(SSL3ProtocolVersion tlsv);
+extern SSL3ProtocolVersion
+dtls_DTLSVersionToTLSVersion(SSL3ProtocolVersion dtlsv);
+
 /********************** misc calls *********************/
 
 extern int ssl_MapLowLevelError(int hiLevelError);
 
 extern PRUint32 ssl_Time(void);
 
 extern void SSL_AtomicIncrementLong(long * x);
 
 SECStatus SSL_DisableDefaultExportCipherSuites(void);
 SECStatus SSL_DisableExportCipherSuites(PRFileDesc * fd);
@@ skipping 20 matching lines @@
 #elif defined(_WIN32_WCE)
 #define SSL_GETPID GetCurrentProcessId
 #elif defined(WIN32)
 extern int __cdecl _getpid(void);
 #define SSL_GETPID _getpid
 #else
 #define SSL_GETPID() 0
 #endif
 
 #endif /* __sslimpl_h_ */