Bug 1619102 - Add workaround option to include both DTLS and TLS versions in DTLS supported_versions. r=mt

Add an experimental function for enabling a DTLS 1.3 supported_versions compatibility workaround.

Differential Revision: https://phabricator.services.mozilla.com/D65735

5 files changed, 65 insertions, 0 deletions

diff --git a/gtests/ssl_gtest/ssl_version_unittest.cc b/gtests/ssl_gtest/ssl_version_unittest.cc
index dd8978c59..419a4052b 100644
--- a/gtests/ssl_gtest/ssl_version_unittest.cc
+++ b/gtests/ssl_gtest/ssl_version_unittest.cc
@@ -355,6 +355,36 @@ TEST_F(DtlsConnectTest, DtlsSupportedVersionsEncoding) {
   EXPECT_EQ(SSL_LIBRARY_VERSION_DTLS_1_0_WIRE, static_cast<int>(version));
 }
 
+// Verify the DTLS 1.3 supported_versions interop workaround.
+TEST_F(DtlsConnectTest, Dtls13VersionWorkaround) {
+  static const uint16_t kExpectVersionsWorkaround[] = {
+      0x7f00 | DTLS_1_3_DRAFT_VERSION, SSL_LIBRARY_VERSION_DTLS_1_2_WIRE,
+      SSL_LIBRARY_VERSION_TLS_1_2, SSL_LIBRARY_VERSION_DTLS_1_0_WIRE,
+      SSL_LIBRARY_VERSION_TLS_1_1};
+  const int min_ver = SSL_LIBRARY_VERSION_TLS_1_1,
+            max_ver = SSL_LIBRARY_VERSION_TLS_1_3;
+
+  // Toggle the workaround, then verify both encodings are present.
+  EnsureTlsSetup();
+  SSL_SetDtls13VersionWorkaround(client_->ssl_fd(), PR_TRUE);
+  SSL_SetDtls13VersionWorkaround(client_->ssl_fd(), PR_FALSE);
+  SSL_SetDtls13VersionWorkaround(client_->ssl_fd(), PR_TRUE);
+  client_->SetVersionRange(min_ver, max_ver);
+  server_->SetVersionRange(min_ver, max_ver);
+  auto capture = MakeTlsFilter<TlsExtensionCapture>(
+      client_, ssl_tls13_supported_versions_xtn);
+  Connect();
+
+  uint32_t version = 0;
+  size_t off = 1;
+  ASSERT_EQ(1 + sizeof(kExpectVersionsWorkaround), capture->extension().len());
+  for (unsigned int i = 0; i < PR_ARRAY_SIZE(kExpectVersionsWorkaround); i++) {
+    ASSERT_TRUE(capture->extension().Read(off, 2, &version));
+    EXPECT_EQ(kExpectVersionsWorkaround[i], static_cast<uint16_t>(version));
+    off += 2;
+  }
+}
+
 // Verify the client sends only TLS versions in supported_versions
 TEST_F(TlsConnectTest, TlsSupportedVersionsEncoding) {
   client_->SetVersionRange(SSL_LIBRARY_VERSION_TLS_1_0,
diff --git a/lib/ssl/sslexp.h b/lib/ssl/sslexp.h
index d23ad9411..fb3d612c1 100644
--- a/lib/ssl/sslexp.h
+++ b/lib/ssl/sslexp.h
@@ -943,6 +943,10 @@ typedef struct SSLMaskingContextStr {
                           unsigned int _maskLen),                           \
                          (ctx, sample, sampleLen, mask, maskLen))
 
+#define SSL_SetDtls13VersionWorkaround(fd, enabled)        \
+    SSL_EXPERIMENTAL_API("SSL_SetDtls13VersionWorkaround", \
+                         (PRFileDesc * _fd, PRBool _enabled), (fd, enabled))
+
 /* Deprecated experimental APIs */
 #define SSL_UseAltServerHelloType(fd, enable) SSL_DEPRECATED_EXPERIMENTAL_API
 #define SSL_SetupAntiReplay(a, b, c) SSL_DEPRECATED_EXPERIMENTAL_API
diff --git a/lib/ssl/sslimpl.h b/lib/ssl/sslimpl.h
index 2ca945562..3894a5510 100644
--- a/lib/ssl/sslimpl.h
+++ b/lib/ssl/sslimpl.h
@@ -281,6 +281,7 @@ typedef struct sslOptionsStr {
     unsigned int enableV2CompatibleHello : 1;
     unsigned int enablePostHandshakeAuth : 1;
     unsigned int enableDelegatedCredentials : 1;
+    unsigned int enableDtls13VersionCompat : 1;
 } sslOptions;
 
 typedef enum { sslHandshakingUndetermined = 0,
@@ -1861,6 +1862,8 @@ SSLExp_HkdfVariantExpandLabelWithMech(PRUint16 version, PRUint16 cipherSuite, PK
                                       CK_MECHANISM_TYPE mech, unsigned int keySize,
                                       SSLProtocolVariant variant, PK11SymKey **keyp);
 
+SECStatus SSLExp_SetDtls13VersionWorkaround(PRFileDesc *fd, PRBool enabled);
+
 SECStatus SSLExp_SetTimeFunc(PRFileDesc *fd, SSLTimeFunc f, void *arg);
 
 extern SECStatus ssl_CreateMaskingContextInner(PRUint16 version, PRUint16 cipherSuite,
diff --git a/lib/ssl/sslsock.c b/lib/ssl/sslsock.c
index cf77c187b..e93201647 100644
--- a/lib/ssl/sslsock.c
+++ b/lib/ssl/sslsock.c
@@ -86,6 +86,7 @@ static sslOptions ssl_defaults = {
     .requireDHENamedGroups = PR_FALSE,
     .enable0RttData = PR_FALSE,
     .enableTls13CompatMode = PR_FALSE,
+    .enableDtls13VersionCompat = PR_FALSE,
     .enableDtlsShortHeader = PR_FALSE,
     .enableHelloDowngradeCheck = PR_FALSE,
     .enableV2CompatibleHello = PR_FALSE,
@@ -4249,6 +4250,7 @@ struct {
     EXP(SendCertificateRequest),
     EXP(SendSessionTicket),
     EXP(SetAntiReplayContext),
+    EXP(SetDtls13VersionWorkaround),
     EXP(SetESNIKeyPair),
     EXP(SetMaxEarlyDataSize),
     EXP(SetResumptionTokenCallback),
@@ -4290,6 +4292,17 @@ ssl_ClearPRCList(PRCList *list, void (*f)(void *))
 }
 
 SECStatus
+SSLExp_SetDtls13VersionWorkaround(PRFileDesc *fd, PRBool enabled)
+{
+    sslSocket *ss = ssl_FindSocket(fd);
+    if (!ss) {
+        return SECFailure;
+    }
+    ss->opt.enableDtls13VersionCompat = enabled;
+    return SECSuccess;
+}
+
+SECStatus
 SSLExp_SetTimeFunc(PRFileDesc *fd, SSLTimeFunc f, void *arg)
 {
     sslSocket *ss = ssl_FindSocket(fd);
diff --git a/lib/ssl/tls13exthandle.c b/lib/ssl/tls13exthandle.c
index 95e2ba9bc..05ce8f27b 100644
--- a/lib/ssl/tls13exthandle.c
+++ b/lib/ssl/tls13exthandle.c
@@ -795,6 +795,21 @@ tls13_ClientSendSupportedVersionsXtn(const sslSocket *ss, TLSExtensionData *xtnD
         if (rv != SECSuccess) {
             return SECFailure;
         }
+
+        if (ss->opt.enableDtls13VersionCompat &&
+            ss->protocolVariant == ssl_variant_datagram) {
+            switch (version) {
+                case SSL_LIBRARY_VERSION_TLS_1_2:
+                case SSL_LIBRARY_VERSION_TLS_1_1:
+                    rv = sslBuffer_AppendNumber(buf, (PRUint16)version, 2);
+                    break;
+                default:
+                    continue;
+            }
+            if (rv != SECSuccess) {
+                return SECFailure;
+            }
+        }
     }
 
     rv = sslBuffer_InsertLength(buf, lengthOffset, 1);