Bug 1515236 - Add a test that SSLKEYLOGFILE and SSLDEBUGFILE init properly r=mt

There is a new test here because the keylog unittest sets the environment
variable for SSLKEYLOG and NSPR provides no mechanism (cross-platform-wise) to
actually delete an environment variable, so I made another file for the base
uninitialized case.

5 files changed, 143 insertions, 31 deletions

diff --git a/gtests/ssl_gtest/Makefile b/gtests/ssl_gtest/Makefile
index 95c111aeb..46f030357 100644
--- a/gtests/ssl_gtest/Makefile
+++ b/gtests/ssl_gtest/Makefile
@@ -36,6 +36,12 @@ CPPSRCS := $(filter-out $(shell grep -l '^TEST_F' $(CPPSRCS)), $(CPPSRCS))
 CFLAGS += -DNSS_DISABLE_TLS_1_3
 endif
 
+ifdef NSS_ALLOW_SSLKEYLOGFILE
+SSLKEYLOGFILE_FILES = ssl_keylog_unittest.cc
+else
+SSLKEYLOGFILE_FILES = $(NULL)
+endif
+
 #######################################################################
 # (5) Execute "global" rules. (OPTIONAL)                              #
 #######################################################################
diff --git a/gtests/ssl_gtest/manifest.mn b/gtests/ssl_gtest/manifest.mn
index 7f4ee7953..b6efb62cd 100644
--- a/gtests/ssl_gtest/manifest.mn
+++ b/gtests/ssl_gtest/manifest.mn
@@ -20,6 +20,7 @@ CPPSRCS = \
       ssl_ciphersuite_unittest.cc \
       ssl_custext_unittest.cc \
       ssl_damage_unittest.cc \
+      ssl_debug_env_unittest.cc \
       ssl_dhe_unittest.cc \
       ssl_drop_unittest.cc \
       ssl_ecdh_unittest.cc \
@@ -31,7 +32,6 @@ CPPSRCS = \
       ssl_gather_unittest.cc \
       ssl_gtest.cc \
       ssl_hrr_unittest.cc \
-      ssl_keylog_unittest.cc \
       ssl_keyupdate_unittest.cc \
       ssl_loopback_unittest.cc \
       ssl_misc_unittest.cc \
@@ -53,6 +53,7 @@ CPPSRCS = \
       tls_filter.cc \
       tls_protect.cc \
       tls_esni_unittest.cc \
+      $(SSLKEYLOGFILE_FILES) \
       $(NULL)
 
 INCLUDES += -I$(CORE_DEPTH)/gtests/google_test/gtest/include \
diff --git a/gtests/ssl_gtest/ssl_debug_env_unittest.cc b/gtests/ssl_gtest/ssl_debug_env_unittest.cc
new file mode 100644
index 000000000..59ec3d393
--- /dev/null
+++ b/gtests/ssl_gtest/ssl_debug_env_unittest.cc
@@ -0,0 +1,53 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include <cstdlib>
+#include <fstream>
+#include <sstream>
+
+#include "gtest_utils.h"
+#include "tls_connect.h"
+
+namespace nss_test {
+
+extern "C" {
+extern FILE* ssl_trace_iob;
+
+#ifdef NSS_ALLOW_SSLKEYLOGFILE
+extern FILE* ssl_keylog_iob;
+#endif
+}
+
+// These tests ensure that when the associated environment variables are unset
+// that the lazily-initialized defaults are what they are supposed to be.
+
+#ifdef DEBUG
+TEST_P(TlsConnectGeneric, DebugEnvTraceFileNotSet) {
+  char* ev = PR_GetEnvSecure("SSLDEBUGFILE");
+  if (ev && ev[0]) {
+    // note: should use GTEST_SKIP when GTest gets updated to support it
+    return;
+  }
+
+  Connect();
+  EXPECT_EQ(stderr, ssl_trace_iob);
+}
+#endif
+
+#ifdef NSS_ALLOW_SSLKEYLOGFILE
+TEST_P(TlsConnectGeneric, DebugEnvKeylogFileNotSet) {
+  char* ev = PR_GetEnvSecure("SSLKEYLOGFILE");
+  if (ev && ev[0]) {
+    // note: should use GTEST_SKIP when GTest gets updated to support it
+    return;
+  }
+
+  Connect();
+  EXPECT_EQ(nullptr, ssl_keylog_iob);
+}
+#endif
+
+}  // namespace nss_test
diff --git a/gtests/ssl_gtest/ssl_gtest.gyp b/gtests/ssl_gtest/ssl_gtest.gyp
index be1c4ea32..ae7d4c78b 100644
--- a/gtests/ssl_gtest/ssl_gtest.gyp
+++ b/gtests/ssl_gtest/ssl_gtest.gyp
@@ -21,6 +21,7 @@
         'ssl_ciphersuite_unittest.cc',
         'ssl_custext_unittest.cc',
         'ssl_damage_unittest.cc',
+        'ssl_debug_env_unittest.cc',
         'ssl_dhe_unittest.cc',
         'ssl_drop_unittest.cc',
         'ssl_ecdh_unittest.cc',
@@ -99,7 +100,8 @@
       '../../lib/ssl'
     ],
     'defines': [
-      'NSS_USE_STATIC_LIBS'
+      'NSS_USE_STATIC_LIBS',
+      'NSS_ALLOW_SSLKEYLOGFILE=1',
     ],
   },
   'variables': {
diff --git a/gtests/ssl_gtest/ssl_keylog_unittest.cc b/gtests/ssl_gtest/ssl_keylog_unittest.cc
index 322b64837..d616dd250 100644
--- a/gtests/ssl_gtest/ssl_keylog_unittest.cc
+++ b/gtests/ssl_gtest/ssl_keylog_unittest.cc
@@ -15,20 +15,59 @@
 
 namespace nss_test {
 
-static const std::string keylog_file_path = "keylog.txt";
-static const std::string keylog_env = "SSLKEYLOGFILE=" + keylog_file_path;
+static const std::string kKeylogFilePath = "keylog.txt";
+static const std::string kKeylogBlankEnv = "SSLKEYLOGFILE=";
+static const std::string kKeylogSetEnv = kKeylogBlankEnv + kKeylogFilePath;
+
+extern "C" {
+extern FILE* ssl_keylog_iob;
+}
+
+class KeyLogFileTestBase : public TlsConnectGeneric {
+ private:
+  std::string env_to_set_;
 
-class KeyLogFileTest : public TlsConnectGeneric {
  public:
+  virtual void CheckKeyLog() = 0;
+
+  KeyLogFileTestBase(std::string env) : env_to_set_(env) {}
+
   void SetUp() override {
     TlsConnectGeneric::SetUp();
     // Remove previous results (if any).
-    (void)remove(keylog_file_path.c_str());
-    PR_SetEnv(keylog_env.c_str());
+    (void)remove(kKeylogFilePath.c_str());
+    PR_SetEnv(env_to_set_.c_str());
+  }
+
+  void ConnectAndCheck() {
+    // This is a child process, ensure that error messages immediately
+    // propagate or else it will not be visible.
+    ::testing::GTEST_FLAG(throw_on_failure) = true;
+
+    if (version_ == SSL_LIBRARY_VERSION_TLS_1_3) {
+      SetupForZeroRtt();
+      client_->Set0RttEnabled(true);
+      server_->Set0RttEnabled(true);
+      ExpectResumption(RESUME_TICKET);
+      ZeroRttSendReceive(true, true);
+      Handshake();
+      ExpectEarlyDataAccepted(true);
+      CheckConnected();
+      SendReceive();
+    } else {
+      Connect();
+    }
+    CheckKeyLog();
+    _exit(0);
   }
+};
 
-  void CheckKeyLog() {
-    std::ifstream f(keylog_file_path);
+class KeyLogFileTest : public KeyLogFileTestBase {
+ public:
+  KeyLogFileTest() : KeyLogFileTestBase(kKeylogSetEnv) {}
+
+  void CheckKeyLog() override {
+    std::ifstream f(kKeylogFilePath);
     std::map<std::string, size_t> labels;
     std::set<std::string> client_randoms;
     for (std::string line; std::getline(f, line);) {
@@ -65,28 +104,6 @@ class KeyLogFileTest : public TlsConnectGeneric {
       ASSERT_EQ(4U, labels["EXPORTER_SECRET"]);
     }
   }
-
-  void ConnectAndCheck() {
-    // This is a child process, ensure that error messages immediately
-    // propagate or else it will not be visible.
-    ::testing::GTEST_FLAG(throw_on_failure) = true;
-
-    if (version_ == SSL_LIBRARY_VERSION_TLS_1_3) {
-      SetupForZeroRtt();
-      client_->Set0RttEnabled(true);
-      server_->Set0RttEnabled(true);
-      ExpectResumption(RESUME_TICKET);
-      ZeroRttSendReceive(true, true);
-      Handshake();
-      ExpectEarlyDataAccepted(true);
-      CheckConnected();
-      SendReceive();
-    } else {
-      Connect();
-    }
-    CheckKeyLog();
-    _exit(0);
-  }
 };
 
 // Tests are run in a separate process to ensure that NSS is not initialized yet
@@ -113,6 +130,39 @@ INSTANTIATE_TEST_CASE_P(
                        TlsConnectTestBase::kTlsV13));
 #endif
 
+class KeyLogFileUnsetTest : public KeyLogFileTestBase {
+ public:
+  KeyLogFileUnsetTest() : KeyLogFileTestBase(kKeylogBlankEnv) {}
+
+  void CheckKeyLog() override {
+    std::ifstream f(kKeylogFilePath);
+    EXPECT_FALSE(f.good());
+
+    EXPECT_EQ(nullptr, ssl_keylog_iob);
+  }
+};
+
+TEST_P(KeyLogFileUnsetTest, KeyLogFile) {
+  testing::GTEST_FLAG(death_test_style) = "threadsafe";
+
+  ASSERT_EXIT(ConnectAndCheck(), ::testing::ExitedWithCode(0), "");
+}
+
+INSTANTIATE_TEST_CASE_P(
+    KeyLogFileDTLS12, KeyLogFileUnsetTest,
+    ::testing::Combine(TlsConnectTestBase::kTlsVariantsDatagram,
+                       TlsConnectTestBase::kTlsV11V12));
+INSTANTIATE_TEST_CASE_P(
+    KeyLogFileTLS12, KeyLogFileUnsetTest,
+    ::testing::Combine(TlsConnectTestBase::kTlsVariantsStream,
+                       TlsConnectTestBase::kTlsV10ToV12));
+#ifndef NSS_DISABLE_TLS_1_3
+INSTANTIATE_TEST_CASE_P(
+    KeyLogFileTLS13, KeyLogFileUnsetTest,
+    ::testing::Combine(TlsConnectTestBase::kTlsVariantsStream,
+                       TlsConnectTestBase::kTlsV13));
+#endif
+
 }  // namespace nss_test
 
 #endif  // NSS_ALLOW_SSLKEYLOGFILE