Bug 1515236 - Add a test that SSLKEYLOGFILE and SSLDEBUGFILE init properly r=KevinJacobs

Backs out backout 92e62d0731bd6ce961359b4c438448f2cf548bf2

Note: Some parts were included in rev 6f442f3ba2239b857c8fd540468f9f83c84b7c0b

original phab link: https://phabricator.services.mozilla.com/D17380

Differential Revision: https://phabricator.services.mozilla.com/D32917

4 files changed, 134 insertions, 29 deletions

diff --git a/gtests/ssl_gtest/manifest.mn b/gtests/ssl_gtest/manifest.mn
index e060f1244..15a971b4d 100644
--- a/gtests/ssl_gtest/manifest.mn
+++ b/gtests/ssl_gtest/manifest.mn
@@ -20,6 +20,7 @@ CPPSRCS = \
       ssl_ciphersuite_unittest.cc \
       ssl_custext_unittest.cc \
       ssl_damage_unittest.cc \
+      ssl_debug_env_unittest.cc \
       ssl_dhe_unittest.cc \
       ssl_drop_unittest.cc \
       ssl_ecdh_unittest.cc \
diff --git a/gtests/ssl_gtest/ssl_debug_env_unittest.cc b/gtests/ssl_gtest/ssl_debug_env_unittest.cc
new file mode 100644
index 000000000..59ec3d393
--- /dev/null
+++ b/gtests/ssl_gtest/ssl_debug_env_unittest.cc
@@ -0,0 +1,53 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include <cstdlib>
+#include <fstream>
+#include <sstream>
+
+#include "gtest_utils.h"
+#include "tls_connect.h"
+
+namespace nss_test {
+
+extern "C" {
+extern FILE* ssl_trace_iob;
+
+#ifdef NSS_ALLOW_SSLKEYLOGFILE
+extern FILE* ssl_keylog_iob;
+#endif
+}
+
+// These tests ensure that when the associated environment variables are unset
+// that the lazily-initialized defaults are what they are supposed to be.
+
+#ifdef DEBUG
+TEST_P(TlsConnectGeneric, DebugEnvTraceFileNotSet) {
+  char* ev = PR_GetEnvSecure("SSLDEBUGFILE");
+  if (ev && ev[0]) {
+    // note: should use GTEST_SKIP when GTest gets updated to support it
+    return;
+  }
+
+  Connect();
+  EXPECT_EQ(stderr, ssl_trace_iob);
+}
+#endif
+
+#ifdef NSS_ALLOW_SSLKEYLOGFILE
+TEST_P(TlsConnectGeneric, DebugEnvKeylogFileNotSet) {
+  char* ev = PR_GetEnvSecure("SSLKEYLOGFILE");
+  if (ev && ev[0]) {
+    // note: should use GTEST_SKIP when GTest gets updated to support it
+    return;
+  }
+
+  Connect();
+  EXPECT_EQ(nullptr, ssl_keylog_iob);
+}
+#endif
+
+}  // namespace nss_test
diff --git a/gtests/ssl_gtest/ssl_gtest.gyp b/gtests/ssl_gtest/ssl_gtest.gyp
index a3531f852..304e8da17 100644
--- a/gtests/ssl_gtest/ssl_gtest.gyp
+++ b/gtests/ssl_gtest/ssl_gtest.gyp
@@ -21,6 +21,7 @@
         'ssl_ciphersuite_unittest.cc',
         'ssl_custext_unittest.cc',
         'ssl_damage_unittest.cc',
+        'ssl_debug_env_unittest.cc',
         'ssl_dhe_unittest.cc',
         'ssl_drop_unittest.cc',
         'ssl_ecdh_unittest.cc',
diff --git a/gtests/ssl_gtest/ssl_keylog_unittest.cc b/gtests/ssl_gtest/ssl_keylog_unittest.cc
index b6d8fcb51..4713e52a2 100644
--- a/gtests/ssl_gtest/ssl_keylog_unittest.cc
+++ b/gtests/ssl_gtest/ssl_keylog_unittest.cc
@@ -13,20 +13,59 @@
 
 namespace nss_test {
 
-static const std::string keylog_file_path = "keylog.txt";
-static const std::string keylog_env = "SSLKEYLOGFILE=" + keylog_file_path;
+static const std::string kKeylogFilePath = "keylog.txt";
+static const std::string kKeylogBlankEnv = "SSLKEYLOGFILE=";
+static const std::string kKeylogSetEnv = kKeylogBlankEnv + kKeylogFilePath;
+
+extern "C" {
+extern FILE* ssl_keylog_iob;
+}
+
+class KeyLogFileTestBase : public TlsConnectGeneric {
+ private:
+  std::string env_to_set_;
 
-class KeyLogFileTest : public TlsConnectGeneric {
  public:
+  virtual void CheckKeyLog() = 0;
+
+  KeyLogFileTestBase(std::string env) : env_to_set_(env) {}
+
   void SetUp() override {
     TlsConnectGeneric::SetUp();
     // Remove previous results (if any).
-    (void)remove(keylog_file_path.c_str());
-    PR_SetEnv(keylog_env.c_str());
+    (void)remove(kKeylogFilePath.c_str());
+    PR_SetEnv(env_to_set_.c_str());
+  }
+
+  void ConnectAndCheck() {
+    // This is a child process, ensure that error messages immediately
+    // propagate or else it will not be visible.
+    ::testing::GTEST_FLAG(throw_on_failure) = true;
+
+    if (version_ == SSL_LIBRARY_VERSION_TLS_1_3) {
+      SetupForZeroRtt();
+      client_->Set0RttEnabled(true);
+      server_->Set0RttEnabled(true);
+      ExpectResumption(RESUME_TICKET);
+      ZeroRttSendReceive(true, true);
+      Handshake();
+      ExpectEarlyDataAccepted(true);
+      CheckConnected();
+      SendReceive();
+    } else {
+      Connect();
+    }
+    CheckKeyLog();
+    _exit(0);
   }
+};
 
-  void CheckKeyLog() {
-    std::ifstream f(keylog_file_path);
+class KeyLogFileTest : public KeyLogFileTestBase {
+ public:
+  KeyLogFileTest() : KeyLogFileTestBase(kKeylogSetEnv) {}
+
+  void CheckKeyLog() override {
+    std::ifstream f(kKeylogFilePath);
     std::map<std::string, size_t> labels;
     std::set<std::string> client_randoms;
     for (std::string line; std::getline(f, line);) {
@@ -63,28 +102,6 @@ class KeyLogFileTest : public TlsConnectGeneric {
       ASSERT_EQ(4U, labels["EXPORTER_SECRET"]);
     }
   }
-
-  void ConnectAndCheck() {
-    // This is a child process, ensure that error messages immediately
-    // propagate or else it will not be visible.
-    ::testing::GTEST_FLAG(throw_on_failure) = true;
-
-    if (version_ == SSL_LIBRARY_VERSION_TLS_1_3) {
-      SetupForZeroRtt();
-      client_->Set0RttEnabled(true);
-      server_->Set0RttEnabled(true);
-      ExpectResumption(RESUME_TICKET);
-      ZeroRttSendReceive(true, true);
-      Handshake();
-      ExpectEarlyDataAccepted(true);
-      CheckConnected();
-      SendReceive();
-    } else {
-      Connect();
-    }
-    CheckKeyLog();
-    _exit(0);
-  }
 };
 
 // Tests are run in a separate process to ensure that NSS is not initialized yet
@@ -111,4 +128,37 @@ INSTANTIATE_TEST_CASE_P(
                        TlsConnectTestBase::kTlsV13));
 #endif
 
+class KeyLogFileUnsetTest : public KeyLogFileTestBase {
+ public:
+  KeyLogFileUnsetTest() : KeyLogFileTestBase(kKeylogBlankEnv) {}
+
+  void CheckKeyLog() override {
+    std::ifstream f(kKeylogFilePath);
+    EXPECT_FALSE(f.good());
+
+    EXPECT_EQ(nullptr, ssl_keylog_iob);
+  }
+};
+
+TEST_P(KeyLogFileUnsetTest, KeyLogFile) {
+  testing::GTEST_FLAG(death_test_style) = "threadsafe";
+
+  ASSERT_EXIT(ConnectAndCheck(), ::testing::ExitedWithCode(0), "");
+}
+
+INSTANTIATE_TEST_CASE_P(
+    KeyLogFileDTLS12, KeyLogFileUnsetTest,
+    ::testing::Combine(TlsConnectTestBase::kTlsVariantsDatagram,
+                       TlsConnectTestBase::kTlsV11V12));
+INSTANTIATE_TEST_CASE_P(
+    KeyLogFileTLS12, KeyLogFileUnsetTest,
+    ::testing::Combine(TlsConnectTestBase::kTlsVariantsStream,
+                       TlsConnectTestBase::kTlsV10ToV12));
+#ifndef NSS_DISABLE_TLS_1_3
+INSTANTIATE_TEST_CASE_P(
+    KeyLogFileTLS13, KeyLogFileUnsetTest,
+    ::testing::Combine(TlsConnectTestBase::kTlsVariantsStream,
+                       TlsConnectTestBase::kTlsV13));
+#endif
+
 }  // namespace nss_test