Properly detect MinTTY when running GHCi on Windows

Before, we detecting the presence of MinTTY on Windows in a very
imprecise way: by checking if the `_` environment variable was set. Not
only is this easy to circumvent, but it also yields false positives on
terminals like ConEmu.

This changes the test to use the `GetFileInformationByHandleEx` function
instead, which provides a far more accurate check for MinTTY's presence.
I've tested this on PowerShell, MSYS2, Cygwin, ConEmu, and Git Bash, and
it does the right thing on each one.

Fixes #12958.

Test Plan: Run GHCi on many different Windows and MinTTY consoles

Reviewers: erikd, Phyx, austin, bgamari

Reviewed By: Phyx, bgamari

Subscribers: thomie, #ghc_windows_task_force

Differential Revision: https://phabricator.haskell.org/D2878

GHC Trac Issues: #12958

2 files changed, 41 insertions, 0 deletions

diff --git a/driver/utils/isMinTTY.c b/driver/utils/isMinTTY.c
new file mode 100644
index 0000000000..3b3ae27bf3
--- /dev/null
+++ b/driver/utils/isMinTTY.c
@@ -0,0 +1,33 @@
+/*
+ * We need Vista headers to use the GetFileInformationByHandleEx function and
+ * the FILE_NAME_INFO struct.
+ */
+#define WINVER 0x0600
+#define _WIN32_WINNT 0x0600
+
+#include <stdbool.h>
+#include <windows.h>
+#include "isMINTTY.h"
+
+bool isMinTTY() {
+  const HANDLE h = GetStdHandle(STD_ERROR_HANDLE);
+  if (h == NULL || h == INVALID_HANDLE_VALUE) {
+    return false;
+  } else if (GetFileType(h) != FILE_TYPE_PIPE) {
+    return false;
+  }
+
+  const unsigned long bufSize = sizeof(DWORD) + MAX_PATH * sizeof(WCHAR);
+  BYTE buf[bufSize];
+  PFILE_NAME_INFO pfni = (PFILE_NAME_INFO) buf;
+
+  if (!GetFileInformationByHandleEx(h, FileNameInfo, buf, bufSize)) {
+    return false;
+  }
+
+  PWSTR fn = pfni->FileName;
+  fn[pfni->FileNameLength] = L'\0';
+
+  return ((wcsstr(fn, L"\\cygwin-") || wcsstr(fn, L"\\msys-")) &&
+           wcsstr(fn, L"-pty") && wcsstr(fn, L"-master"));
+}
diff --git a/driver/utils/isMinTTY.h b/driver/utils/isMinTTY.h
new file mode 100644
index 0000000000..22e9a473cc
--- /dev/null
+++ b/driver/utils/isMinTTY.h
@@ -0,0 +1,8 @@
+#ifndef ISMINTTY_H
+#define ISMINTTY_H
+
+#include <stdbool.h>
+
+bool isMinTTY();
+
+#endif /* ISMINTTY_H */