Remove MAX_PATH restrictions from RTS, I/O manager and various utilities

Summary:
This shims out fopen and sopen so that they use modern APIs under the hood
along with namespaced paths.

This lifts the MAX_PATH restrictions from Haskell programs and makes the new
limit ~32k.

There are only some slight caveats that have been documented.

Some utilities have not been upgraded such as lndir, since all these things are
different cabal packages I have been forced to copy the source in different places
which is less than ideal. But it's the only way to keep sdist working.

Test Plan: ./validate

Reviewers: hvr, bgamari, erikd, simonmar

Reviewed By: bgamari

Subscribers: rwbarton, thomie, carter

GHC Trac Issues: #10822

Differential Revision: https://phabricator.haskell.org/D4416

13 files changed, 41 insertions, 11 deletions

diff --git a/rts/Hpc.c b/rts/Hpc.c
index 7575e34ce0..9ba9b04b61 100644
--- a/rts/Hpc.c
+++ b/rts/Hpc.c
@@ -13,6 +13,7 @@
 #include <ctype.h>
 #include <string.h>
 #include <assert.h>
+#include <fs_rts.h>
 
 #if defined(HAVE_SYS_TYPES_H)
 #include <sys/types.h>
@@ -233,7 +234,7 @@ startupHpc(void)
     sprintf(tixFilename, "%s.tix", prog_name);
   }
 
-  if (init_open(fopen(tixFilename,"r"))) {
+  if (init_open(__rts_fopen(tixFilename,"r"))) {
     readTix();
   }
 }
@@ -387,7 +388,7 @@ exitHpc(void) {
   // not clober the .tix file.
 
   if (hpc_pid == getpid()) {
-    FILE *f = fopen(tixFilename,"w");
+    FILE *f = __rts_fopen(tixFilename,"w");
     writeTix(f);
   }
 
diff --git a/rts/Linker.c b/rts/Linker.c
index 8f55c296a6..aa6ec7fe7a 100644
--- a/rts/Linker.c
+++ b/rts/Linker.c
@@ -48,6 +48,7 @@
 #include <string.h>
 #include <stdio.h>
 #include <assert.h>
+#include <fs_rts.h>
 
 #if defined(HAVE_SYS_STAT_H)
 #include <sys/stat.h>
@@ -711,7 +712,7 @@ addDLL( pathchar *dll_name )
       strncpy(line, (errmsg+(match[1].rm_so)),match_length);
       line[match_length] = '\0'; // make sure string is null-terminated
       IF_DEBUG(linker, debugBelch ("file name = '%s'\n", line));
-      if ((fp = fopen(line, "r")) == NULL) {
+      if ((fp = __rts_fopen(line, "r")) == NULL) {
          return errmsg; // return original error if open fails
       }
       // try to find a GROUP or INPUT ( ... ) command
diff --git a/rts/PathUtils.h b/rts/PathUtils.h
index 152606a7b0..0b35b214e0 100644
--- a/rts/PathUtils.h
+++ b/rts/PathUtils.h
@@ -14,7 +14,7 @@
 #if defined(mingw32_HOST_OS)
 #define pathcmp wcscmp
 #define pathlen wcslen
-#define pathopen _wfopen
+#define pathopen __rts_fwopen
 #define pathstat _wstat
 #define struct_stat struct _stat
 #define open wopen
diff --git a/rts/ProfHeap.c b/rts/ProfHeap.c
index cb6a13c897..4a7b6d38a8 100644
--- a/rts/ProfHeap.c
+++ b/rts/ProfHeap.c
@@ -23,6 +23,7 @@
 #include "Trace.h"
 #include "sm/GCThread.h"
 
+#include <fs_rts.h>
 #include <string.h>
 
 /* -----------------------------------------------------------------------------
@@ -340,7 +341,7 @@ void initProfiling (void)
     sprintf(hp_filename, "%s.hp", prog);
 
     /* open the log file */
-    if ((hp_file = fopen(hp_filename, "w")) == NULL) {
+    if ((hp_file = __rts_fopen(hp_filename, "w")) == NULL) {
       debugBelch("Can't open profiling report file %s\n",
               hp_filename);
       RtsFlags.ProfFlags.doHeapProfile = 0;
diff --git a/rts/Profiling.c b/rts/Profiling.c
index 803f86befc..9f1a442951 100644
--- a/rts/Profiling.c
+++ b/rts/Profiling.c
@@ -22,6 +22,7 @@
 #include "Printer.h"
 #include "Capability.h"
 
+#include <fs_rts.h>
 #include <string.h>
 
 #if defined(DEBUG)
@@ -264,7 +265,7 @@ initProfilingLogFile(void)
         sprintf(prof_filename, "%s.prof", stem);
 
         /* open the log file */
-        if ((prof_file = fopen(prof_filename, "w")) == NULL) {
+        if ((prof_file = __rts_fopen(prof_filename, "w")) == NULL) {
             debugBelch("Can't open profiling report file %s\n", prof_filename);
             RtsFlags.CcFlags.doCostCentres = 0;
             // Retainer profiling (`-hr` or `-hr<cc> -h<x>`) writes to
@@ -281,7 +282,7 @@ initProfilingLogFile(void)
         sprintf(hp_filename, "%s.hp", stem);
 
         /* open the log file */
-        if ((hp_file = fopen(hp_filename, "w")) == NULL) {
+        if ((hp_file = __rts_fopen(hp_filename, "w")) == NULL) {
             debugBelch("Can't open profiling report file %s\n",
                        hp_filename);
             RtsFlags.ProfFlags.doHeapProfile = 0;
diff --git a/rts/RtsFlags.c b/rts/RtsFlags.c
index b674e9b685..abb980039d 100644
--- a/rts/RtsFlags.c
+++ b/rts/RtsFlags.c
@@ -31,6 +31,8 @@
 #include <sys/types.h>
 #endif
 
+#include <fs_rts.h>
+
 // Flag Structure
 RTS_FLAGS RtsFlags;
 
@@ -1697,7 +1699,7 @@ openStatsFile (char *filename,           // filename, or NULL
         f = NULL; /* NULL means use debugBelch */
     } else {
         if (*filename != '\0') {  /* stats file specified */
-            f = fopen(filename,"w");
+            f = __rts_fopen (filename,"w");
         } else {
             if (filename_fmt == NULL) {
                 errorBelch("Invalid stats filename format (NULL)\n");
@@ -1707,7 +1709,7 @@ openStatsFile (char *filename,           // filename, or NULL
             char stats_filename[STATS_FILENAME_MAXLEN];
             snprintf(stats_filename, STATS_FILENAME_MAXLEN, filename_fmt,
                 prog_name);
-            f = fopen(stats_filename,"w");
+            f = __rts_fopen (stats_filename,"w");
         }
         if (f == NULL) {
             errorBelch("Can't open stats file %s\n", filename);
diff --git a/rts/RtsSymbols.c b/rts/RtsSymbols.c
index d5800fd336..aa95660945 100644
--- a/rts/RtsSymbols.c
+++ b/rts/RtsSymbols.c
@@ -106,6 +106,7 @@
       RTS_WIN64_ONLY(SymI_HasProto(__imp__environ))      \
       RTS_WIN32_ONLY(SymI_HasProto(_imp___iob))          \
       RTS_WIN64_ONLY(SymI_HasProto(__iob_func))          \
+      RTS_WIN64_ONLY(SymI_HasProto(__mingw_vsnwprintf))  \
       /* see Note [Symbols for MinGW's printf] */        \
       SymI_HasProto(_lock_file)                          \
       SymI_HasProto(_unlock_file)
diff --git a/rts/eventlog/EventLogWriter.c b/rts/eventlog/EventLogWriter.c
index d8e5a44192..e6f560fc24 100644
--- a/rts/eventlog/EventLogWriter.c
+++ b/rts/eventlog/EventLogWriter.c
@@ -14,6 +14,7 @@
 
 #include <string.h>
 #include <stdio.h>
+#include <fs_rts.h>
 #if defined(HAVE_SYS_TYPES_H)
 #include <sys/types.h>
 #endif
@@ -71,7 +72,7 @@ initEventLogFileWriter(void)
     stgFree(prog);
 
     /* Open event log file for writing. */
-    if ((event_log_file = fopen(event_log_filename, "wb")) == NULL) {
+    if ((event_log_file = __rts_fopen(event_log_filename, "wb")) == NULL) {
         sysErrorBelch(
             "initEventLogFileWriter: can't open %s", event_log_filename);
         stg_exit(EXIT_FAILURE);
diff --git a/rts/fs_rts.h b/rts/fs_rts.h
new file mode 100644
index 0000000000..12c27ccc8c
--- /dev/null
+++ b/rts/fs_rts.h
@@ -0,0 +1,15 @@
+/* -----------------------------------------------------------------------------
+ *
+ * (c) Tamar Christina 2018
+ *
+ * Hack to get around linkewhole issues on linux. The FS utilities need to be in
+ * a different namespace to allow the linking.
+ *
+ * ---------------------------------------------------------------------------*/
+
+#pragma once
+
+#undef FS_NAMESPACE
+#define FS_NAMESPACE rts
+
+#include "fs.h"
diff --git a/rts/ghc.mk b/rts/ghc.mk
index 761cc43b8f..9d1c2a50f6 100644
--- a/rts/ghc.mk
+++ b/rts/ghc.mk
@@ -332,7 +332,7 @@ endif
 
 STANDARD_OPTS += $(addprefix -I,$(GHC_INCLUDE_DIRS)) -Irts -Irts/dist/build
 # COMPILING_RTS is only used when building Win32 DLL support.
-STANDARD_OPTS += -DCOMPILING_RTS
+STANDARD_OPTS += -DCOMPILING_RTS -DFS_NAMESPACE=rts
 
 # HC_OPTS is included in both .c and .cmm compilations, whereas CC_OPTS is
 # only included in .c compilations.  HC_OPTS included the WAY_* opts, which
diff --git a/rts/linker/LoadArchive.c b/rts/linker/LoadArchive.c
index 3c4bd44a28..7f00edbed8 100644
--- a/rts/linker/LoadArchive.c
+++ b/rts/linker/LoadArchive.c
@@ -22,6 +22,7 @@
 #include <string.h>
 #include <stddef.h>
 #include <ctype.h>
+#include <fs_rts.h>
 
 #define FAIL(...) do {\
    errorBelch("loadArchive: "__VA_ARGS__); \
diff --git a/rts/linker/PEi386.c b/rts/linker/PEi386.c
index 364f7780b2..49aa16d3e3 100644
--- a/rts/linker/PEi386.c
+++ b/rts/linker/PEi386.c
@@ -2025,6 +2025,9 @@ SymbolAddr *lookupSymbol_PEi386(SymbolName *lbl)
         zapTrailingAtSign ( (unsigned char*)lbl );
 #endif
         sym = lookupSymbolInDLLs((unsigned char*)lbl);
+        /* TODO: We should really cache this symbol now that we've loaded it.
+                 The system loader is fast, but not fast enough to keep wasting
+                 cycles like this.  */
         return sym; // might be NULL if not found
     } else {
 #if defined(mingw32_HOST_OS)
diff --git a/rts/rts.cabal.in b/rts/rts.cabal.in
index 67fb9c5d92..0b9a8c4efd 100644
--- a/rts/rts.cabal.in
+++ b/rts/rts.cabal.in
@@ -129,6 +129,8 @@ library
                       rts/Timer.h
                       rts/Types.h
                       rts/Utils.h
+                      rts/fs.h
+                      rts/fs_rts.h
                       rts/prof/CCS.h
                       rts/prof/LDV.h
                       rts/storage/Block.h
@@ -429,6 +431,7 @@ library
                sm/Storage.c
                sm/Sweep.c
                xxhash.c
+               fs.c
                -- I wish we had wildcards..., this would be:
                -- *.c hooks/**/*.c sm/**/*.c eventlog/**/*.c linker/**/*.c
     if os(windows)