file.c: handle long pathnames on Windows

Windows supports pathnames up to 32767 UTF-16 characters, but using
the standard interfaces only up to 260 characters. Wrap the functions
that take filenames on Windows.

Clean up the compatiblity layers some more for reduced #ifdefs.

Signed-off-by: H. Peter Anvin (Intel) <hpa@zytor.com>

2 files changed, 295 insertions, 101 deletions

diff --git a/nasmlib/file.c b/nasmlib/file.c
index 4902bb51..55cfc372 100644
--- a/nasmlib/file.c
+++ b/nasmlib/file.c
@@ -78,15 +78,17 @@ void fwritezero(off_t bytes, FILE *fp)
 {
     size_t blksize;
 
-#ifdef nasm_ftruncate
+#ifdef os_ftruncate
     if (bytes >= BUFSIZ && !ferror(fp) && !feof(fp)) {
 	off_t pos = ftello(fp);
-	if (pos >= 0) {
-            pos += bytes;
-	    if (!fflush(fp) &&
-		!nasm_ftruncate(fileno(fp), pos) &&
-		!fseeko(fp, pos, SEEK_SET))
-		    return;
+	if (pos != (off_t)-1) {
+            off_t end = pos + bytes;
+	    if (!fflush(fp) && !os_ftruncate(fileno(fp), end)) {
+                fseeko(fp, 0, SEEK_END);
+                pos = ftello(fp);
+                if (pos != (off_t)-1)
+                    bytes = end - pos; /* This SHOULD be zero */
+            }
 	}
     }
 #endif
@@ -99,24 +101,89 @@ void fwritezero(off_t bytes, FILE *fp)
     }
 }
 
-FILE *nasm_open_read(const char *filename, enum file_flags flags)
+#ifdef _WIN32
+
+/*
+ * On Windows, we want to use _wfopen(), as fopen() has a much smaller limit
+ * on the path length that it supports. Furthermore, we want to prefix the
+ * path name with \\?\ in order to let the Windows kernel know that
+ * we are not limited to PATH_MAX characters.
+ */
+
+os_filename os_mangle_filename(const char *filename)
 {
-    FILE *f = NULL;
-    bool again = true;
+    mbstate_t ps;
+    size_t wclen;
+    wchar_t *buf;
+    const char *p;
+
+    /* If the filename is already prefixed with \\?\, don't add it again */
+    if ((filename[0] == '\\' || filename[0] == '/') &&
+        (filename[1] == '\\' || filename[1] == '/') &&
+        filename[2] == '?' &&
+        (filename[3] == '\\' || filename[3] == '/'))
+        filename += 4;
 
-#ifdef __GLIBC__
     /*
-     * Try to open this file with memory mapping for speed, unless we are
-     * going to do it "manually" with nasm_map_file()
+     * Note: mbsrtowcs() return (size_t)-1 on error, otherwise
+     * the length of the string *without* final NUL in wchar_t
+     * units. Thus we add 1 for the final NUL; the error value
+     * now becomes 0.
      */
-    if (!(flags & NF_FORMAP)) {
-        f = fopen(filename, (flags & NF_TEXT) ? "rtm" : "rbm");
-        again = (!f) && (errno == EINVAL); /* Not supported, try without m */
+    memset(&ps, 0, sizeof ps);  /* Begin in the initial state */
+    p = filename;
+    wclen = mbsrtowcs(NULL, &p, 0, &ps) + 1;
+    if (!wclen)
+        return NULL;
+
+    buf = nasm_malloc((wclen+4) * sizeof(wchar_t));
+    memcpy(buf, L"\\\\?\\", 4*sizeof(wchar_t));
+
+    memset(&ps, 0, sizeof ps);  /* Begin in the initial state */
+    p = filename;
+    if (mbsrtowcs(buf+4, &p, wclen, &ps) + 1 != wclen || p) {
+        nasm_free(buf);
+        return NULL;
     }
+
+    return buf;
+}
+
+#endif
+
+FILE *nasm_open_read(const char *filename, enum file_flags flags)
+{
+    FILE *f = NULL;
+    os_filename osfname;
+
+    osfname = os_mangle_filename(filename);
+    if (osfname) {
+        os_fopenflag fopen_flags[4];
+        memset(fopen_flags, 0, sizeof fopen_flags);
+
+        fopen_flags[0] = 'r';
+        fopen_flags[1] = (flags & NF_TEXT) ? 't' : 'b';
+
+#if defined(__GLIBC__) || defined(__linux__)
+        /*
+         * Try to open this file with memory mapping for speed, unless we are
+         * going to do it "manually" with nasm_map_file()
+         */
+        if (!(flags & NF_FORMAP))
+            fopen_flags[2] = 'm';
 #endif
 
-    if (again)
-        f = fopen(filename, (flags & NF_TEXT) ? "rt" : "rb");
+        while (true) {
+            f = os_fopen(osfname, fopen_flags);
+            if (f || errno != EINVAL || !fopen_flags[2])
+                break;
+
+            /* We got EINVAL but with 'm'; try again without 'm' */
+            fopen_flags[2] = '\0';
+        }
+
+        os_free_filename(osfname);
+    }
 
     if (!f && (flags & NF_FATAL))
         nasm_fatalf(ERR_NOFILE, "unable to open input file: `%s': %s",
@@ -127,9 +194,20 @@ FILE *nasm_open_read(const char *filename, enum file_flags flags)
 
 FILE *nasm_open_write(const char *filename, enum file_flags flags)
 {
-    FILE *f;
+    FILE *f = NULL;
+    os_filename osfname;
+
+    osfname = os_mangle_filename(filename);
+    if (osfname) {
+        os_fopenflag fopen_flags[3];
 
-    f = fopen(filename, (flags & NF_TEXT) ? "wt" : "wb");
+        fopen_flags[0] = 'w';
+        fopen_flags[1] = (flags & NF_TEXT) ? 't' : 'b';
+        fopen_flags[2] = '\0';
+
+        f = os_fopen(osfname, fopen_flags);
+        os_free_filename(osfname);
+    }
 
     if (!f && (flags & NF_FATAL))
         nasm_fatalf(ERR_NOFILE, "unable to open output file: `%s': %s",
@@ -143,43 +221,66 @@ FILE *nasm_open_write(const char *filename, enum file_flags flags)
  */
 bool nasm_file_exists(const char *filename)
 {
-#if defined(HAVE_FACCESSAT) && defined(AT_EACCESS)
-    return faccessat(AT_FDCWD, filename, R_OK, AT_EACCESS) == 0;
-#elif defined(HAVE_ACCESS)
-    return access(filename, R_OK) == 0;
+    os_filename osfname;
+    bool exists;
+
+    osfname = os_mangle_filename(filename);
+    if (!osfname)
+        return false;
+
+#ifdef os_access
+    exists = os_access(osfname, R_OK) == 0;
 #else
     FILE *f;
 
-    f = fopen(filename, "rb");
-    if (f) {
+    f = os_fopen(osfname, "rb");
+    exists = f != NULL;
+    if (f)
         fclose(f);
-        return true;
-    } else {
-        return false;
-    }
 #endif
+
+    os_free_filename(osfname);
+    return exists;
 }
 
 /*
- * Report file size.  This MAY move the file pointer.
+ * Report the file size of an open file.  This MAY move the file pointer.
  */
 off_t nasm_file_size(FILE *f)
 {
-#ifdef HAVE__FILELENGTHI64
-    return _filelengthi64(fileno(f));
-#elif defined(nasm_fstat)
-    nasm_struct_stat st;
+    off_t where, end;
+    os_struct_stat st;
 
-    if (nasm_fstat(fileno(f), &st))
-        return (off_t)-1;
+    if (!os_fstat(fileno(f), &st) && S_ISREG(st.st_mode))
+        return st.st_size;
+
+    /* Do it the hard way... this tests for seekability */
+
+    if (fseeko(f, 0, SEEK_CUR))
+        goto fail;              /* Not seekable, don't even try */
+
+    where = ftello(f);
+    if (where == (off_t)-1)
+        goto fail;
 
-    return st.st_size;
-#else
     if (fseeko(f, 0, SEEK_END))
-        return (off_t)-1;
+        goto fail;
 
-    return ftello(f);
-#endif
+    end = ftello(f);
+    if (end == (off_t)-1)
+        goto fail;
+
+    /*
+     * Move the file pointer back. If this fails, this is probably
+     * not a plain file.
+     */
+    if (fseeko(f, where, SEEK_SET))
+        goto fail;
+
+    return end;
+
+fail:
+    return -1;
 }
 
 /*
@@ -187,26 +288,28 @@ off_t nasm_file_size(FILE *f)
  */
 off_t nasm_file_size_by_path(const char *pathname)
 {
-#ifdef nasm_stat
-    nasm_struct_stat st;
+    os_filename osfname;
+    off_t len = -1;
+    os_struct_stat st;
+    FILE *fp;
+    os_fopenflag fopen_flags[3];
 
-    if (nasm_stat(pathname, &st))
-        return (off_t)-1;
+    osfname = os_mangle_filename(pathname);
 
-    return st.st_size;
-#else
-    FILE *fp;
-    off_t len;
+    if (!os_stat(osfname, &st) && S_ISREG(st.st_mode))
+        len = st.st_size;
 
-    fp = nasm_open_read(pathname, NF_BINARY);
-    if (!fp)
-        return (off_t)-1;
+    fopen_flags[0] = 'r';
+    fopen_flags[1] = 'b';
+    fopen_flags[2] = '\0';
 
-    len = nasm_file_size(fp);
-    fclose(fp);
+    fp = os_fopen(osfname, fopen_flags);
+    if (fp) {
+        len = nasm_file_size(fp);
+        fclose(fp);
+    }
 
     return len;
-#endif
 }
 
 /*
@@ -214,14 +317,20 @@ off_t nasm_file_size_by_path(const char *pathname)
  */
 bool nasm_file_time(time_t *t, const char *pathname)
 {
-#ifdef nasm_stat
-    nasm_struct_stat st;
+#ifdef os_stat
+    os_filename osfname;
+    os_struct_stat st;
+    bool rv = false;
 
-    if (nasm_stat(pathname, &st))
+    osfname = os_mangle_filename(pathname);
+    if (!osfname)
         return false;
 
+    rv = !os_stat(osfname, &st);
     *t = st.st_mtime;
-    return true;
+    os_free_filename(osfname);
+
+    return rv;
 #else
     return false;               /* No idea how to do this on this OS */
 #endif
diff --git a/nasmlib/file.h b/nasmlib/file.h
index 399e3190..4f0420ec 100644
--- a/nasmlib/file.h
+++ b/nasmlib/file.h
@@ -58,70 +58,155 @@
 # include <sys/mman.h>
 #endif
 
-#if !defined(HAVE_ACCESS) && defined(HAVE__ACCESS)
-# define HAVE_ACCESS 1
-# define access _access
-#endif
 #ifndef R_OK
 # define R_OK 4                 /* Classic Unix constant, same on Windows */
 #endif
 
 /* Can we adjust the file size without actually writing all the bytes? */
 #ifdef HAVE__CHSIZE_S
-# define nasm_ftruncate(fd,size) _chsize_s(fd,size)
+# define os_ftruncate(fd,size)	_chsize_s(fd,size)
 #elif defined(HAVE__CHSIZE)
-# define nasm_ftruncate(fd,size) _chsize(fd,size)
+# define os_ftruncate(fd,size)	_chsize(fd,size)
 #elif defined(HAVE_FTRUNCATE)
-# define nasm_ftruncate(fd,size) ftruncate(fd,size)
+# define os_ftruncate(fd,size)	ftruncate(fd,size)
 #endif
 
 /*
- * On Win32/64, stat has a 32-bit file size but _stati64 has a 64-bit file
- * size.  Things get complicated because some of these may be macros,
- * which autoconf won't pick up on as the standard autoconf tests do
- * #undef.
+ * On Windows, we want to use _wfopen(), as fopen() has a much smaller limit
+ * on the path length that it supports. Furthermore, we want to prefix the
+ * path name with \\?\ in order to let the Windows kernel know that
+ * we are not limited to PATH_MAX characters. Thus, we wrap all the functions
+ * which take filenames...
  */
-#ifdef _stati64
-# define HAVE_STRUCT__STATI64 1
-# define HAVE__STATI64 1
-#endif
-#ifdef _fstati64
-# define HAVE__FSTATI64 1
+#ifdef _WIN32
+# include <wchar.h>
+typedef wchar_t *os_filename;
+typedef wchar_t  os_fopenflag;
+
+os_filename os_mangle_filename(const char *filename);
+static inline void os_free_filename(os_filename filename)
+{
+    nasm_free(filename);
+}
+
+# define os_fopen  _wfopen
+# define os_access _waccess
+
+/*
+ * On Win32/64, we have to use the _wstati64() function. Note that
+ * we can't use _wstat64() without depending on a needlessly new
+ * version os MSVCRT.
+ */
+
+typedef struct _stati64 os_struct_stat;
+
+# define os_stat  _wstati64
+# define os_fstat _fstati64
+
+#else  /* not _WIN32 */
+
+typedef const char *os_filename;
+typedef char os_fopenflag;
+
+static inline os_filename os_mangle_filename(const char *filename)
+{
+    return filename;
+}
+static inline void os_free_filename(os_filename filename)
+{
+    (void)filename;             /* Nothing to do */
+}
+
+# define os_fopen  fopen
+
+#if defined(HAVE_FACCESSAT) && defined(AT_EACCESS)
+static inline int os_access(os_filename pathname, int mode)
+{
+    return faccessat(AT_FDCWD, pathname, mode, AT_EACCESS);
+}
+# define os_access os_access
+#elif defined(HAVE_ACCESS)
+# define os_access access
 #endif
 
-#ifdef HAVE_STRUCT__STATI64
-typedef struct _stati64 nasm_struct_stat;
-# ifdef HAVE__STATI64
-#  define nasm_stat _stati64
-# endif
-# ifdef HAVE__FSTATI64
-#  define nasm_fstat _fstati64
-# endif
-#elif defined(HAVE_STRUCT_STAT)
-typedef struct stat nasm_struct_stat;
+#ifdef HAVE_STRUCT_STAT
+typedef struct stat os_struct_stat;
 # ifdef HAVE_STAT
-#  define nasm_stat stat
+#  define os_stat stat
 # endif
 # ifdef HAVE_FSTAT
-#  define nasm_fstat fstat
+#  define os_fstat fstat
 # endif
+#else
+struct dummy_struct_stat {
+    int st_mode;
+    int st_size;
+};
+typedef struct dummy_struct_stat os_struct_stat;
 #endif
 
-#ifndef HAVE_FILENO
-# ifdef fileno                  /* autoconf doesn't always pick up macros */
-#  define HAVE_FILENO 1
-# elif defined(HAVE__FILENO)
-#  define HAVE_FILENO 1
-#  define fileno _fileno
-# endif
+#endif  /* Not _WIN32 */
+
+#ifdef S_ISREG
+/* all good */
+#elif defined(HAVE_S_ISREG)
+/* exists, but not a macro */
+# define S_ISREG S_ISREG
+#elif defined(S_IFMT) && defined(S_IFREG)
+# define S_ISREG(m) (((m) & S_IFMT) == S_IFREG)
+#elif defined(_S_IFMT) && defined(_S_IFREG)
+# define S_ISREG(m) (((m) & _S_IFMT) == _S_IFREG)
+#endif
+
+#ifdef fileno
+/* all good */
+#elif defined(HAVE_FILENO)
+/* exists, but not a macro */
+# define fileno fileno
+#elif defined(_fileno) || defined(HAVE__FILENO)
+# define fileno _fileno
+#endif
+
+#ifndef S_ISREG
+# undef os_stat
+# undef os_fstat
 #endif
 
-/* These functions are utterly useless without fileno() */
-#ifndef HAVE_FILENO
-# undef nasm_fstat
-# undef nasm_ftruncate
+/* Disable these functions if they don't support something we need */
+#ifndef fileno
+# undef os_fstat
+# undef os_ftruncate
 # undef HAVE_MMAP
-# undef HAVE__FILELENGTHI64
+#endif
+
+/*
+ * If we don't have functional versions of these functions,
+ * stub them out so we don't need so many #ifndefs
+ */
+#ifndef os_stat
+static inline int os_stat(os_filename osfname, os_struct_stat *st)
+{
+    (void)osfname;
+    (void)st;
+    return -1;
+}
+#endif
+
+#ifndef os_fstat
+static inline int os_fstat(int fd, os_struct_stat *st)
+{
+    (void)osfname;
+    (void)st;
+    return -1;
+}
+#endif
+
+#ifndef S_ISREG
+static inline bool S_ISREG(int m)
+{
+    (void)m;
+    return false;
+}
 #endif
 
 #endif /* NASMLIB_FILE_H */