rts/linker: Catch archives masquerading as object files

Check the file's header to catch static archive bearing the `.o`
extension, as may happen on Windows after the Clang refactoring.

See #21068

3 files changed, 33 insertions, 2 deletions

diff --git a/rts/Linker.c b/rts/Linker.c
index a98f0bd2cb..88734bd597 100644
--- a/rts/Linker.c
+++ b/rts/Linker.c
@@ -1356,7 +1356,7 @@ preloadObjectFile (pathchar *path)
 
    image = stgMallocBytes(fileSize, "loadObj(image)");
 
-#endif
+#endif /* !defined(darwin_HOST_OS) */
 
    int n;
    n = fread ( image, 1, fileSize, f );
@@ -1401,6 +1401,15 @@ static HsInt loadObj_ (pathchar *path)
        return 1; // success
    }
 
+   if (isArchive(path)) {
+       if (loadArchive_(path)) {
+            return 1; // success
+       } else {
+            IF_DEBUG(linker,
+                        debugBelch("tried and failed to load %" PATH_FMT " as an archive\n", path));
+       }
+   }
+
    ObjectCode *oc = preloadObjectFile(path);
    if (oc == NULL) return 0;
 
diff --git a/rts/LinkerInternals.h b/rts/LinkerInternals.h
index 158f90990e..eb76a979bc 100644
--- a/rts/LinkerInternals.h
+++ b/rts/LinkerInternals.h
@@ -406,6 +406,10 @@ pathchar*
 resolveSymbolAddr (pathchar* buffer, int size,
                    SymbolAddr* symbol, uintptr_t* top);
 
+/* defined in LoadArchive.c */
+bool isArchive (pathchar *path);
+HsInt loadArchive_ (pathchar *path);
+
 /*************************************************
  * Various bits of configuration
  *************************************************/
diff --git a/rts/linker/LoadArchive.c b/rts/linker/LoadArchive.c
index 99e405db22..86fa5cb94b 100644
--- a/rts/linker/LoadArchive.c
+++ b/rts/linker/LoadArchive.c
@@ -241,7 +241,7 @@ lookupGNUArchiveIndex(int gnuFileIndexSize, char **fileName_,
     return true;
 }
 
-static HsInt loadArchive_ (pathchar *path)
+HsInt loadArchive_ (pathchar *path)
 {
     char *image = NULL;
     HsInt retcode = 0;
@@ -632,3 +632,21 @@ HsInt loadArchive (pathchar *path)
    RELEASE_LOCK(&linker_mutex);
    return r;
 }
+
+bool isArchive (pathchar *path)
+{
+    static const char ARCHIVE_HEADER[] = "!<arch>\n";
+    char buffer[10];
+    FILE *f = pathopen(path, WSTR("rb"));
+    if (f == NULL) {
+        return false;
+    }
+
+    size_t ret = fread(buffer, 1, sizeof(buffer), f);
+    if (ret < sizeof(buffer)) {
+        return false;
+    }
+    fclose(f);
+    return strncmp(ARCHIVE_HEADER, buffer, sizeof(ARCHIVE_HEADER)-1) == 0;
+}
+