Bugzilla Bug 280984: we found that the L_IGNOREUNLOAD flag is not yet

supported on 64-bit AIX, so we have to work around that.  The patch is
contributed by Philip K. Warren (IBM) <pkwarren@gmail.com>. r=wtc.
 Tag: NSPRPUB_PRE_4_2_CLIENT_BRANCH

1 files changed, 15 insertions, 1 deletions

diff --git a/pr/src/linking/prlink.c b/pr/src/linking/prlink.c
index 1b14058a..c80c8c1f 100644
--- a/pr/src/linking/prlink.c
+++ b/pr/src/linking/prlink.c
@@ -1753,6 +1753,7 @@ PR_GetLibraryFilePathname(const char *name, PRFuncPtr addr)
     struct ld_info *info;
     unsigned int info_length = LD_INFO_INCREMENT * sizeof(struct ld_info);
     struct ld_info *infop;
+    int loadflags = L_GETINFO | L_IGNOREUNLOAD;
 
     for (;;) {
         info = PR_Malloc(info_length);
@@ -1760,9 +1761,22 @@ PR_GetLibraryFilePathname(const char *name, PRFuncPtr addr)
             return NULL;
         }
         /* If buffer is too small, loadquery fails with ENOMEM. */
-        if (loadquery(L_GETINFO | L_IGNOREUNLOAD, info, info_length) != -1) {
+        if (loadquery(loadflags, info, info_length) != -1) {
             break;
         }
+        /*
+         * Calling loadquery when compiled for 64-bit with the
+         * L_IGNOREUNLOAD flag can cause an invalid argument error
+         * on AIX 5.1. Detect this error the first time that
+         * loadquery is called, and try calling it again without
+         * this flag set.
+         */
+        if (errno == EINVAL && (loadflags & L_IGNOREUNLOAD)) {
+            loadflags &= ~L_IGNOREUNLOAD;
+            if (loadquery(loadflags, info, info_length) != -1) {
+                break;
+            }
+        }
         PR_Free(info);
         if (errno != ENOMEM) {
             /* should not happen */