havelib: Match the bitness when searching for libraries.

* m4/lib-prefix.m4 (AC_LIB_PREPARE_MULTILIB): Define a function
acl_is_expected_elfclass.
* m4/lib-link.m4 (AC_LIB_LINKFLAGS_BODY): When testing whether a library
file exists, in ELF, also test whether it has the ELF class that
corresponds to the host's bitness.

2 files changed, 70 insertions, 10 deletions

diff --git a/m4/lib-link.m4 b/m4/lib-link.m4
index e9158f2204..5efbbffe26 100644
--- a/m4/lib-link.m4
+++ b/m4/lib-link.m4
@@ -1,4 +1,4 @@
-# lib-link.m4 serial 29
+# lib-link.m4 serial 30
 dnl Copyright (C) 2001-2019 Free Software Foundation, Inc.
 dnl This file is free software; the Free Software Foundation
 dnl gives unlimited permission to copy and/or distribute it,
@@ -274,7 +274,7 @@ AC_DEFUN([AC_LIB_LINKFLAGS_BODY],
             dnl The same code as in the loop below:
             dnl First look for a shared library.
             if test -n "$acl_shlibext"; then
-              if test -f "$dir/$libname$shrext"; then
+              if test -f "$dir/$libname$shrext" && acl_is_expected_elfclass < "$dir/$libname$shrext"; then
                 found_dir="$dir"
                 found_so="$dir/$libname$shrext"
               else
@@ -284,14 +284,14 @@ AC_DEFUN([AC_LIB_LINKFLAGS_BODY],
                         | sed -e "s,^$libname$shrext\\\\.,," \
                         | sort -t '.' -n -r -k1,1 -k2,2 -k3,3 -k4,4 -k5,5 \
                         | sed 1q ) 2>/dev/null`
-                  if test -n "$ver" && test -f "$dir/$libname$shrext.$ver"; then
+                  if test -n "$ver" && test -f "$dir/$libname$shrext.$ver" && acl_is_expected_elfclass < "$dir/$libname$shrext.$ver"; then
                     found_dir="$dir"
                     found_so="$dir/$libname$shrext.$ver"
                   fi
                 else
                   eval library_names=\"$acl_library_names_spec\"
                   for f in $library_names; do
-                    if test -f "$dir/$f"; then
+                    if test -f "$dir/$f" && acl_is_expected_elfclass < "$dir/$f"; then
                       found_dir="$dir"
                       found_so="$dir/$f"
                       break
@@ -302,7 +302,7 @@ AC_DEFUN([AC_LIB_LINKFLAGS_BODY],
             fi
             dnl Then look for a static library.
             if test "X$found_dir" = "X"; then
-              if test -f "$dir/$libname.$acl_libext"; then
+              if test -f "$dir/$libname.$acl_libext" && ${AR-ar} -p "$dir/$libname.$acl_libext" | acl_is_expected_elfclass; then
                 found_dir="$dir"
                 found_a="$dir/$libname.$acl_libext"
               fi
@@ -321,7 +321,7 @@ AC_DEFUN([AC_LIB_LINKFLAGS_BODY],
                   dir=`echo "X$x" | sed -e 's/^X-L//'`
                   dnl First look for a shared library.
                   if test -n "$acl_shlibext"; then
-                    if test -f "$dir/$libname$shrext"; then
+                    if test -f "$dir/$libname$shrext" && acl_is_expected_elfclass < "$dir/$libname$shrext"; then
                       found_dir="$dir"
                       found_so="$dir/$libname$shrext"
                     else
@@ -331,14 +331,14 @@ AC_DEFUN([AC_LIB_LINKFLAGS_BODY],
                               | sed -e "s,^$libname$shrext\\\\.,," \
                               | sort -t '.' -n -r -k1,1 -k2,2 -k3,3 -k4,4 -k5,5 \
                               | sed 1q ) 2>/dev/null`
-                        if test -n "$ver" && test -f "$dir/$libname$shrext.$ver"; then
+                        if test -n "$ver" && test -f "$dir/$libname$shrext.$ver" && acl_is_expected_elfclass < "$dir/$libname$shrext.$ver"; then
                           found_dir="$dir"
                           found_so="$dir/$libname$shrext.$ver"
                         fi
                       else
                         eval library_names=\"$acl_library_names_spec\"
                         for f in $library_names; do
-                          if test -f "$dir/$f"; then
+                          if test -f "$dir/$f" && acl_is_expected_elfclass < "$dir/$f"; then
                             found_dir="$dir"
                             found_so="$dir/$f"
                             break
@@ -349,7 +349,7 @@ AC_DEFUN([AC_LIB_LINKFLAGS_BODY],
                   fi
                   dnl Then look for a static library.
                   if test "X$found_dir" = "X"; then
-                    if test -f "$dir/$libname.$acl_libext"; then
+                    if test -f "$dir/$libname.$acl_libext" && ${AR-ar} -p "$dir/$libname.$acl_libext" | acl_is_expected_elfclass; then
                       found_dir="$dir"
                       found_a="$dir/$libname.$acl_libext"
                     fi
diff --git a/m4/lib-prefix.m4 b/m4/lib-prefix.m4
index dbe4e85f18..976c1a7099 100644
--- a/m4/lib-prefix.m4
+++ b/m4/lib-prefix.m4
@@ -1,4 +1,4 @@
-# lib-prefix.m4 serial 15
+# lib-prefix.m4 serial 16
 dnl Copyright (C) 2001-2005, 2008-2019 Free Software Foundation, Inc.
 dnl This file is free software; the Free Software Foundation
 dnl gives unlimited permission to copy and/or distribute it,
@@ -147,6 +147,8 @@ AC_DEFUN([AC_LIB_WITH_FINAL_PREFIX],
 ])
 
 dnl AC_LIB_PREPARE_MULTILIB creates
+dnl - a function acl_is_expected_elfclass, that tests whether standard input
+dn;   has a 32-bit or 64-bit ELF header, depending on the host CPU ABI,
 dnl - a variable acl_libdirstem, containing the basename of the libdir, either
 dnl   "lib" or "lib64" or "lib/64",
 dnl - a variable acl_libdirstem2, as a secondary possible value for
@@ -169,6 +171,64 @@ AC_DEFUN([AC_LIB_PREPARE_MULTILIB],
   AC_REQUIRE([AC_CANONICAL_HOST])
   AC_REQUIRE([gl_HOST_CPU_C_ABI_32BIT])
 
+  AC_CACHE_CHECK([for ELF binary format], [gl_cv_elf],
+    [AC_EGREP_CPP([Extensible Linking Format],
+       [#ifdef __ELF__
+        Extensible Linking Format
+        #endif
+       ],
+       [gl_cv_elf=yes],
+       [gl_cv_elf=no])
+     ])
+  if test $gl_cv_elf; then
+    # Extract the ELF class of a file (5th byte) in decimal.
+    # Cf. https://en.wikipedia.org/wiki/Executable_and_Linkable_Format#File_header
+    if od -A x < /dev/null >/dev/null 2>/dev/null; then
+      # Use POSIX od.
+      func_elfclass ()
+      {
+        od -A n -t d1 -j 4 -N 1
+      }
+    else
+      # Use BSD hexdump.
+      func_elfclass ()
+      {
+        dd bs=1 count=1 skip=4 2>/dev/null | hexdump -e '1/1 "%3d "'
+        echo
+      }
+    fi
+changequote(,)dnl
+    case $HOST_CPU_C_ABI_32BIT in
+      yes)
+        # 32-bit ABI.
+        acl_is_expected_elfclass ()
+        {
+          test "`func_elfclass | sed -e 's/[ 	]//g'`" = 1
+        }
+        ;;
+      no)
+        # 64-bit ABI.
+        acl_is_expected_elfclass ()
+        {
+          test "`func_elfclass | sed -e 's/[ 	]//g'`" = 2
+        }
+        ;;
+      *)
+        # Unknown.
+        acl_is_expected_elfclass ()
+        {
+          :
+        }
+        ;;
+    esac
+changequote([,])dnl
+  else
+    acl_is_expected_elfclass ()
+    {
+      :
+    }
+  fi
+
   dnl Allow the user to override the result by setting acl_cv_libdirstems.
   AC_CACHE_CHECK([for the common suffixes of directories in the library search path],
     [acl_cv_libdirstems],