Add dladdr() for AIX

Although it deviates from the actual prototype of DSO_dsobyaddr(), this
is now ISO C compliant and gcc -Wpedantic accepts the code.

Added DATA segment checking to catch ptrgl virtual addresses. Avoid
memleaks with every AIX/dladdr() call. Removed debug-fprintf()s.
Added test case for DSO_dsobyaddr(), which will eventually call dladdr().
Removed unecessary AIX ifdefs again.

The implementation can only lookup function symbols, no data symbols.
Added PIC-flag to aix*-cc build targets.

As AIX is missing a dladdr() implementation it is currently uncertain our
exit()-handlers can still be called when the application exits. After
dlclose() the whole library might have been unloaded already.

Signed-off-by: Matthias Kraft <makr@gmx.eu>

Reviewed-by: Richard Levitte <levitte@openssl.org>
Reviewed-by: Rich Salz <rsalz@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/5668)

2 files changed, 51 insertions, 4 deletions

diff --git a/test/recipes/90-test_shlibload.t b/test/recipes/90-test_shlibload.t
index aa8d98de29..04d5265890 100644
--- a/test/recipes/90-test_shlibload.t
+++ b/test/recipes/90-test_shlibload.t
@@ -1,5 +1,5 @@
 #! /usr/bin/env perl
-# Copyright 2016 The OpenSSL Project Authors. All Rights Reserved.
+# Copyright 2016-2018 The OpenSSL Project Authors. All Rights Reserved.
 #
 # Licensed under the OpenSSL license (the "License").  You may not use
 # this file except in compliance with the License.  You can obtain a copy
@@ -20,7 +20,7 @@ use configdata;
 
 plan skip_all => "Test only supported in a shared build" if disabled("shared");
 
-plan tests => 3;
+plan tests => 4;
 
 my $libcrypto_idx = $unified_info{rename}->{libcrypto} // "libcrypto";
 my $libssl_idx = $unified_info{rename}->{libssl} // "libssl";
@@ -35,4 +35,6 @@ ok(run(test(["shlibloadtest", "-ssl_first", $libcrypto, $libssl])),
    "running shlibloadtest -ssl_first");
 ok(run(test(["shlibloadtest", "-just_crypto", $libcrypto, $libssl])),
    "running shlibloadtest -just_crypto");
+ok(run(test(["shlibloadtest", "-dso_ref", $libcrypto, $libssl])),
+   "running shlibloadtest -dso_ref");
 
diff --git a/test/shlibloadtest.c b/test/shlibloadtest.c
index 062853fae2..f759a3198c 100644
--- a/test/shlibloadtest.c
+++ b/test/shlibloadtest.c
@@ -15,16 +15,21 @@
 #include <openssl/ossl_typ.h>
 #include "testutil.h"
 
+typedef void DSO;
+
 typedef const SSL_METHOD * (*TLS_method_t)(void);
 typedef SSL_CTX * (*SSL_CTX_new_t)(const SSL_METHOD *meth);
 typedef void (*SSL_CTX_free_t)(SSL_CTX *);
 typedef unsigned long (*ERR_get_error_t)(void);
 typedef unsigned long (*OpenSSL_version_num_t)(void);
+typedef DSO * (*DSO_dsobyaddr_t)(void (*addr)(), int flags);
+typedef int (*DSO_free_t)(DSO *dso);
 
 typedef enum test_types_en {
     CRYPTO_FIRST,
     SSL_FIRST,
-    JUST_CRYPTO
+    JUST_CRYPTO,
+    DSO_REFTEST
 } TEST_TYPE;
 
 static TEST_TYPE test_type;
@@ -102,6 +107,8 @@ static int test_lib(void)
     SSL_CTX_free_t mySSL_CTX_free;
     ERR_get_error_t myERR_get_error;
     OpenSSL_version_num_t myOpenSSL_version_num;
+    DSO_dsobyaddr_t myDSO_dsobyaddr;
+    DSO_free_t myDSO_free;
     int result = 0;
 
     switch (test_type) {
@@ -119,9 +126,13 @@ static int test_lib(void)
                 || !TEST_true(shlib_load(path_crypto, &cryptolib)))
             goto end;
         break;
+    case DSO_REFTEST:
+        if (!TEST_true(shlib_load(path_crypto, &cryptolib)))
+            goto end;
+        break;
     }
 
-    if (test_type != JUST_CRYPTO) {
+    if (test_type != JUST_CRYPTO && test_type != DSO_REFTEST) {
         if (!TEST_true(shlib_sym(ssllib, "TLS_method", &symbols[0].sym))
                 || !TEST_true(shlib_sym(ssllib, "SSL_CTX_new", &symbols[1].sym))
                 || !TEST_true(shlib_sym(ssllib, "SSL_CTX_free", &symbols[2].sym)))
@@ -157,6 +168,34 @@ static int test_lib(void)
                      OPENSSL_VERSION_NUMBER & ~COMPATIBILITY_MASK)
         goto end;
 
+    if (test_type == DSO_REFTEST) {
+# ifdef DSO_DLFCN
+        /*
+         * This is resembling the code used in ossl_init_base() and
+         * OPENSSL_atexit() to block unloading the library after dlclose().
+         * We are not testing this on Windows, because it is done there in a
+         * completely different way. Especially as a call to DSO_dsobyaddr()
+         * will always return an error, because DSO_pathbyaddr() is not
+         * implemented there.
+         */
+        if (!TEST_true(shlib_sym(cryptolib, "DSO_dsobyaddr", &symbols[0].sym))
+                || !TEST_true(shlib_sym(cryptolib, "DSO_free",
+                                        &symbols[1].sym)))
+            goto end;
+
+        myDSO_dsobyaddr = (DSO_dsobyaddr_t)symbols[0].func;
+        myDSO_free = (DSO_free_t)symbols[1].func;
+
+        {
+            DSO *hndl;
+            /* use known symbol from crypto module */
+            if (!TEST_ptr(hndl = DSO_dsobyaddr((void (*)())ERR_get_error, 0)))
+                goto end;
+            DSO_free(hndl);
+        }
+# endif /* DSO_DLFCN */
+    }
+
     switch (test_type) {
     case JUST_CRYPTO:
         if (!TEST_true(shlib_close(cryptolib)))
@@ -172,6 +211,10 @@ static int test_lib(void)
                 || !TEST_true(shlib_close(cryptolib)))
             goto end;
         break;
+    case DSO_REFTEST:
+        if (!TEST_true(shlib_close(cryptolib)))
+            goto end;
+        break;
     }
 
     result = 1;
@@ -191,6 +234,8 @@ int setup_tests(void)
         test_type = SSL_FIRST;
     } else if (strcmp(p, "-just_crypto") == 0) {
         test_type = JUST_CRYPTO;
+    } else if (strcmp(p, "-dso_ref") == 0) {
+        test_type = JUST_CRYPTO;
     } else {
         TEST_error("Unrecognised argument");
         return 0;