WT-2402 Misaligned structure accesses lead to undefined behavior (#3203)

* Don't specify -std=c11 with Solaris/gcc, it's a strict build and so
we're missing lots of standard function prototypes, like posix_memalign,
strdup and strcasecmp.

Try and reorder the autoconf compiler flag code to do important flags
first (-pthreads, -D GNU_SOURCE), before less-important flags
(--enable-strict and other options flags). I'm assuming -D GNU_SOURCE
would reasonably be needed during options testing.

Don't use uname and look for SunOS, instead, use $host_os and look for
solaris.

* "access" is a C library call, don't use the same name.

Don't hard-code a buffer size.

The "ret" value was never used.

2 files changed, 47 insertions, 43 deletions

diff --git a/build_posix/configure.ac.in b/build_posix/configure.ac.in
index c6258f38f01..952c9ae607d 100644
--- a/build_posix/configure.ac.in
+++ b/build_posix/configure.ac.in
@@ -22,37 +22,12 @@ AC_PROG_CC(cc gcc)
 AC_PROG_CXX(c++ g++)
 AM_PROG_AS(as gas)
 
-# WiredTiger uses anonymous unions to pad structures. It's part of C11, but
-# some compilers require a flag to support them.
-AX_CHECK_COMPILE_FLAG([-std=c11], [AM_CFLAGS="$AM_CFLAGS -std=c11"])
-
-# Configure options.
-AM_OPTIONS
-
 define([AC_LIBTOOL_LANG_CXX_CONFIG], [:])dnl
 define([AC_LIBTOOL_LANG_F77_CONFIG], [:])dnl
+
 LT_PREREQ(2.2.6)
 LT_INIT([pic-only])
 AC_SUBST([LIBTOOL_DEPS])
-
-# If enable-strict is configured, turn on as much error checking as we can for
-# this compiler. Intended for developers, and only works for gcc/clang, but it
-# fills a need.
-if test "$wt_cv_enable_strict" = "yes"; then
-	wt_cv_cc_version="`$CC --version | sed -eq`"
-	case "$wt_cv_cc_version" in
-	*clang*)
-		AM_CLANG_WARNINGS($wt_cv_cc_version);;
-	*cc*|*CC*)					# cc, CC, gcc, GCC
-		AM_GCC_WARNINGS($wt_cv_cc_version);;
-	*)
-		AC_MSG_ERROR(
-		    [--enable-strict does not support "$wt_cv_cc_version".]);;
-	esac
-
-	AM_CFLAGS="$AM_CFLAGS $wt_cv_strict_warnings"
-fi
-
 AM_CONDITIONAL([POSIX_HOST], [true])
 AM_CONDITIONAL([WINDOWS_HOST], [false])
 
@@ -73,6 +48,7 @@ AS_CASE([$host_cpu],
 	[aarch64*], [wt_cv_arm64="yes"],
 	[wt_cv_arm64="no"])
 AM_CONDITIONAL([ARM64_HOST], [test "$wt_cv_arm64" = "yes"])
+AS_CASE([$host_os], [*solaris*], [wt_cv_solaris="yes"], [wt_cv_solaris="no"])
 
 # This is a workaround as part of WT-2459. Currently, clang (v3.7) does not
 # support compiling the ASM code we have to perform the CRC checks on PowerPC.
@@ -86,9 +62,17 @@ if test "$wt_cv_powerpc" = "yes" -a "$CC" != "$CCAS"; then
 fi
 AC_SUBST(AM_LIBTOOLFLAGS)
 
+# WiredTiger uses anonymous unions to pad structures. It's part of C11, but
+# some compilers require -std=c11 to support them. Turn on that flag for any
+# compiler that supports it, except for Solaris, where gcc -std=c11 makes
+# some none-C11 prototypes unavailable.
+if test "$wt_cv_solaris" = "no"; then
+	AX_CHECK_COMPILE_FLAG([-std=c11], [AM_CFLAGS="$AM_CFLAGS -std=c11"])
+fi
+
 if test "$GCC" = "yes"; then
 	# The Solaris gcc compiler gets the additional -pthreads flag.
-	if test "`uname -s`" = "SunOS"; then
+	if test "$wt_cv_solaris" = "yes"; then
 		AM_CFLAGS="$AM_CFLAGS -pthreads"
 	fi
 
@@ -99,11 +83,35 @@ if test "$GCC" = "yes"; then
 	fi
 else
 	# The Solaris native compiler gets the additional -mt flag.
-	if test "`uname -s`" = "SunOS"; then
+	if test "$wt_cv_solaris" = "yes"; then
 		AM_CFLAGS="$AM_CFLAGS -mt"
 	fi
 fi
 
+# Linux requires _GNU_SOURCE to be defined
+AS_CASE([$host_os], [linux*], [AM_CFLAGS="$AM_CFLAGS -D_GNU_SOURCE"])
+
+# If enable-strict is configured, turn on as much error checking as we can for
+# this compiler. Intended for developers, and only works for gcc/clang, but it
+# fills a need.
+if test "$wt_cv_enable_strict" = "yes"; then
+	wt_cv_cc_version="`$CC --version | sed -eq`"
+	case "$wt_cv_cc_version" in
+	*clang*)
+		AM_CLANG_WARNINGS($wt_cv_cc_version);;
+	*cc*|*CC*)					# cc, CC, gcc, GCC
+		AM_GCC_WARNINGS($wt_cv_cc_version);;
+	*)
+		AC_MSG_ERROR(
+		    [--enable-strict does not support "$wt_cv_cc_version".]);;
+	esac
+
+	AM_CFLAGS="$AM_CFLAGS $wt_cv_strict_warnings"
+fi
+
+# Configure options.
+AM_OPTIONS
+
 # Java and Python APIs
 if test "$wt_cv_enable_java" = "yes" -o "$wt_cv_enable_python" = "yes"; then
 	# Only a warning, we need to build release packages without SWIG.
@@ -163,11 +171,6 @@ if test "$ac_cv_sizeof_void_p" != "8" ; then
 fi
 AC_MSG_RESULT(yes)
 
-# Linux requires _GNU_SOURCE to be defined
-case "$host_os" in
-linux*)	AM_CFLAGS="$AM_CFLAGS -D_GNU_SOURCE" ;;
-esac
-
 # Linux requires buffers aligned to 4KB boundaries for O_DIRECT to work.
 BUFFER_ALIGNMENT=0
 if test "$ax_cv_func_posix_memalign_works" = "yes" ; then
diff --git a/test/format/util.c b/test/format/util.c
index 2575130e4bb..b9788f1ac75 100644
--- a/test/format/util.c
+++ b/test/format/util.c
@@ -468,37 +468,38 @@ void *
 alter(void *arg)
 {
 	WT_CONNECTION *conn;
-	WT_DECL_RET;
 	WT_SESSION *session;
 	u_int period;
-	bool access;
+	bool access_value;
 	char buf[32];
 
 	(void)(arg);
 	conn = g.wts_conn;
 
 	/*
-	 * Only alter the access pattern hint.  If we alter the
-	 * cache resident setting we may end up with a setting that
-	 * fills cache and doesn't allow it to be evicted.
+	 * Only alter the access pattern hint.  If we alter the cache resident
+	 * setting we may end up with a setting that fills cache and doesn't
+	 * allow it to be evicted.
 	 */
-	access = false;
+	access_value = false;
+
 	/* Open a session */
 	testutil_check(conn->open_session(conn, NULL, NULL, &session));
 
 	while (!g.workers_finished) {
 		period = mmrand(NULL, 1, 10);
 
-		snprintf(buf, 32, "access_pattern_hint=%s",
-		    access ? "random" : "none");
-		access = !access;
-		if ((ret = session->alter(session, g.uri, buf)) != 0)
+		snprintf(buf, sizeof(buf),
+		    "access_pattern_hint=%s", access_value ? "random" : "none");
+		access_value = !access_value;
+		if (session->alter(session, g.uri, buf) != 0)
 			break;
 		while (period > 0 && !g.workers_finished) {
 			--period;
 			sleep(1);
 		}
 	}
+
 	testutil_check(session->close(session, NULL));
 	return (NULL);
 }