testutil: ensure good treatment of argv on non-Unix platforms

From a Unix point of view, some other platform families have certain
quirks.  Windows command prompt doesn't expand globs into actual file
names, so we must do this.  VMS has some oddity with argv pointer size
that can cause crashes if you're not careful (by copying it to a less
surprising pointer size array).

The fixups already exist and are used in the apps/ code.  However, the
testutil code started using the opt routines from apps/ without
including the non-Unix fixups.  This change fixes that.

For VMS' sake, libtestutil gets an app_malloc() shim, to avoid sucking
in all of apps/apps.c.

Reviewed-by: Paul Dale <paul.dale@oracle.com>
(Merged from https://github.com/openssl/openssl/pull/8381)

2 files changed, 31 insertions, 0 deletions

diff --git a/test/testutil/apps_mem.c b/test/testutil/apps_mem.c
new file mode 100644
index 0000000000..fa60bc6848
--- /dev/null
+++ b/test/testutil/apps_mem.c
@@ -0,0 +1,19 @@
+/*
+ * Copyright 2019 The OpenSSL Project Authors. All Rights Reserved.
+ *
+ * Licensed under the Apache License 2.0 (the "License").  You may not use
+ * this file except in compliance with the License.  You can obtain a copy
+ * in the file LICENSE in the source distribution or at
+ * https://www.openssl.org/source/license.html
+ */
+
+#include "apps.h"
+
+/* shim that avoids sucking in too much from apps/apps.c */
+
+void* app_malloc(int sz, const char *what)
+{
+    void *vp = OPENSSL_malloc(sz);
+
+    return vp;
+}
diff --git a/test/testutil/driver.c b/test/testutil/driver.c
index df62625fc0..10d74e2ea4 100644
--- a/test/testutil/driver.c
+++ b/test/testutil/driver.c
@@ -17,6 +17,8 @@
 #include "internal/nelem.h"
 #include <openssl/bio.h>
 
+#include "platform.h"            /* From libapps */
+
 #ifdef _WIN32
 # define strdup _strdup
 #endif
@@ -132,6 +134,16 @@ int setup_test_framework(int argc, char *argv[])
         CRYPTO_mem_ctrl(CRYPTO_MEM_CHECK_ON);
     }
 #endif
+
+#if defined(OPENSSL_SYS_VMS) && defined(__DECC)
+    argv = copy_argv(&argc, argv);
+#elif defined(_WIN32)
+    /*
+     * Replace argv[] with UTF-8 encoded strings.
+     */
+    win32_utf8argv(&argc, &argv);
+#endif
+
     if (!opt_init(argc, argv, test_get_options()))
         return 0;
     return 1;