unconditionally include unistring code

That simplifies internationalization support, at the cost of
including a version of libunistring, which is used on systems
which do not ship it.

6 files changed, 23 insertions, 385 deletions

diff --git a/configure.ac b/configure.ac
index 8c143e60d3..954829efe6 100644
--- a/configure.ac
+++ b/configure.ac
@@ -256,7 +256,7 @@ AC_C_BIGENDIAN
 
 dnl No fork on MinGW, disable some self-tests until we fix them.
 dnl Check clock_gettime and pthread_mutex_lock in libc (avoid linking to other libs)
-AC_CHECK_FUNCS([fork setitimer inet_ntop inet_pton getrusage getpwuid_r nanosleep daemon getpid clock_gettime iconv localtime fmemopen vasprintf mmap],,)
+AC_CHECK_FUNCS([fork setitimer inet_ntop inet_pton getrusage getpwuid_r nanosleep daemon getpid clock_gettime localtime fmemopen vasprintf mmap],,)
 if test "$ac_cv_func_vasprintf" != "yes";then
   AC_MSG_CHECKING([for va_copy])
   AC_LINK_IFELSE([AC_LANG_PROGRAM([
@@ -325,27 +325,24 @@ if test "$ac_cv_func_clock_gettime" != "yes";then
   gnutls_needs_librt=yes
 fi
 
-AC_LIB_HAVE_LINKFLAGS(unistring,, [#include <uninorm.h>], [u8_normalize(0, 0, 0, 0, 0);])
+AC_ARG_WITH(included-unistring, AS_HELP_STRING([--with-included-unistring],
+                                 [disable linking with system libunistring]),
+            included_unistring="$withval", 
+            included_unistring=no)
 
-ac_have_unicode=no
-if test "$HAVE_LIBUNISTRING" = "yes";then
-  ac_have_unicode=yes
-  ac_have_unistring=yes
+if test "$included_unistring" = yes;then
+  ac_have_unistring=no
 else
-  if test "$ac_cv_func_iconv" != "yes";then
-    AC_LIB_HAVE_LINKFLAGS(iconv,, [#include <iconv.h>], [iconv (0, 0, 0, 0, 0);])
-  fi
+  AC_LIB_HAVE_LINKFLAGS(unistring,, [#include <uninorm.h>], [u8_normalize(0, 0, 0, 0, 0);])
 
-  if test "$ac_cv_func_iconv" = "yes" || test "$HAVE_LIBICONV" = "yes";then
-    ac_have_unicode="partial (iconv)"
+  if test "$HAVE_LIBUNISTRING" = "yes";then
+    included_unistring=no
+    ac_have_unistring=yes
   else
-    ac_have_unicode=no
-  fi
-fi
-
-if test "$ac_have_unicode" != "yes";then
-  if test "$have_win" = "yes";then
-    ac_have_unicode="partial (winapi)"
+      AC_MSG_ERROR([[
+  ***
+  *** Libunistring was not found. To use the included one, use --with-included-unistring
+  ]])
   fi
 fi
 
@@ -991,6 +988,7 @@ AC_MSG_NOTICE([summary of build options:
   Library types:        Shared=${enable_shared}, Static=${enable_static}
   Local libopts:        ${included_libopts}
   Local libtasn1:       ${included_libtasn1}
+  Local unistring:      ${included_unistring}
   Use nettle-mini:      ${mini_nettle}
   Documentation:        ${enable_doc} (manpages: ${enable_manpages})
 ])
@@ -1028,7 +1026,6 @@ if features are disabled)
   Anon auth support:    $ac_enable_anon
   Heartbeat support:    $ac_enable_heartbeat
   IDNA support:         $with_libidn
-  Unicode support:      $ac_have_unicode
   Self checks:          $enable_self_checks
   Non-SuiteB curves:    $enable_non_suiteb
   FIPS140 mode:         $enable_fips
diff --git a/lib/str-unicode.c b/lib/str-unicode.c
index 315482d654..f77b2d443f 100644
--- a/lib/str-unicode.c
+++ b/lib/str-unicode.c
@@ -23,8 +23,6 @@
 #include "gnutls_int.h"
 #include "errors.h"
 #include "str.h"
-
-#if defined(HAVE_LIBUNISTRING)
 #include <uninorm.h>
 #include <unistr.h>
 #include <unictype.h>
@@ -37,8 +35,7 @@
  * @flags: should be zero
  *
  * This function will convert the provided UTF-8 password according
- * to the normalization rules in RFC7613. If GnuTLS is compiled without
- * unicode support this function will return %GNUTLS_E_UNIMPLEMENTED_FEATURE.
+ * to the normalization rules in RFC7613.
  *
  * If the flag %GNUTLS_UTF8_IGNORE_ERRS is specified, any UTF-8 encoding
  * errors will be ignored, and in that case the output will be a copy of the input.
@@ -143,19 +140,3 @@ int gnutls_utf8_password_normalize(const unsigned char *password, unsigned passw
 	gnutls_free(nrmu8);
 	return ret;
 }
-#else
-int gnutls_utf8_password_normalize(const uint8_t *password, unsigned password_len,
-				   gnutls_datum_t *out, unsigned flags)
-{
-	if (!(flags & GNUTLS_UTF8_NORM_INTERNAL))
-		return gnutls_assert_val(GNUTLS_E_UNIMPLEMENTED_FEATURE);
-
-	out->data = gnutls_malloc(password_len+1);
-	if (out->data == NULL)
-		return GNUTLS_E_MEMORY_ERROR;
-	memcpy(out->data, password, password_len);
-	out->data[password_len] = 0;
-	out->size = password_len;
-	return 0;
-}
-#endif
diff --git a/lib/str.h b/lib/str.h
index 54c0652553..7800eec2eb 100644
--- a/lib/str.h
+++ b/lib/str.h
@@ -36,13 +36,12 @@
 # define N_(String) String
 #endif
 
-#define GNUTLS_UTF8_NORM_INTERNAL (1<<16)
 int gnutls_utf8_password_normalize(const uint8_t *password, unsigned password_len,
 				   gnutls_datum_t *out, unsigned flags);
 
 #define _gnutls_utf8_password_normalize(p, plen, out, ignore_errs) \
 	gnutls_utf8_password_normalize((unsigned char*)p, plen, out, \
-		ignore_errs?(GNUTLS_UTF8_NORM_INTERNAL|GNUTLS_UTF8_IGNORE_ERRS):GNUTLS_UTF8_NORM_INTERNAL)
+		ignore_errs?(GNUTLS_UTF8_IGNORE_ERRS):0)
 
 void _gnutls_str_cpy(char *dest, size_t dest_tot_size, const char *src);
 void _gnutls_mem_cpy(char *dest, size_t dest_tot_size, const char *src,
diff --git a/lib/system/iconv.c b/lib/system/iconv.c
index 387b7f83d1..fdc5738d6c 100644
--- a/lib/system/iconv.c
+++ b/lib/system/iconv.c
@@ -1,5 +1,6 @@
 /*
  * Copyright (C) 2010-2016 Free Software Foundation, Inc.
+ * Copyright (C) 2015-2016 Red Hat, Inc.
  *
  * Author: Nikos Mavrogiannopoulos
  *
@@ -29,7 +30,9 @@
 #include <errno.h>
 #include <sys/stat.h>
 #include <sys/types.h>
-#include <c-ctype.h>
+#include <unistr.h>
+#include <uninorm.h>
+#include "num.h"
 
 static void change_u16_endianness(uint8_t *dst, const uint8_t *src, unsigned size, unsigned be)
 {
@@ -58,166 +61,6 @@ static void change_u16_endianness(uint8_t *dst, const uint8_t *src, unsigned siz
 	}
 }
 
-#if defined(_WIN32)
-#include <windows.h>
-#include <winnls.h>
-
-int _gnutls_ucs2_to_utf8(const void *data, size_t size,
-			 gnutls_datum_t * output, unsigned be)
-{
-	int ret;
-	unsigned i;
-	int len = 0, src_len;
-	char *dst = NULL;
-	char *src = NULL;
-	static unsigned flags = 0;
-	static int checked = 0;
-
-	if (checked == 0) {
-		/* Not all windows versions support MB_ERR_INVALID_CHARS */
-		ret =
-		    WideCharToMultiByte(CP_UTF8, MB_ERR_INVALID_CHARS,
-				L"hello", -1, NULL, 0, NULL, NULL);
-		if (ret > 0)
-			flags = MB_ERR_INVALID_CHARS;
-		checked = 1;
-	}
-
-	if (size > 2 && ((uint8_t *) data)[size-1] == 0 && ((uint8_t *) data)[size-2] == 0) {
-		size -= 2;
-	}
-
-	src = gnutls_malloc(size+2);
-	if (src == NULL)
-		return gnutls_assert_val(GNUTLS_E_MEMORY_ERROR);
-
-	/* convert to LE if needed */
-	change_u16_endianness(src, data, size, be);
-
-	src[size] = 0;
-	src[size+1] = 0;
-
-	src_len = wcslen((void*)src);
-
-	ret =
-	    WideCharToMultiByte(CP_UTF8, flags,
-				(void *) src, src_len, NULL, 0,
-				NULL, NULL);
-	if (ret == 0) {
-		_gnutls_debug_log("WideCharToMultiByte: %d\n", (int)GetLastError());
-		ret = gnutls_assert_val(GNUTLS_E_PARSING_ERROR);
-		goto fail;
-	}
-
-	len = ret + 1;
-	dst = gnutls_malloc(len);
-	if (dst == NULL) {
-		ret = gnutls_assert_val(GNUTLS_E_MEMORY_ERROR);
-		goto fail;
-	}
-	dst[0] = 0;
-
-	ret =
-	    WideCharToMultiByte(CP_UTF8, flags,
-				(void *) src, src_len, dst, len-1, NULL,
-				NULL);
-	if (ret == 0) {
-		ret = gnutls_assert_val(GNUTLS_E_PARSING_ERROR);
-		goto fail;
-	}
-	dst[len - 1] = 0;
-
-	output->data = (uint8_t*)dst;
-	output->size = ret;
-
-	ret = 0;
-	goto cleanup;
-
-      fail:
-	gnutls_free(dst);
-
-      cleanup:
-	gnutls_free(src);
-	return ret;
-}
-
-int _gnutls_utf8_to_ucs2(const void *data, size_t size,
-			 gnutls_datum_t * output)
-{
-	int ret;
-	unsigned i;
-	int len = 0;
-	char *dst = NULL;
-	static unsigned flags = MB_PRECOMPOSED;
-	static int checked = 0;
-	uint8_t tmp;
-
-	if (checked == 0) {
-		/* Not all windows versions support MB_ERR_INVALID_CHARS */
-		ret =
-		    MultiByteToWideChar(CP_UTF8, MB_ERR_INVALID_CHARS,
-				"\xff\xff\xff\xff\xff\xff\x00", -1, NULL, 0);
-		if (ret > 0)
-			flags |= MB_ERR_INVALID_CHARS;
-		checked = 1;
-	}
-
-	if (((uint8_t *) data)[size-1] == 0) {
-		size --;
-	}
-
-	ret =
-	    MultiByteToWideChar(CP_UTF8, flags,
-				data, size, NULL, 0);
-	if (ret == 0) {
-		_gnutls_debug_log("WideCharToMultiByte: %d\n", (int)GetLastError());
-		ret = gnutls_assert_val(GNUTLS_E_PARSING_ERROR);
-		goto fail;
-	}
-
-	/* we got the number of characters needed, allocate some extra
-	 * bytes for more complex encodings */
-	len = ret*2;
-	dst = gnutls_calloc(1, len+2);
-	if (dst == NULL) {
-		ret = gnutls_assert_val(GNUTLS_E_MEMORY_ERROR);
-		goto fail;
-	}
-	dst[0] = 0;
-
-	ret =
-	    MultiByteToWideChar(CP_UTF8, flags,
-				data, size, (void*)dst, len/2);
-	if (ret == 0) {
-		ret = gnutls_assert_val(GNUTLS_E_PARSING_ERROR);
-		goto fail;
-	}
-
-	/* convert to BE if needed */
-	change_u16_endianness(dst, dst, len, 1);
-
-	dst[len] = 0;
-	dst[len+1] = 0;
-
-	output->data = (uint8_t*)dst;
-	output->size = len;
-
-	ret = 0;
-	goto cleanup;
-
-      fail:
-	gnutls_free(dst);
-
-      cleanup:
-	return ret;
-}
-
-#elif defined(HAVE_LIBUNISTRING)
-
-#include <unistr.h>
-#include <uninorm.h>
-#include "num.h"
-
 int _gnutls_ucs2_to_utf8(const void *data, size_t size,
 			 gnutls_datum_t * output, unsigned be)
 {
@@ -328,168 +171,3 @@ int _gnutls_utf8_to_ucs2(const void *data, size_t size,
 	return ret;
 }
 
-#elif defined(HAVE_ICONV) || defined(HAVE_LIBICONV)
-
-#include <iconv.h>
-
-int _gnutls_ucs2_to_utf8(const void *data, size_t size,
-			 gnutls_datum_t * output, unsigned be)
-{
-	iconv_t conv;
-	int ret;
-	size_t orig, dstlen = size * 2;
-	char *src = (void *) data;
-	char *dst = NULL, *pdst;
-
-	if (size == 0)
-		return gnutls_assert_val(GNUTLS_E_INVALID_REQUEST);
-
-	if (be) {
-		conv = iconv_open("UTF-8", "UTF-16BE");
-	} else {
-		conv = iconv_open("UTF-8", "UTF-16LE");
-	}
-	if (conv == (iconv_t) - 1)
-		return gnutls_assert_val(GNUTLS_E_MEMORY_ERROR);
-
-	/* Note that dstlen has enough size for every possible input characters.
-	 * (remember the in UTF-16 the characters in data are at most size/2, 
-	 *  and we allocate 4 bytes per character).
-	 */
-	pdst = dst = gnutls_malloc(dstlen + 1);
-	if (dst == NULL) {
-		ret = gnutls_assert_val(GNUTLS_E_MEMORY_ERROR);
-		goto fail;
-	}
-
-	orig = dstlen;
-	ret = iconv(conv, &src, &size, &pdst, &dstlen);
-	if (ret == -1) {
-		ret = gnutls_assert_val(GNUTLS_E_PARSING_ERROR);
-		goto fail;
-	}
-
-	output->data = (void *) dst;
-	output->size = orig - dstlen;
-	output->data[output->size] = 0;
-
-	ret = 0;
-	goto cleanup;
-
-      fail:
-	gnutls_free(dst);
-
-      cleanup:
-	iconv_close(conv);
-
-	return ret;
-}
-
-int _gnutls_utf8_to_ucs2(const void *data, size_t size,
-			 gnutls_datum_t * output)
-{
-	iconv_t conv;
-	int ret;
-	size_t orig, dstlen = size * 4;
-	char *src = (void *) data;
-	uint8_t *dst = NULL;
-	char *pdst;
-
-	if (size == 0)
-		return gnutls_assert_val(GNUTLS_E_INVALID_REQUEST);
-
-	conv = iconv_open("UTF-16BE", "UTF-8");
-	if (conv == (iconv_t) - 1)
-		return gnutls_assert_val(GNUTLS_E_MEMORY_ERROR);
-
-	dst = gnutls_malloc(dstlen+2);
-	pdst = (char*)dst;
-	if (dst == NULL) {
-		ret = gnutls_assert_val(GNUTLS_E_MEMORY_ERROR);
-		goto fail;
-	}
-
-	orig = dstlen;
-	ret = iconv(conv, &src, &size, &pdst, &dstlen);
-	if (ret == -1) {
-		ret = gnutls_assert_val(GNUTLS_E_PARSING_ERROR);
-		goto fail;
-	}
-
-	output->data = (void *) dst;
-	output->size = orig - dstlen;
-	output->data[output->size] = 0;
-	output->data[output->size+1] = 0;
-
-	ret = 0;
-	goto cleanup;
-
-      fail:
-	gnutls_free(dst);
-
-      cleanup:
-	iconv_close(conv);
-
-	return ret;
-}
-
-#else
-
-/* Can convert only english (ASCII) */
-int _gnutls_ucs2_to_utf8(const void *data, size_t size,
-			 gnutls_datum_t * output, unsigned be)
-{
-	unsigned int i, j;
-	char *dst;
-	const char *src = data;
-
-	if (size == 0 || size % 2 != 0)
-		return gnutls_assert_val(GNUTLS_E_INVALID_REQUEST);
-
-	dst = gnutls_malloc(size + 1);
-	if (dst == NULL)
-		return gnutls_assert_val(GNUTLS_E_MEMORY_ERROR);
-
-	for (i = j = 0; i < size; i += 2, j++) {
-		if (src[i] != 0 || !c_isascii(src[i + 1]))
-			return gnutls_assert_val(GNUTLS_E_PARSING_ERROR);
-		if (be)
-			dst[j] = src[i + 1];
-		else
-			dst[j] = src[i];
-	}
-
-	output->data = (void *) dst;
-	output->size = j;
-	output->data[output->size] = 0;
-
-	return 0;
-}
-
-int _gnutls_utf8_to_ucs2(const void *data, size_t size,
-			 gnutls_datum_t * output)
-{
-	unsigned int i, j;
-	char *dst;
-	const char *src = data;
-
-	dst = gnutls_malloc(2*size + 2);
-	if (dst == NULL)
-		return gnutls_assert_val(GNUTLS_E_MEMORY_ERROR);
-
-	for (i = j = 0; i < size; i += 2, j++) {
-		if (!c_isascii(src[j]))
-			return gnutls_assert_val(GNUTLS_E_PARSING_ERROR);
-
-		dst[i] = 0;
-		dst[i+1] = src[j];
-	}
-
-	output->data = (void *) dst;
-	output->size = i;
-	output->data[output->size] = 0;
-	output->data[output->size+1] = 0;
-
-	return 0;
-}
-#endif
diff --git a/tests/conv-utf8.c b/tests/conv-utf8.c
index 205c55ac58..73f4ff3abd 100644
--- a/tests/conv-utf8.c
+++ b/tests/conv-utf8.c
@@ -109,19 +109,15 @@ UTF16_MATCH(check_utf16_ok3, "简体中文", "\x7B\x80\x4F\x53\x4E\x2D\x65\x87")
 UTF16_MATCH(check_utf16_ok4, "Σὲ γνωρίζω ἀπὸ", "\x03\xA3\x1F\x72\x00\x20\x03\xB3\x03\xBD\x03\xC9\x03\xC1\x03\xAF\x03\xB6\x03\xC9\x00\x20\x1F\x00\x03\xC0\x1F\x78");
 
 UTF8_FAIL(check_utf8_fail1, "\xfe\xff\xaa\x80\xff", 5);
-#ifndef _WIN32
 UTF8_FAIL(check_utf8_fail2, "\x64\x00\x62\xf3\x64\x65", 6);
 UTF16_FAIL(check_utf16_fail1, "\xd8\x00\xdb\xff\x00\x63\x00\x04", 8);
-#endif
 
 int main(void)
 {
 	const struct CMUnitTest tests[] = {
 		cmocka_unit_test(check_utf8_fail1),
-#ifndef _WIN32
 		cmocka_unit_test(check_utf8_fail2),
 		cmocka_unit_test(check_utf16_fail1),
-#endif
 		cmocka_unit_test(check_utf8_ok1),
 		cmocka_unit_test(check_utf8_ok2),
 		cmocka_unit_test(check_utf8_ok3),
diff --git a/tests/str-unicode.c b/tests/str-unicode.c
index 5602ec30d5..67c6113a37 100644
--- a/tests/str-unicode.c
+++ b/tests/str-unicode.c
@@ -29,10 +29,7 @@
 #include <stdlib.h>
 #include <string.h>
 #include <gnutls/gnutls.h>
-
-#if defined(HAVE_LIBUNISTRING)
-
-# include <cmocka.h>
+#include <cmocka.h>
 
 #define MATCH_FUNC(fname, password, normalized) \
 static void fname(void **glob_state) \
@@ -92,13 +89,3 @@ int main(void)
 	};
 	return cmocka_run_group_tests(tests, NULL, NULL);
 }
-#else
-int main(void)
-{
-	gnutls_datum_t out;
-	int ret = gnutls_utf8_password_normalize((uint8_t*)"xxx", strlen("xxx"), &out, 0);
-	if (ret != GNUTLS_E_UNIMPLEMENTED_FEATURE)
-		exit(1);
-	exit(77);
-}
-#endif