I#105 - Invalid UTF-8 letters in a mail body get lost

Closes https://gitlab.gnome.org/GNOME/evolution-data-server/issues/105

5 files changed, 101 insertions, 32 deletions

diff --git a/docs/reference/camel/camel-docs.sgml.in b/docs/reference/camel/camel-docs.sgml.in
index 717b5606d..f56d42ef6 100644
--- a/docs/reference/camel/camel-docs.sgml.in
+++ b/docs/reference/camel/camel-docs.sgml.in
@@ -297,6 +297,10 @@
     <title>Index of deprecated symbols</title>
     <xi:include href="xml/api-index-deprecated.xml"><xi:fallback /></xi:include>
   </index>
+  <index id="api-index-3-34" role="3.34">
+    <title>Index of new symbols in 3.34</title>
+    <xi:include href="xml/api-index-3.34.xml"><xi:fallback /></xi:include>
+  </index>
   <index id="api-index-3-32" role="3.32">
     <title>Index of new symbols in 3.32</title>
     <xi:include href="xml/api-index-3.32.xml"><xi:fallback /></xi:include>
diff --git a/src/camel/camel-mime-filter-tohtml.c b/src/camel/camel-mime-filter-tohtml.c
index b9fdfd12a..cbdfbb6ad 100644
--- a/src/camel/camel-mime-filter-tohtml.c
+++ b/src/camel/camel-mime-filter-tohtml.c
@@ -164,16 +164,31 @@ exit:
 
 static gchar *
 writeln (CamelMimeFilter *mime_filter,
-         const guchar *in,
-         const guchar *inend,
+         const gchar *in_anycharset,
+         const gchar *inend_char,
          gchar *outptr,
          gchar **outend)
 {
 	CamelMimeFilterToHTMLPrivate *priv;
-	const guchar *inptr = in;
+	const guchar *inptr, *inend, *inbegin;
+	gchar *in_utf8 = NULL;
 
 	priv = CAMEL_MIME_FILTER_TOHTML_GET_PRIVATE (mime_filter);
 
+	if (!g_utf8_validate (in_anycharset, inend_char - in_anycharset, NULL)) {
+		in_utf8 = camel_utf8_make_valid_len (in_anycharset, inend_char - in_anycharset);
+
+		if (!in_utf8)
+			return outptr;
+
+		inptr = (const guchar *) in_utf8;
+		inend = inptr + strlen (in_utf8);
+	} else {
+		inptr = (const guchar *) in_anycharset;
+		inend = (const guchar *) inend_char;
+	}
+
+	inbegin = inptr;
 	while (inptr < inend) {
 		guint32 u;
 
@@ -216,7 +231,7 @@ writeln (CamelMimeFilter *mime_filter,
 			/* falls through */
 		case ' ':
 			if (priv->flags & CAMEL_MIME_FILTER_TOHTML_CONVERT_SPACES
-			    && ((inptr == (in + 1) || (inptr < inend && (*inptr == ' ' || *inptr == '\t'))))) {
+			    && ((inptr == (inbegin + 1) || (inptr < inend && (*inptr == ' ' || *inptr == '\t'))))) {
 				outptr = g_stpcpy (outptr, "&nbsp;");
 				priv->column++;
 				break;
@@ -228,7 +243,7 @@ writeln (CamelMimeFilter *mime_filter,
 				   only if not converting the new-line breaks */
 				if (!(priv->flags & CAMEL_MIME_FILTER_TOHTML_CONVERT_NL))
 					*outptr++ = u;
-			} else if (u >= 20 && u <0x80) {
+			} else if (u >= 0x20 && u < 0x80) {
 				*outptr++ = u;
 			} else {
 				if (priv->flags & CAMEL_MIME_FILTER_TOHTML_ESCAPE_8BIT)
@@ -241,6 +256,8 @@ writeln (CamelMimeFilter *mime_filter,
 		}
 	}
 
+	g_free (in_utf8);
+
 	return outptr;
 }
 
@@ -375,9 +392,8 @@ html_convert (CamelMimeFilter *mime_filter,
 					/* write out anything before the first regex match */
 					outptr = writeln (
 						mime_filter,
-						(const guchar *) start,
-						(const guchar *) start +
-						match.um_so,
+						start,
+						start + match.um_so,
 						outptr, &outend);
 
 					start += match.um_so;
@@ -390,15 +406,13 @@ html_convert (CamelMimeFilter *mime_filter,
 					/* prefix shouldn't need escaping, but let's be safe */
 					outptr = writeln (
 						mime_filter,
-						(const guchar *) match.prefix,
-						(const guchar *) match.prefix +
-						strlen (match.prefix),
+						match.prefix,
+						match.prefix + strlen (match.prefix),
 						outptr, &outend);
 					outptr = writeln (
 						mime_filter,
-						(const guchar *) start,
-						(const guchar *) start +
-						matchlen,
+						start,
+						start + matchlen,
 						outptr, &outend);
 					outptr = append_string_verbatim (
 						mime_filter, "\">",
@@ -407,9 +421,8 @@ html_convert (CamelMimeFilter *mime_filter,
 					/* now write the matched string */
 					outptr = writeln (
 						mime_filter,
-						(const guchar *) start,
-						(const guchar *) start +
-						matchlen,
+						start,
+						start + matchlen,
 						outptr, &outend);
 					priv->column += matchlen;
 					start += matchlen;
@@ -423,8 +436,8 @@ html_convert (CamelMimeFilter *mime_filter,
 					/* nothing matched so write out the remainder of this line buffer */
 					outptr = writeln (
 						mime_filter,
-						(const guchar *) start,
-						(const guchar *) start + len,
+						start,
+						start + len,
 						outptr, &outend);
 					break;
 				}
@@ -432,8 +445,8 @@ html_convert (CamelMimeFilter *mime_filter,
 		} else {
 			outptr = writeln (
 				mime_filter,
-				(const guchar *) start,
-				(const guchar *) inptr,
+				start,
+				inptr,
 				outptr, &outend);
 		}
 
@@ -459,8 +472,8 @@ html_convert (CamelMimeFilter *mime_filter,
 		if (start < inend)
 			outptr = writeln (
 				mime_filter,
-				(const guchar *) start,
-				(const guchar *) inend,
+				start,
+				inend,
 				outptr, &outend);
 
 		while (priv->blockquote_depth > 0) {
diff --git a/src/camel/camel-utf8.c b/src/camel/camel-utf8.c
index fd1ba91f2..bd80f2f1b 100644
--- a/src/camel/camel-utf8.c
+++ b/src/camel/camel-utf8.c
@@ -403,16 +403,65 @@ camel_ucs2_utf8 (const gchar *ptr)
 gchar *
 camel_utf8_make_valid (const gchar *text)
 {
-	gchar *res = g_strdup (text), *p;
+	return camel_utf8_make_valid_len (text, -1);
+}
+
+/**
+ * camel_utf8_make_valid_len:
+ * @text: a text to make valid
+ * @text_len: length of the @text, or -1 if NUL-terminated
+ *
+ * Ensures the returned text will be valid UTF-8 string, with incorrect letters
+ * changed to question marks.
+ *
+ * Returns: (transfer full): Valid UTF-8 string, with replaced incorrect letters.
+ *    Free it with g_free(), when no longer needed.
+ *
+ * Since: 3.34
+ **/
+gchar *
+camel_utf8_make_valid_len (const gchar *text,
+			   gssize text_len)
+{
+	/* almost identical copy of glib's _g_utf8_make_valid() */
+	GString *string;
+	const gchar *remainder, *invalid;
+	gint remaining_bytes, valid_bytes;
+
+	if (text && text_len < 0)
+		text_len = strlen (text);
+
+	if (!text || text_len <= 0 || !*text)
+		return g_strdup (text);
+
+	string = NULL;
+	remainder = (gchar *) text,
+	remaining_bytes = text_len;
+
+	while (remaining_bytes != 0) {
+		if (g_utf8_validate (remainder, remaining_bytes, &invalid))
+			break;
+
+		valid_bytes = invalid - remainder;
+
+		if (!string)
+			string = g_string_sized_new (remaining_bytes);
 
-	if (!res)
-		return res;
+		g_string_append_len (string, remainder, valid_bytes);
+		/* append U+FFFD REPLACEMENT CHARACTER */
+		g_string_append (string, "\357\277\275");
 
-	p = res;
-	while (!g_utf8_validate (p, -1, (const gchar **) &p)) {
-		/* make all invalid characters appear as question marks */
-		*p = '?';
+		remaining_bytes -= valid_bytes + 1;
+		remainder = invalid + 1;
 	}
 
-	return res;
+	if (!string)
+		return g_strndup (text, text_len);
+
+	if (remaining_bytes > 0)
+		g_string_append_len (string, remainder, remaining_bytes);
+
+	g_warn_if_fail (g_utf8_validate (string->str, -1, NULL));
+
+	return g_string_free (string, FALSE);
 }
diff --git a/src/camel/camel-utf8.h b/src/camel/camel-utf8.h
index 48c5e86f5..e600f7292 100644
--- a/src/camel/camel-utf8.h
+++ b/src/camel/camel-utf8.h
@@ -42,6 +42,8 @@ gchar *camel_ucs2_utf8 (const gchar *ptr);
 
 /* make valid utf8 string */
 gchar *camel_utf8_make_valid (const gchar *text);
+gchar *camel_utf8_make_valid_len (const gchar *text,
+				  gssize text_len);
 
 G_END_DECLS
 
diff --git a/src/libedataserver/e-data-server-util.c b/src/libedataserver/e-data-server-util.c
index 4d85234c6..88aa696d4 100644
--- a/src/libedataserver/e-data-server-util.c
+++ b/src/libedataserver/e-data-server-util.c
@@ -650,7 +650,8 @@ e_util_utf8_data_make_valid (const gchar *data,
 	if (string == NULL)
 		return g_strndup ((gchar *) data, data_bytes);
 
-	g_string_append (string, remainder);
+	if (remaining_bytes > 0)
+		g_string_append_len (string, remainder, remaining_bytes);
 
 	g_warn_if_fail (g_utf8_validate (string->str, -1, NULL));