I18N: change default on smtp transport, to downconvert-when-neededexim-4.93+fixes

(cherry picked from commit b16c9f8bdcfd4cce490af9e493489e128f5c8f52)

8 files changed, 24 insertions, 8 deletions

diff --git a/doc/doc-docbook/spec.xfpt b/doc/doc-docbook/spec.xfpt
index 1e7fa1fe2..ef77146bd 100644
--- a/doc/doc-docbook/spec.xfpt
+++ b/doc/doc-docbook/spec.xfpt
@@ -25328,13 +25328,17 @@ The &%tls_verify_certificates%& option must also be set.
 If both this option and &%tls_try_verify_hosts%& are unset
 operation is as if this option selected all hosts.
 
-.option utf8_downconvert smtp integer!! unset
+.option utf8_downconvert smtp integer&!! -1
 .cindex utf8 "address downconversion"
 .cindex i18n "utf8 address downconversion"
 If built with internationalization support,
 this option controls conversion of UTF-8 in message addresses
 to a-label form.
-For details see section &<<SECTi18nMTA>>&.
+If, after expansion, the value is 1, 0, or -1 then this value overrides
+any value previously set for the message.  Otherwise, any previously
+set value is used.  To permit use of a previous value,
+set this option to an empty string.
+For details on the values see section &<<SECTi18nMTA>>&.
 
 
 
diff --git a/doc/doc-txt/ChangeLog b/doc/doc-txt/ChangeLog
index e0cc69319..3c915a417 100644
--- a/doc/doc-txt/ChangeLog
+++ b/doc/doc-txt/ChangeLog
@@ -116,6 +116,16 @@ JH/43 Fix possible long line in DSN.  Previously when a very long SMTP error
       response was received it would be used unchecked in a fail-DSN, violating
       standards on line-length limits.  Truncate if needed.
 
+JH/42 Internationalisation: change the default for downconversion in the smtp
+      transport to be "if needed".  Previously it was "as previously set" for
+      the message, which usually meant "if needed" for message-submission but
+      "no" for everything else.  However, MTAs have been seen using SMTPUTF8
+      even when the envelope addresses did not need it, resulting in forwarding
+      failures to non-supporting MTAs.  A downconvert in such cases will be
+      a no-op on the addresses, merely dropping the use of SMTPUTF8 by the
+      transport.  The change does mean that addresses needing conversion will
+      be converted when previously a delivery failure would occur.
+
 
 Exim version 4.93
 -----------------
diff --git a/src/src/transports/smtp.c b/src/src/transports/smtp.c
index 8fd9ff94a..279981b9b 100644
--- a/src/src/transports/smtp.c
+++ b/src/src/transports/smtp.c
@@ -309,7 +309,7 @@ smtp_transport_options_block smtp_transport_option_defaults = {
   .tls_verify_cert_hostnames =	US"*",
 #endif
 #ifdef SUPPORT_I18N
-  .utf8_downconvert =		NULL,
+  .utf8_downconvert =		US"-1",
 #endif
 #ifndef DISABLE_DKIM
  .dkim =
diff --git a/test/confs/4211 b/test/confs/4211
index bcd5a4fd1..c23091d46 100644
--- a/test/confs/4211
+++ b/test/confs/4211
@@ -117,5 +117,6 @@ rmt_smtp:
   driver = smtp
   hosts_require_tls = *
   tls_try_verify_hosts = :
+  utf8_downconvert =	0
 
 # End
diff --git a/test/confs/4221 b/test/confs/4221
index bcd5a4fd1..c23091d46 100644
--- a/test/confs/4221
+++ b/test/confs/4221
@@ -117,5 +117,6 @@ rmt_smtp:
   driver = smtp
   hosts_require_tls = *
   tls_try_verify_hosts = :
+  utf8_downconvert =	0
 
 # End
diff --git a/test/scripts/4200-International/4203 b/test/scripts/4200-International/4203
index 310cbd7ff..914ed917d 100644
--- a/test/scripts/4200-International/4203
+++ b/test/scripts/4200-International/4203
@@ -6,7 +6,7 @@ exim -DSERVER=server -DOPTION="" -bd -oX PORT_D
 ****
 #
 # forwarding fails when target does not support SMTPUTF8
-exim -bs -odi
+exim -DSTRICT=0 -bs -odi
 EHLO client.ffail
 MAIL FROM: <यहलोगहिन्दीक्योंनहींबोलसकतेहैं@japanese.なぜみんな日本語を話してくれないのか.local> SMTPUTF8
 RCPT TO: <userz@test.ex>
diff --git a/test/scripts/4200-International/4204 b/test/scripts/4200-International/4204
index 3063f5ac3..f8efcc33b 100644
--- a/test/scripts/4200-International/4204
+++ b/test/scripts/4200-International/4204
@@ -41,7 +41,7 @@ exim -DSERVER=server -DOPTION="" -bd -oX PORT_D
 ****
 #
 # Recipient verify callout, fail
-exim -bs -odi -DCONTROL="verify=recipient/callout"
+exim -bs -odi -DCONTROL="verify=recipient/callout" -DSTRICT=0
 EHLO client.ffail
 MAIL FROM: <CALLER@spanish.PorquénopuedensimplementehablarenEspañol.local> SMTPUTF8
 RCPT TO: <userS@test.ex>
@@ -49,7 +49,7 @@ QUIT
 ****
 #
 # Recipient+random verify callout, fail
-exim -bs -odi -DCONTROL="verify=recipient/callout=random"
+exim -bs -odi -DCONTROL="verify=recipient/callout=random" -DSTRICT=0
 EHLO client.ffail
 MAIL FROM: <CALLER@vietnamese.TạisaohọkhôngthểchỉnóitiếngViệt.local> SMTPUTF8
 RCPT TO: <userT@test.ex>
diff --git a/test/scripts/4200-International/4206 b/test/scripts/4200-International/4206
index 4c558f405..658f2cb30 100644
--- a/test/scripts/4200-International/4206
+++ b/test/scripts/4200-International/4206
@@ -39,14 +39,14 @@ exim -DSERVER=server -DNOTDAEMON -qqff
 exim -DSERVER=server -DOPTION="" -bd -oX PORT_D
 ****
 # sender verify callout, fail
-exim -bs -odi -DCONTROL="verify=sender/callout"
+exim -bs -odi -DCONTROL="verify=sender/callout" -DSTRICT=0
 EHLO client.sfail
 MAIL FROM: <userA@test.ex> SMTPUTF8
 RCPT TO: <user.यष्टिमधु@test.ex>
 QUIT
 ****
 # sender+random verify callout, fail
-exim -bs -odi -DCONTROL="verify=sender/callout=random"
+exim -bs -odi -DCONTROL="verify=sender/callout=random" -DSTRICT=0
 EHLO client.sfail
 MAIL FROM: <userB.જેઠીમધ@test.ex> SMTPUTF8
 RCPT TO: <user.ქართული@test.ex>