Merge branch 'upstream-curl' into update-curl

* upstream-curl:
  curl 2020-03-04 (b8d13668)

1 files changed, 299 insertions, 54 deletions

diff --git a/Utilities/cmcurl/lib/smtp.c b/Utilities/cmcurl/lib/smtp.c
index 4a3462b84b..e1872871dc 100644
--- a/Utilities/cmcurl/lib/smtp.c
+++ b/Utilities/cmcurl/lib/smtp.c
@@ -5,7 +5,7 @@
  *                            | (__| |_| |  _ <| |___
  *                             \___|\___/|_| \_\_____|
  *
- * Copyright (C) 1998 - 2019, Daniel Stenberg, <daniel@haxx.se>, et al.
+ * Copyright (C) 1998 - 2020, Daniel Stenberg, <daniel@haxx.se>, et al.
  *
  * This software is licensed as described in the file COPYING, which
  * you should have received as part of this distribution. The terms
@@ -27,6 +27,9 @@
  * RFC4752 The Kerberos V5 ("GSSAPI") SASL Mechanism
  * RFC4954 SMTP Authentication
  * RFC5321 SMTP protocol
+ * RFC5890 Internationalized Domain Names for Applications (IDNA)
+ * RFC6531 SMTP Extension for Internationalized Email
+ * RFC6532 Internationalized Email Headers
  * RFC6749 OAuth 2.0 Authorization Framework
  * RFC8314 Use of TLS for Email Submission and Access
  * Draft   SMTP URL Interface   <draft-earhart-url-smtp-00.txt>
@@ -95,13 +98,14 @@ static CURLcode smtp_done(struct connectdata *conn, CURLcode status,
 static CURLcode smtp_connect(struct connectdata *conn, bool *done);
 static CURLcode smtp_disconnect(struct connectdata *conn, bool dead);
 static CURLcode smtp_multi_statemach(struct connectdata *conn, bool *done);
-static int smtp_getsock(struct connectdata *conn, curl_socket_t *socks,
-                        int numsocks);
+static int smtp_getsock(struct connectdata *conn, curl_socket_t *socks);
 static CURLcode smtp_doing(struct connectdata *conn, bool *dophase_done);
 static CURLcode smtp_setup_connection(struct connectdata *conn);
 static CURLcode smtp_parse_url_options(struct connectdata *conn);
 static CURLcode smtp_parse_url_path(struct connectdata *conn);
 static CURLcode smtp_parse_custom_request(struct connectdata *conn);
+static CURLcode smtp_parse_address(struct connectdata *conn, const char *fqma,
+                                   char **address, struct hostname *host);
 static CURLcode smtp_perform_auth(struct connectdata *conn, const char *mech,
                                   const char *initresp);
 static CURLcode smtp_continue_auth(struct connectdata *conn, const char *resp);
@@ -359,10 +363,8 @@ static CURLcode smtp_perform_helo(struct connectdata *conn)
  */
 static CURLcode smtp_perform_starttls(struct connectdata *conn)
 {
-  CURLcode result = CURLE_OK;
-
   /* Send the STARTTLS command */
-  result = Curl_pp_sendf(&conn->proto.smtpc.pp, "%s", "STARTTLS");
+  CURLcode result = Curl_pp_sendf(&conn->proto.smtpc.pp, "%s", "STARTTLS");
 
   if(!result)
     state(conn, SMTP_STARTTLS);
@@ -378,11 +380,10 @@ static CURLcode smtp_perform_starttls(struct connectdata *conn)
  */
 static CURLcode smtp_perform_upgrade_tls(struct connectdata *conn)
 {
-  CURLcode result = CURLE_OK;
-  struct smtp_conn *smtpc = &conn->proto.smtpc;
-
   /* Start the SSL connection */
-  result = Curl_ssl_connect_nonblocking(conn, FIRSTSOCKET, &smtpc->ssldone);
+  struct smtp_conn *smtpc = &conn->proto.smtpc;
+  CURLcode result = Curl_ssl_connect_nonblocking(conn, FIRSTSOCKET,
+                                                 &smtpc->ssldone);
 
   if(!result) {
     if(smtpc->state != SMTP_UPGRADETLS)
@@ -485,13 +486,55 @@ static CURLcode smtp_perform_command(struct connectdata *conn)
   struct Curl_easy *data = conn->data;
   struct SMTP *smtp = data->req.protop;
 
-  /* Send the command */
-  if(smtp->rcpt)
-    result = Curl_pp_sendf(&conn->proto.smtpc.pp, "%s %s",
-                           smtp->custom && smtp->custom[0] != '\0' ?
-                           smtp->custom : "VRFY",
-                           smtp->rcpt->data);
+  if(smtp->rcpt) {
+    /* We notify the server we are sending UTF-8 data if a) it supports the
+       SMTPUTF8 extension and b) The mailbox contains UTF-8 charaacters, in
+       either the local address or host name parts. This is regardless of
+       whether the host name is encoded using IDN ACE */
+    bool utf8 = FALSE;
+
+    if((!smtp->custom) || (!smtp->custom[0])) {
+      char *address = NULL;
+      struct hostname host = { NULL, NULL, NULL, NULL };
+
+      /* Parse the mailbox to verify into the local address and host name
+         parts, converting the host name to an IDN A-label if necessary */
+      result = smtp_parse_address(conn, smtp->rcpt->data,
+                                  &address, &host);
+      if(result)
+        return result;
+
+      /* Establish whether we should report SMTPUTF8 to the server for this
+         mailbox as per RFC-6531 sect. 3.1 point 6 */
+      utf8 = (conn->proto.smtpc.utf8_supported) &&
+             ((host.encalloc) || (!Curl_is_ASCII_name(address)) ||
+              (!Curl_is_ASCII_name(host.name)));
+
+      /* Send the VRFY command (Note: The host name part may be absent when the
+         host is a local system) */
+      result = Curl_pp_sendf(&conn->proto.smtpc.pp, "VRFY %s%s%s%s",
+                             address,
+                             host.name ? "@" : "",
+                             host.name ? host.name : "",
+                             utf8 ? " SMTPUTF8" : "");
+
+      Curl_free_idnconverted_hostname(&host);
+      free(address);
+    }
+    else {
+      /* Establish whether we should report that we support SMTPUTF8 for EXPN
+         commands to the server as per RFC-6531 sect. 3.1 point 6 */
+      utf8 = (conn->proto.smtpc.utf8_supported) &&
+             (!strcmp(smtp->custom, "EXPN"));
+
+      /* Send the custom recipient based command such as the EXPN command */
+      result = Curl_pp_sendf(&conn->proto.smtpc.pp, "%s %s%s", smtp->custom,
+                             smtp->rcpt->data,
+                             utf8 ? " SMTPUTF8" : "");
+    }
+  }
   else
+    /* Send the non-recipient based command such as HELP */
     result = Curl_pp_sendf(&conn->proto.smtpc.pp, "%s",
                            smtp->custom && smtp->custom[0] != '\0' ?
                            smtp->custom : "HELP");
@@ -516,22 +559,86 @@ static CURLcode smtp_perform_mail(struct connectdata *conn)
   CURLcode result = CURLE_OK;
   struct Curl_easy *data = conn->data;
 
+  /* We notify the server we are sending UTF-8 data if a) it supports the
+     SMTPUTF8 extension and b) The mailbox contains UTF-8 charaacters, in
+     either the local address or host name parts. This is regardless of
+     whether the host name is encoded using IDN ACE */
+  bool utf8 = FALSE;
+
   /* Calculate the FROM parameter */
-  if(!data->set.str[STRING_MAIL_FROM])
+  if(data->set.str[STRING_MAIL_FROM]) {
+    char *address = NULL;
+    struct hostname host = { NULL, NULL, NULL, NULL };
+
+    /* Parse the FROM mailbox into the local address and host name parts,
+       converting the host name to an IDN A-label if necessary */
+    result = smtp_parse_address(conn, data->set.str[STRING_MAIL_FROM],
+                                &address, &host);
+    if(result)
+      return result;
+
+    /* Establish whether we should report SMTPUTF8 to the server for this
+       mailbox as per RFC-6531 sect. 3.1 point 4 and sect. 3.4 */
+    utf8 = (conn->proto.smtpc.utf8_supported) &&
+           ((host.encalloc) || (!Curl_is_ASCII_name(address)) ||
+            (!Curl_is_ASCII_name(host.name)));
+
+    if(host.name) {
+      from = aprintf("<%s@%s>", address, host.name);
+
+      Curl_free_idnconverted_hostname(&host);
+    }
+    else
+      /* An invalid mailbox was provided but we'll simply let the server worry
+         about that and reply with a 501 error */
+      from = aprintf("<%s>", address);
+
+    free(address);
+  }
+  else
     /* Null reverse-path, RFC-5321, sect. 3.6.3 */
     from = strdup("<>");
-  else if(data->set.str[STRING_MAIL_FROM][0] == '<')
-    from = aprintf("%s", data->set.str[STRING_MAIL_FROM]);
-  else
-    from = aprintf("<%s>", data->set.str[STRING_MAIL_FROM]);
 
   if(!from)
     return CURLE_OUT_OF_MEMORY;
 
   /* Calculate the optional AUTH parameter */
   if(data->set.str[STRING_MAIL_AUTH] && conn->proto.smtpc.sasl.authused) {
-    if(data->set.str[STRING_MAIL_AUTH][0] != '\0')
-      auth = aprintf("%s", data->set.str[STRING_MAIL_AUTH]);
+    if(data->set.str[STRING_MAIL_AUTH][0] != '\0') {
+      char *address = NULL;
+      struct hostname host = { NULL, NULL, NULL, NULL };
+
+      /* Parse the AUTH mailbox into the local address and host name parts,
+         converting the host name to an IDN A-label if necessary */
+      result = smtp_parse_address(conn, data->set.str[STRING_MAIL_AUTH],
+                                  &address, &host);
+      if(result) {
+        free(from);
+        return result;
+      }
+
+      /* Establish whether we should report SMTPUTF8 to the server for this
+         mailbox as per RFC-6531 sect. 3.1 point 4 and sect. 3.4 */
+      if((!utf8) && (conn->proto.smtpc.utf8_supported) &&
+         ((host.encalloc) || (!Curl_is_ASCII_name(address)) ||
+          (!Curl_is_ASCII_name(host.name))))
+        utf8 = TRUE;
+
+      if(host.name) {
+        free(from);
+        from = aprintf("<%s@%s>", address, host.name);
+
+        Curl_free_idnconverted_hostname(&host);
+      }
+      else
+        /* An invalid mailbox was provided but we'll simply let the server
+           worry about it */
+        auth = aprintf("<%s>", address);
+
+      free(address);
+      if(!from)
+        return CURLE_OUT_OF_MEMORY;
+    }
     else
       /* Empty AUTH, RFC-2554, sect. 5 */
       auth = strdup("<>");
@@ -565,6 +672,7 @@ static CURLcode smtp_perform_mail(struct connectdata *conn)
     if(result) {
       free(from);
       free(auth);
+
       return result;
     }
 
@@ -587,19 +695,33 @@ static CURLcode smtp_perform_mail(struct connectdata *conn)
     }
   }
 
+  /* If the mailboxes in the FROM and AUTH parameters don't include a UTF-8
+     based address then quickly scan through the recipient list and check if
+     any there do, as we need to correctly identify our support for SMTPUTF8
+     in the envelope, as per RFC-6531 sect. 3.4 */
+  if(conn->proto.smtpc.utf8_supported && !utf8) {
+    struct SMTP *smtp = data->req.protop;
+    struct curl_slist *rcpt = smtp->rcpt;
+
+    while(rcpt && !utf8) {
+      /* Does the host name contain non-ASCII characters? */
+      if(!Curl_is_ASCII_name(rcpt->data))
+        utf8 = TRUE;
+
+      rcpt = rcpt->next;
+    }
+  }
+
   /* Send the MAIL command */
-  if(!auth && !size)
-    result = Curl_pp_sendf(&conn->proto.smtpc.pp,
-                           "MAIL FROM:%s", from);
-  else if(auth && !size)
-    result = Curl_pp_sendf(&conn->proto.smtpc.pp,
-                           "MAIL FROM:%s AUTH=%s", from, auth);
-  else if(auth && size)
-    result = Curl_pp_sendf(&conn->proto.smtpc.pp,
-                           "MAIL FROM:%s AUTH=%s SIZE=%s", from, auth, size);
-  else
-    result = Curl_pp_sendf(&conn->proto.smtpc.pp,
-                           "MAIL FROM:%s SIZE=%s", from, size);
+  result = Curl_pp_sendf(&conn->proto.smtpc.pp,
+                         "MAIL FROM:%s%s%s%s%s%s",
+                         from,                 /* Mandatory                 */
+                         auth ? " AUTH=" : "", /* Optional on AUTH support  */
+                         auth ? auth : "",     /*                           */
+                         size ? " SIZE=" : "", /* Optional on SIZE support  */
+                         size ? size : "",     /*                           */
+                         utf8 ? " SMTPUTF8"    /* Internationalised mailbox */
+                               : "");          /* included in our envelope  */
 
   free(from);
   free(auth);
@@ -623,14 +745,28 @@ static CURLcode smtp_perform_rcpt_to(struct connectdata *conn)
   CURLcode result = CURLE_OK;
   struct Curl_easy *data = conn->data;
   struct SMTP *smtp = data->req.protop;
+  char *address = NULL;
+  struct hostname host = { NULL, NULL, NULL, NULL };
+
+  /* Parse the recipient mailbox into the local address and host name parts,
+     converting the host name to an IDN A-label if necessary */
+  result = smtp_parse_address(conn, smtp->rcpt->data,
+                              &address, &host);
+  if(result)
+    return result;
 
   /* Send the RCPT TO command */
-  if(smtp->rcpt->data[0] == '<')
-    result = Curl_pp_sendf(&conn->proto.smtpc.pp, "RCPT TO:%s",
-                           smtp->rcpt->data);
+  if(host.name)
+    result = Curl_pp_sendf(&conn->proto.smtpc.pp, "RCPT TO:<%s@%s>", address,
+                           host.name);
   else
-    result = Curl_pp_sendf(&conn->proto.smtpc.pp, "RCPT TO:<%s>",
-                           smtp->rcpt->data);
+    /* An invalid mailbox was provided but we'll simply let the server worry
+       about that and reply with a 501 error */
+    result = Curl_pp_sendf(&conn->proto.smtpc.pp, "RCPT TO:<%s>", address);
+
+  Curl_free_idnconverted_hostname(&host);
+  free(address);
+
   if(!result)
     state(conn, SMTP_RCPT);
 
@@ -645,10 +781,8 @@ static CURLcode smtp_perform_rcpt_to(struct connectdata *conn)
  */
 static CURLcode smtp_perform_quit(struct connectdata *conn)
 {
-  CURLcode result = CURLE_OK;
-
   /* Send the QUIT command */
-  result = Curl_pp_sendf(&conn->proto.smtpc.pp, "%s", "QUIT");
+  CURLcode result = Curl_pp_sendf(&conn->proto.smtpc.pp, "%s", "QUIT");
 
   if(!result)
     state(conn, SMTP_QUIT);
@@ -720,7 +854,7 @@ static CURLcode smtp_state_ehlo_resp(struct connectdata *conn, int smtpcode,
       result = CURLE_REMOTE_ACCESS_DENIED;
     }
   }
-  else {
+  else if(len >= 4) {
     line += 4;
     len -= 4;
 
@@ -732,6 +866,10 @@ static CURLcode smtp_state_ehlo_resp(struct connectdata *conn, int smtpcode,
     else if(len >= 4 && !memcmp(line, "SIZE", 4))
       smtpc->size_supported = TRUE;
 
+    /* Does the server support the UTF-8 capability? */
+    else if(len >= 8 && !memcmp(line, "SMTPUTF8", 8))
+      smtpc->utf8_supported = TRUE;
+
     /* Does the server support authentication? */
     else if(len >= 5 && !memcmp(line, "AUTH ", 5)) {
       smtpc->auth_supported = TRUE;
@@ -791,6 +929,10 @@ static CURLcode smtp_state_ehlo_resp(struct connectdata *conn, int smtpcode,
         result = smtp_perform_authentication(conn);
     }
   }
+  else {
+    failf(data, "Unexpectedly short EHLO response");
+    result = CURLE_WEIRD_SERVER_REPLY;
+  }
 
   return result;
 }
@@ -917,25 +1059,53 @@ static CURLcode smtp_state_rcpt_resp(struct connectdata *conn, int smtpcode,
   CURLcode result = CURLE_OK;
   struct Curl_easy *data = conn->data;
   struct SMTP *smtp = data->req.protop;
+  bool is_smtp_err = FALSE;
+  bool is_smtp_blocking_err = FALSE;
 
   (void)instate; /* no use for this yet */
 
-  if(smtpcode/100 != 2) {
-    failf(data, "RCPT failed: %d", smtpcode);
-    result = CURLE_SEND_ERROR;
+  is_smtp_err = (smtpcode/100 != 2) ? TRUE : FALSE;
+
+  /* If there's multiple RCPT TO to be issued, it's possible to ignore errors
+     and proceed with only the valid addresses. */
+  is_smtp_blocking_err =
+    (is_smtp_err && !data->set.mail_rcpt_allowfails) ? TRUE : FALSE;
+
+  if(is_smtp_err) {
+    /* Remembering the last failure which we can report if all "RCPT TO" have
+       failed and we cannot proceed. */
+    smtp->rcpt_last_error = smtpcode;
+
+    if(is_smtp_blocking_err) {
+      failf(data, "RCPT failed: %d", smtpcode);
+      result = CURLE_SEND_ERROR;
+    }
   }
   else {
+    /* Some RCPT TO commands have succeeded. */
+    smtp->rcpt_had_ok = TRUE;
+  }
+
+  if(!is_smtp_blocking_err) {
     smtp->rcpt = smtp->rcpt->next;
 
     if(smtp->rcpt)
       /* Send the next RCPT TO command */
       result = smtp_perform_rcpt_to(conn);
     else {
-      /* Send the DATA command */
-      result = Curl_pp_sendf(&conn->proto.smtpc.pp, "%s", "DATA");
+      /* We weren't able to issue a successful RCPT TO command while going
+         over recipients (potentially multiple). Sending back last error. */
+      if(!smtp->rcpt_had_ok) {
+        failf(data, "RCPT failed: %d (last error)", smtp->rcpt_last_error);
+        result = CURLE_SEND_ERROR;
+      }
+      else {
+        /* Send the DATA command */
+        result = Curl_pp_sendf(&conn->proto.smtpc.pp, "%s", "DATA");
 
-      if(!result)
-        state(conn, SMTP_DATA);
+        if(!result)
+          state(conn, SMTP_DATA);
+      }
     }
   }
 
@@ -1119,10 +1289,9 @@ static CURLcode smtp_init(struct connectdata *conn)
 }
 
 /* For the SMTP "protocol connect" and "doing" phases only */
-static int smtp_getsock(struct connectdata *conn, curl_socket_t *socks,
-                        int numsocks)
+static int smtp_getsock(struct connectdata *conn, curl_socket_t *socks)
 {
-  return Curl_pp_getsock(&conn->proto.smtpc.pp, socks, numsocks);
+  return Curl_pp_getsock(&conn->proto.smtpc.pp, socks);
 }
 
 /***********************************************************************
@@ -1290,6 +1459,12 @@ static CURLcode smtp_perform(struct connectdata *conn, bool *connected,
   /* Store the first recipient (or NULL if not specified) */
   smtp->rcpt = data->set.mail_rcpt;
 
+  /* Track of whether we've successfully sent at least one RCPT TO command */
+  smtp->rcpt_had_ok = FALSE;
+
+  /* Track of the last error we've received by sending RCPT TO command */
+  smtp->rcpt_last_error = 0;
+
   /* Initial data character is the first character in line: it is implicitly
      preceded by a virtual CRLF. */
   smtp->trailing_crlf = TRUE;
@@ -1540,6 +1715,76 @@ static CURLcode smtp_parse_custom_request(struct connectdata *conn)
   return result;
 }
 
+/***********************************************************************
+ *
+ * smtp_parse_address()
+ *
+ * Parse the fully qualified mailbox address into a local address part and the
+ * host name, converting the host name to an IDN A-label, as per RFC-5890, if
+ * necessary.
+ *
+ * Parameters:
+ *
+ * conn  [in]              - The connection handle.
+ * fqma  [in]              - The fully qualified mailbox address (which may or
+ *                           may not contain UTF-8 characters).
+ * address        [in/out] - A new allocated buffer which holds the local
+ *                           address part of the mailbox. This buffer must be
+ *                           free'ed by the caller.
+ * host           [in/out] - The host name structure that holds the original,
+ *                           and optionally encoded, host name.
+ *                           Curl_free_idnconverted_hostname() must be called
+ *                           once the caller has finished with the structure.
+ *
+ * Returns CURLE_OK on success.
+ *
+ * Notes:
+ *
+ * Should a UTF-8 host name require conversion to IDN ACE and we cannot honor
+ * that convertion then we shall return success. This allow the caller to send
+ * the data to the server as a U-label (as per RFC-6531 sect. 3.2).
+ *
+ * If an mailbox '@' seperator cannot be located then the mailbox is considered
+ * to be either a local mailbox or an invalid mailbox (depending on what the
+ * calling function deems it to be) then the input will simply be returned in
+ * the address part with the host name being NULL.
+ */
+static CURLcode smtp_parse_address(struct connectdata *conn, const char *fqma,
+                                   char **address, struct hostname *host)
+{
+  CURLcode result = CURLE_OK;
+  size_t length;
+
+  /* Duplicate the fully qualified email address so we can manipulate it,
+     ensuring it doesn't contain the delimiters if specified */
+  char *dup = strdup(fqma[0] == '<' ? fqma + 1  : fqma);
+  if(!dup)
+    return CURLE_OUT_OF_MEMORY;
+
+  length = strlen(dup);
+  if(dup[length - 1] == '>')
+    dup[length - 1] = '\0';
+
+  /* Extract the host name from the addresss (if we can) */
+  host->name = strpbrk(dup, "@");
+  if(host->name) {
+    *host->name = '\0';
+    host->name = host->name + 1;
+
+    /* Attempt to convert the host name to IDN ACE */
+    (void) Curl_idnconvert_hostname(conn, host);
+
+    /* If Curl_idnconvert_hostname() fails then we shall attempt to continue
+       and send the host name using UTF-8 rather than as 7-bit ACE (which is
+       our preference) */
+  }
+
+  /* Extract the local address from the mailbox */
+  *address = dup;
+
+  return result;
+}
+
 CURLcode Curl_smtp_escape_eob(struct connectdata *conn, const ssize_t nread)
 {
   /* When sending a SMTP payload we must detect CRLF. sequences making sure