get default before processing value

This ensures the default value is processed like other values. Some
type converters were adjusted to accept values that are already the
correct type.

Use parameter source instead of value to determine if argument was
supplied on the command line during completion.

Add a parameter source for values from prompt.

3 files changed, 75 insertions, 67 deletions

diff --git a/src/click/core.py b/src/click/core.py
index 0f76156..ddd9990 100644
--- a/src/click/core.py
+++ b/src/click/core.py
@@ -152,6 +152,8 @@ class ParameterSource(enum.Enum):
     """Used the default specified by the parameter."""
     DEFAULT_MAP = enum.auto()
     """Used a default provided by :attr:`Context.default_map`."""
+    PROMPT = enum.auto()
+    """Used a prompt to confirm a default or provide a value."""
 
 
 class Context:
@@ -277,10 +279,6 @@ class Context:
         #: Map of parameter names to their parsed values. Parameters
         #: with ``expose_value=False`` are not stored.
         self.params = {}
-        # This tracks the actual param objects that were parsed, even if
-        # they didn't expose a value. Used by completion system to know
-        # what parameters to exclude.
-        self._seen_params = set()
         #: the leftover arguments.
         self.args = []
         #: protected arguments.  These are arguments that are prepended
@@ -738,7 +736,7 @@ class Context:
         :param name: The name of the parameter.
         :rtype: ParameterSource
         """
-        return self._parameter_source[name]
+        return self._parameter_source.get(name)
 
 
 class BaseCommand:
@@ -1283,7 +1281,11 @@ class Command(BaseCommand):
                 if (
                     not isinstance(param, Option)
                     or param.hidden
-                    or (not param.multiple and param in ctx._seen_params)
+                    or (
+                        not param.multiple
+                        and ctx.get_parameter_source(param.name)
+                        is ParameterSource.COMMANDLINE
+                    )
                 ):
                     continue
 
@@ -1926,10 +1928,11 @@ class Parameter:
             value = ctx.lookup_default(self.name)
             source = ParameterSource.DEFAULT_MAP
 
-        if value is not None:
-            ctx.set_parameter_source(self.name, source)
+        if value is None:
+            value = self.get_default(ctx)
+            source = ParameterSource.DEFAULT
 
-        return value
+        return value, source
 
     def type_cast_value(self, ctx, value):
         """Given a value this runs it properly through the type system.
@@ -1975,12 +1978,6 @@ class Parameter:
     def full_process_value(self, ctx, value):
         value = self.process_value(ctx, value)
 
-        if value is None and not ctx.resilient_parsing:
-            value = self.get_default(ctx)
-
-            if value is not None:
-                ctx.set_parameter_source(self.name, ParameterSource.DEFAULT)
-
         if self.required and self.value_is_missing(value):
             raise MissingParameter(ctx=ctx, param=self)
 
@@ -2025,26 +2022,23 @@ class Parameter:
 
     def handle_parse_result(self, ctx, opts, args):
         with augment_usage_errors(ctx, param=self):
-            value = self.consume_value(ctx, opts)
+            value, source = self.consume_value(ctx, opts)
+            ctx.set_parameter_source(self.name, source)
+
             try:
                 value = self.full_process_value(ctx, value)
+
+                if self.callback is not None:
+                    value = self.callback(ctx, self, value)
             except Exception:
                 if not ctx.resilient_parsing:
                     raise
+
                 value = None
-            if self.callback is not None:
-                try:
-                    value = self.callback(ctx, self, value)
-                except Exception:
-                    if not ctx.resilient_parsing:
-                        raise
 
         if self.expose_value:
             ctx.params[self.name] = value
 
-        if value is not None:
-            ctx._seen_params.add(self)
-
         return value, args
 
     def get_help_record(self, ctx):
@@ -2423,11 +2417,20 @@ class Option(Parameter):
                 rv = batch(rv, self.nargs)
         return rv
 
-    def full_process_value(self, ctx, value):
-        if value is None and self.prompt is not None and not ctx.resilient_parsing:
-            return self.prompt_for_value(ctx)
+    def consume_value(self, ctx, opts):
+        value, source = super().consume_value(ctx, opts)
+
+        # The value wasn't set, or used the param's default, prompt if
+        # prompting is enabled.
+        if (
+            source in {None, ParameterSource.DEFAULT}
+            and self.prompt is not None
+            and not ctx.resilient_parsing
+        ):
+            value = self.prompt_for_value(ctx)
+            source = ParameterSource.PROMPT
 
-        return super().full_process_value(ctx, value)
+        return value, source
 
 
 class Argument(Parameter):
diff --git a/src/click/shell_completion.py b/src/click/shell_completion.py
index efefba1..9b10e25 100644
--- a/src/click/shell_completion.py
+++ b/src/click/shell_completion.py
@@ -4,6 +4,7 @@ import re
 from .core import Argument
 from .core import MultiCommand
 from .core import Option
+from .core import ParameterSource
 from .parser import split_arg_string
 from .utils import echo
 
@@ -395,29 +396,27 @@ def get_completion_class(shell):
     return _available_shells.get(shell)
 
 
-def _is_incomplete_argument(values, param):
+def _is_incomplete_argument(ctx, param):
     """Determine if the given parameter is an argument that can still
     accept values.
 
-    :param values: Dict of param names and values parsed from the
-        command line args.
+    :param ctx: Invocation context for the command represented by the
+        parsed complete args.
     :param param: Argument object being checked.
     """
     if not isinstance(param, Argument):
         return False
 
-    value = values[param.name]
-
-    if value is None:
-        return True
-
-    if param.nargs == -1:
-        return True
-
-    if isinstance(value, list) and param.nargs > 1 and len(value) < param.nargs:
-        return True
-
-    return False
+    value = ctx.params[param.name]
+    return (
+        param.nargs == -1
+        or ctx.get_parameter_source(param.name) is not ParameterSource.COMMANDLINE
+        or (
+            param.nargs > 1
+            and isinstance(value, (tuple, list))
+            and len(value) < param.nargs
+        )
+    )
 
 
 def _start_of_option(value):
@@ -523,16 +522,18 @@ def _resolve_incomplete(ctx, args, incomplete):
     if "--" not in args and _start_of_option(incomplete):
         return ctx.command, incomplete
 
+    params = ctx.command.get_params(ctx)
+
     # If the last complete arg is an option name with an incomplete
     # value, the option will provide value completions.
-    for param in ctx.command.get_params(ctx):
+    for param in params:
         if _is_incomplete_option(args, param):
             return param, incomplete
 
     # It's not an option name or value. The first argument without a
     # parsed value will provide value completions.
-    for param in ctx.command.get_params(ctx):
-        if _is_incomplete_argument(ctx.params, param):
+    for param in params:
+        if _is_incomplete_argument(ctx, param):
             return param, incomplete
 
     # There were no unparsed arguments, the command may be a group that
diff --git a/src/click/types.py b/src/click/types.py
index 658f13c..8c886d6 100644
--- a/src/click/types.py
+++ b/src/click/types.py
@@ -181,7 +181,7 @@ class StringParamType(ParamType):
                 else:
                     value = value.decode("utf-8", "replace")
             return value
-        return value
+        return str(value)
 
     def __repr__(self):
         return "STRING"
@@ -255,11 +255,9 @@ class Choice(ParamType):
         if normed_value in normed_choices:
             return normed_choices[normed_value]
 
-        self.fail(
-            f"invalid choice: {value}. (choose from {', '.join(self.choices)})",
-            param,
-            ctx,
-        )
+        one_of = "one of " if len(self.choices) > 1 else ""
+        choices_str = ", ".join(repr(c) for c in self.choices)
+        self.fail(f"{value!r} is not {one_of}{choices_str}.", param, ctx)
 
     def __repr__(self):
         return f"Choice({list(self.choices)})"
@@ -320,14 +318,19 @@ class DateTime(ParamType):
             return None
 
     def convert(self, value, param, ctx):
-        # Exact match
+        if isinstance(value, datetime):
+            return value
+
         for format in self.formats:
-            dtime = self._try_to_convert_date(value, format)
-            if dtime:
-                return dtime
+            converted = self._try_to_convert_date(value, format)
 
+            if converted is not None:
+                return converted
+
+        plural = "s" if len(self.formats) > 1 else ""
+        formats_str = ", ".join(repr(f) for f in self.formats)
         self.fail(
-            f"invalid datetime format: {value}. (choose from {', '.join(self.formats)})"
+            f"{value!r} does not match the format{plural} {formats_str}.", param, ctx
         )
 
     def __repr__(self):
@@ -341,7 +344,7 @@ class _NumberParamTypeBase(ParamType):
         try:
             return self._number_class(value)
         except ValueError:
-            self.fail(f"{value} is not a valid {self.name}", param, ctx)
+            self.fail(f"{value!r} is not a valid {self.name}.", param, ctx)
 
 
 class _NumberRangeBase(_NumberParamTypeBase):
@@ -495,7 +498,7 @@ class BoolParamType(ParamType):
     name = "boolean"
 
     def convert(self, value, param, ctx):
-        if isinstance(value, bool):
+        if value in {False, True}:
             return bool(value)
 
         norm = value.strip().lower()
@@ -506,7 +509,7 @@ class BoolParamType(ParamType):
         if norm in {"0", "false", "f", "no", "n", "off"}:
             return False
 
-        self.fail(f"{value!r} is not a valid boolean value.", param, ctx)
+        self.fail(f"{value!r} is not a valid boolean.", param, ctx)
 
     def __repr__(self):
         return "BOOL"
@@ -518,10 +521,15 @@ class UUIDParameterType(ParamType):
     def convert(self, value, param, ctx):
         import uuid
 
+        if isinstance(value, uuid.UUID):
+            return value
+
+        value = value.strip()
+
         try:
             return uuid.UUID(value)
         except ValueError:
-            self.fail(f"{value} is not a valid UUID value", param, ctx)
+            self.fail(f"{value!r} is not a valid UUID.", param, ctx)
 
     def __repr__(self):
         return "UUID"
@@ -610,11 +618,7 @@ class File(ParamType):
                     ctx.call_on_close(safecall(f.flush))
             return f
         except OSError as e:  # noqa: B014
-            self.fail(
-                f"Could not open file: {filename_to_ui(value)}: {get_strerror(e)}",
-                param,
-                ctx,
-            )
+            self.fail(f"{filename_to_ui(value)!r}: {get_strerror(e)}", param, ctx)
 
     def shell_complete(self, ctx, param, incomplete):
         """Return a special completion marker that tells the completion