Disable "check unicode returns" under Python 3

Disabled the "unicode returns" check that runs on dialect startup when
running under Python 3, which for many years has occurred in order to test
the current DBAPI's behavior for whether or not it returns Python Unicode
or Py2K strings for the VARCHAR and NVARCHAR datatypes.  The check still
occurs by default under Python 2, however the mechanism to test the
behavior will be removed in SQLAlchemy 2.0 when Python 2 support is also
removed.

This logic was very effective when it was needed, however now that Python 3
is standard, all DBAPIs are expected to return Python 3 strings for
character datatypes.  In the unlikely case that a third party DBAPI does
not support this, the conversion logic within :class:`.String` is still
available and the third party dialect may specify this in its upfront
dialect flags by setting the dialect level flag ``returns_unicode_strings``
to one of :attr:`.String.RETURNS_CONDITIONAL` or
:attr:`.String.RETURNS_BYTES`, both of which will enable Unicode conversion
even under Python 3.

As part of this change, disabling testing of the doctest tutorials under
Python 2.

Fixes: #5315
Change-Id: I1260e894611409d3b7fe1a92bd90c52043bbcf19

2 files changed, 79 insertions, 6 deletions

diff --git a/lib/sqlalchemy/engine/default.py b/lib/sqlalchemy/engine/default.py
index 20f731116..d9b4cdda6 100644
--- a/lib/sqlalchemy/engine/default.py
+++ b/lib/sqlalchemy/engine/default.py
@@ -94,12 +94,12 @@ class DefaultDialect(interfaces.Dialect):
     if util.py3k:
         supports_unicode_statements = True
         supports_unicode_binds = True
-        returns_unicode_strings = True
+        returns_unicode_strings = sqltypes.String.RETURNS_UNICODE
         description_encoding = None
     else:
         supports_unicode_statements = False
         supports_unicode_binds = False
-        returns_unicode_strings = False
+        returns_unicode_strings = sqltypes.String.RETURNS_UNKNOWN
         description_encoding = "use_encoding"
 
     name = "default"
@@ -325,7 +325,14 @@ class DefaultDialect(interfaces.Dialect):
         except NotImplementedError:
             self.default_isolation_level = None
 
-        self.returns_unicode_strings = self._check_unicode_returns(connection)
+        if self.returns_unicode_strings is sqltypes.String.RETURNS_UNKNOWN:
+            if util.py3k:
+                raise exc.InvalidRequestError(
+                    "RETURNS_UNKNOWN is unsupported in Python 3"
+                )
+            self.returns_unicode_strings = self._check_unicode_returns(
+                connection
+            )
 
         if (
             self.description_encoding is not None
@@ -415,9 +422,13 @@ class DefaultDialect(interfaces.Dialect):
         results = {check_unicode(test) for test in tests}
 
         if results.issuperset([True, False]):
-            return "conditional"
+            return sqltypes.String.RETURNS_CONDITIONAL
         else:
-            return results == {True}
+            return (
+                sqltypes.String.RETURNS_UNICODE
+                if results == {True}
+                else sqltypes.String.RETURNS_BYTES
+            )
 
     def _check_unicode_description(self, connection):
         # all DBAPIs on Py2K return cursor.description as encoded
diff --git a/lib/sqlalchemy/sql/sqltypes.py b/lib/sqlalchemy/sql/sqltypes.py
index a65989e93..8684a7922 100644
--- a/lib/sqlalchemy/sql/sqltypes.py
+++ b/lib/sqlalchemy/sql/sqltypes.py
@@ -135,6 +135,67 @@ class String(Concatenable, TypeEngine):
 
     __visit_name__ = "string"
 
+    RETURNS_UNICODE = util.symbol(
+        "RETURNS_UNICODE",
+        """Indicates that the DBAPI returns Python Unicode for VARCHAR,
+        NVARCHAR, and other character-based datatypes in all cases.
+
+        This is the default value for
+        :attr:`.DefaultDialect.returns_unicode_strings` under Python 3.
+
+        .. versionadded:: 1.4
+
+        """,
+    )
+
+    RETURNS_BYTES = util.symbol(
+        "RETURNS_BYTES",
+        """Indicates that the DBAPI returns byte objects under Python 3
+        or non-Unicode string objects under Python 2 for VARCHAR, NVARCHAR,
+        and other character-based datatypes in all cases.
+
+        This may be applied to the
+        :attr:`.DefaultDialect.returns_unicode_strings` attribute.
+
+        .. versionadded:: 1.4
+
+        """,
+    )
+
+    RETURNS_CONDITIONAL = util.symbol(
+        "RETURNS_CONDITIONAL",
+        """Indicates that the DBAPI may return Unicode or bytestrings for
+        VARCHAR, NVARCHAR, and other character-based datatypes, and that
+        SQLAlchemy's default String datatype will need to test on a per-row
+        basis for Unicode or bytes.
+
+        This may be applied to the
+        :attr:`.DefaultDialect.returns_unicode_strings` attribute.
+
+        .. versionadded:: 1.4
+
+        """,
+    )
+
+    RETURNS_UNKNOWN = util.symbol(
+        "RETURNS_UNKNOWN",
+        """Indicates that the dialect should test on first connect what the
+        string-returning behavior of character-based datatypes is.
+
+        This is the default value for DefaultDialect.unicode_returns under
+        Python 2.
+
+        This may be applied to the
+        :attr:`.DefaultDialect.returns_unicode_strings` attribute under
+        Python 2 only.   The value is disallowed under Python 3.
+
+        .. versionadded:: 1.4
+
+        .. deprecated:: 1.4  This value will be removed in SQLAlchemy 2.0.
+
+        """,
+    )
+
     @util.deprecated_params(
         convert_unicode=(
             "1.3",
@@ -293,12 +354,13 @@ class String(Concatenable, TypeEngine):
     def result_processor(self, dialect, coltype):
         wants_unicode = self._expect_unicode or dialect.convert_unicode
         needs_convert = wants_unicode and (
-            dialect.returns_unicode_strings is not True
+            dialect.returns_unicode_strings is not String.RETURNS_UNICODE
             or self._expect_unicode in ("force", "force_nocheck")
         )
         needs_isinstance = (
             needs_convert
             and dialect.returns_unicode_strings
+            in (String.RETURNS_CONDITIONAL, String.RETURNS_UNICODE,)
             and self._expect_unicode != "force_nocheck"
         )
         if needs_convert: