diff options
22 files changed, 177 insertions, 76 deletions
diff --git a/doc/build/changelog/changelog_14.rst b/doc/build/changelog/changelog_14.rst index f7aa344e8..b33f4c45b 100644 --- a/doc/build/changelog/changelog_14.rst +++ b/doc/build/changelog/changelog_14.rst @@ -14,10 +14,69 @@ This document details individual issue-level changes made throughout .. changelog:: - :version: 1.4.44 + :version: 1.4.45 :include_notes_from: unreleased_14 .. changelog:: + :version: 1.4.44 + :released: November 12, 2022 + + .. change:: + :tags: bug, sql + :tickets: 8790 + + Fixed critical memory issue identified in cache key generation, where for + very large and complex ORM statements that make use of lots of ORM aliases + with subqueries, cache key generation could produce excessively large keys + that were orders of magnitude bigger than the statement itself. Much thanks + to Rollo Konig Brock for their very patient, long term help in finally + identifying this issue. + + .. change:: + :tags: bug, postgresql, mssql + :tickets: 8770 + + For the PostgreSQL and SQL Server dialects only, adjusted the compiler so + that when rendering column expressions in the RETURNING clause, the "non + anon" label that's used in SELECT statements is suggested for SQL + expression elements that generate a label; the primary example is a SQL + function that may be emitting as part of the column's type, where the label + name should match the column's name by default. This restores a not-well + defined behavior that had changed in version 1.4.21 due to :ticket:`6718`, + :ticket:`6710`. The Oracle dialect has a different RETURNING implementation + and was not affected by this issue. Version 2.0 features an across the + board change for its widely expanded support of RETURNING on other + backends. + + + .. change:: + :tags: bug, oracle + + Fixed issue in the Oracle dialect where an INSERT statement that used + ``insert(some_table).values(...).returning(some_table)`` against a full + :class:`.Table` object at once would fail to execute, raising an exception. + + .. change:: + :tags: bug, tests + :tickets: 8793 + + Fixed issue where the ``--disable-asyncio`` parameter to the test suite + would fail to not actually run greenlet tests and would also not prevent + the suite from using a "wrapping" greenlet for the whole suite. This + parameter now ensures that no greenlet or asyncio use will occur within the + entire run when set. + + .. change:: + :tags: bug, tests + + Adjusted the test suite which tests the Mypy plugin to accommodate for + changes in Mypy 0.990 regarding how it handles message output, which affect + how sys.path is interpreted when determining if notes and errors should be + printed for particular files. The change broke the test suite as the files + within the test directory itself no longer produced messaging when run + under the mypy API. + +.. changelog:: :version: 1.4.43 :released: November 4, 2022 diff --git a/doc/build/changelog/unreleased_14/8770.rst b/doc/build/changelog/unreleased_14/8770.rst deleted file mode 100644 index 8968b0361..000000000 --- a/doc/build/changelog/unreleased_14/8770.rst +++ /dev/null @@ -1,23 +0,0 @@ -.. change:: - :tags: bug, postgresql, mssql - :tickets: 8770 - - For the PostgreSQL and SQL Server dialects only, adjusted the compiler so - that when rendering column expressions in the RETURNING clause, the "non - anon" label that's used in SELECT statements is suggested for SQL - expression elements that generate a label; the primary example is a SQL - function that may be emitting as part of the column's type, where the label - name should match the column's name by default. This restores a not-well - defined behavior that had changed in version 1.4.21 due to :ticket:`6718`, - :ticket:`6710`. The Oracle dialect has a different RETURNING implementation - and was not affected by this issue. Version 2.0 features an across the - board change for its widely expanded support of RETURNING on other - backends. - - -.. change:: - :tags: bug, oracle - - Fixed issue in the Oracle dialect where an INSERT statement that used - ``insert(some_table).values(...).returning(some_table)`` against a full - :class:`.Table` object at once would fail to execute, raising an exception. diff --git a/doc/build/changelog/unreleased_14/8790.rst b/doc/build/changelog/unreleased_14/8790.rst deleted file mode 100644 index a3214801c..000000000 --- a/doc/build/changelog/unreleased_14/8790.rst +++ /dev/null @@ -1,10 +0,0 @@ -.. change:: - :tags: bug, sql - :tickets: 8790 - - Fixed critical memory issue identified in cache key generation, where for - very large and complex ORM statements that make use of lots of ORM aliases - with subqueries, cache key generation could produce excessively large keys - that were orders of magnitude bigger than the statement itself. Much thanks - to Rollo Konig Brock for their very patient, long term help in finally - identifying this issue. diff --git a/doc/build/changelog/unreleased_14/8793.rst b/doc/build/changelog/unreleased_14/8793.rst deleted file mode 100644 index 36f1003cc..000000000 --- a/doc/build/changelog/unreleased_14/8793.rst +++ /dev/null @@ -1,9 +0,0 @@ -.. change:: - :tags: bug, tests - :tickets: 8793 - - Fixed issue where the ``--disable-asyncio`` parameter to the test suite - would fail to not actually run greenlet tests and would also not prevent - the suite from using a "wrapping" greenlet for the whole suite. This - parameter now ensures that no greenlet or asyncio use will occur within the - entire run when set. diff --git a/doc/build/changelog/unreleased_14/8800.rst b/doc/build/changelog/unreleased_14/8800.rst new file mode 100644 index 000000000..8a42975df --- /dev/null +++ b/doc/build/changelog/unreleased_14/8800.rst @@ -0,0 +1,8 @@ +.. change:: + :tags: usecase, sql + :tickets: 8800 + + An informative re-raise is now thrown in the case where any "literal + bindparam" render operation fails, indicating the value itself and + the datatype in use, to assist in debugging when literal params + are being rendered in a statement. diff --git a/doc/build/changelog/unreleased_14/mypy_fixes.rst b/doc/build/changelog/unreleased_14/mypy_fixes.rst deleted file mode 100644 index 32e4f1465..000000000 --- a/doc/build/changelog/unreleased_14/mypy_fixes.rst +++ /dev/null @@ -1,9 +0,0 @@ -.. change:: - :tags: bug, tests - - Adjusted the test suite which tests the Mypy plugin to accommodate for - changes in Mypy 0.990 regarding how it handles message output, which affect - how sys.path is interpreted when determining if notes and errors should be - printed for particular files. The change broke the test suite as the files - within the test directory itself no longer produced messaging when run - under the mypy API. diff --git a/doc/build/changelog/unreleased_20/8776.rst b/doc/build/changelog/unreleased_20/8776.rst new file mode 100644 index 000000000..fa4156eae --- /dev/null +++ b/doc/build/changelog/unreleased_20/8776.rst @@ -0,0 +1,7 @@ +.. change:: + :tags: bug, orm + :tickets: 8776 + + Fixed issue where passing a callbale function returning an iterable + of column elements to :paramref:`_orm.relationship.order_by` was + flagged as an error in type checkers. diff --git a/doc/build/tutorial/dbapi_transactions.rst b/doc/build/tutorial/dbapi_transactions.rst index 00178936b..d40aaf5b8 100644 --- a/doc/build/tutorial/dbapi_transactions.rst +++ b/doc/build/tutorial/dbapi_transactions.rst @@ -479,7 +479,7 @@ the block with a "commit as you go" commit. .. tip:: The :class:`_orm.Session` doesn't actually hold onto the :class:`_engine.Connection` object after it ends the transaction. It gets a new :class:`_engine.Connection` from the :class:`_engine.Engine` - when executing SQL against the database is next needed. + the next time it needs to execute SQL against the database. The :class:`_orm.Session` obviously has a lot more tricks up its sleeve than that, however understanding that it has a :meth:`_orm.Session.execute` diff --git a/lib/sqlalchemy/orm/query.py b/lib/sqlalchemy/orm/query.py index 9ac6d07da..2d97754f4 100644 --- a/lib/sqlalchemy/orm/query.py +++ b/lib/sqlalchemy/orm/query.py @@ -1963,9 +1963,10 @@ class Query( q = session.query(Entity).order_by(Entity.id, Entity.name) - All existing ORDER BY criteria may be cancelled by passing - ``None`` by itself. New ORDER BY criteria may then be added by - invoking :meth:`_orm.Query.order_by` again, e.g.:: + Calling this method multiple times is equivalent to calling it once + with all the clauses concatenated. All existing ORDER BY criteria may + be cancelled by passing ``None`` by itself. New ORDER BY criteria may + then be added by invoking :meth:`_orm.Query.order_by` again, e.g.:: # will erase all ORDER BY and ORDER BY new_col alone q = q.order_by(None).order_by(new_col) diff --git a/lib/sqlalchemy/orm/relationships.py b/lib/sqlalchemy/orm/relationships.py index e0922a538..986093e02 100644 --- a/lib/sqlalchemy/orm/relationships.py +++ b/lib/sqlalchemy/orm/relationships.py @@ -164,6 +164,7 @@ _ORMOrderByArgument = Union[ Literal[False], str, _ColumnExpressionArgument[Any], + Callable[[], Iterable[ColumnElement[Any]]], Iterable[Union[str, _ColumnExpressionArgument[Any]]], ] _ORMBackrefArgument = Union[str, Tuple[str, Dict[str, Any]]] diff --git a/lib/sqlalchemy/sql/compiler.py b/lib/sqlalchemy/sql/compiler.py index 97397e9cf..9a00afc91 100644 --- a/lib/sqlalchemy/sql/compiler.py +++ b/lib/sqlalchemy/sql/compiler.py @@ -63,6 +63,7 @@ from . import roles from . import schema from . import selectable from . import sqltypes +from . import util as sql_util from ._typing import is_column_element from ._typing import is_dml from .base import _from_objects @@ -1530,7 +1531,8 @@ class SQLCompiler(Compiled): replacement_expressions[ escaped_name ] = self.render_literal_bindparam( - parameter, render_literal_value=value + parameter, + render_literal_value=value, ) continue @@ -3154,10 +3156,22 @@ class SQLCompiler(Compiled): processor = type_._cached_literal_processor(self.dialect) if processor: - return processor(value) + try: + return processor(value) + except Exception as e: + raise exc.CompileError( + f"Could not render literal value " + f'"{sql_util._repr_single_value(value)}" ' + f"with datatype " + f"{type_}; see parent stack trace for " + "more detail." + ) from e + else: - raise NotImplementedError( - "Don't know how to literal-quote value %r" % value + raise exc.CompileError( + f"No literal value renderer is available for literal value " + f'"{sql_util._repr_single_value(value)}" ' + f"with datatype {type_}" ) def _truncate_bindparam(self, bindparam): diff --git a/lib/sqlalchemy/sql/selectable.py b/lib/sqlalchemy/sql/selectable.py index 3d3aea3f2..08d01c883 100644 --- a/lib/sqlalchemy/sql/selectable.py +++ b/lib/sqlalchemy/sql/selectable.py @@ -4096,9 +4096,10 @@ class GenerativeSelect(SelectBase, Generative): stmt = select(table).order_by(table.c.id, table.c.name) - All existing ORDER BY criteria may be cancelled by passing - ``None`` by itself. New ORDER BY criteria may then be added by - invoking :meth:`_sql.Select.order_by` again, e.g.:: + Calling this method multiple times is equivalent to calling it once + with all the clauses concatenated. All existing ORDER BY criteria may + be cancelled by passing ``None`` by itself. New ORDER BY criteria may + then be added by invoking :meth:`_orm.Query.order_by` again, e.g.:: # will erase all ORDER BY and ORDER BY new_col alone stmt = stmt.order_by(None).order_by(new_col) diff --git a/lib/sqlalchemy/sql/sqltypes.py b/lib/sqlalchemy/sql/sqltypes.py index 414ff03c3..b98a16b6f 100644 --- a/lib/sqlalchemy/sql/sqltypes.py +++ b/lib/sqlalchemy/sql/sqltypes.py @@ -3381,12 +3381,7 @@ class NullType(TypeEngine[None]): _isnull = True def literal_processor(self, dialect): - def process(value): - raise exc.CompileError( - "Don't know how to render literal SQL value: %r" % (value,) - ) - - return process + return None class Comparator(TypeEngine.Comparator[_T]): __slots__ = () diff --git a/lib/sqlalchemy/sql/util.py b/lib/sqlalchemy/sql/util.py index 623a3f896..ec8ea757f 100644 --- a/lib/sqlalchemy/sql/util.py +++ b/lib/sqlalchemy/sql/util.py @@ -581,6 +581,12 @@ class _repr_base: return rep +def _repr_single_value(value): + rp = _repr_base() + rp.max_chars = 300 + return rp.trunc(value) + + class _repr_row(_repr_base): """Provide a string view of a row.""" diff --git a/lib/sqlalchemy/testing/requirements.py b/lib/sqlalchemy/testing/requirements.py index 38d962fef..3c63e9362 100644 --- a/lib/sqlalchemy/testing/requirements.py +++ b/lib/sqlalchemy/testing/requirements.py @@ -1439,6 +1439,10 @@ class SuiteRequirements(Requirements): ) @property + def is64bit(self): + return exclusions.only_if(lambda: util.is64bit, "64bit required") + + @property def patch_library(self): def check_lib(): try: diff --git a/lib/sqlalchemy/util/__init__.py b/lib/sqlalchemy/util/__init__.py index bb4642a4f..e82cfd769 100644 --- a/lib/sqlalchemy/util/__init__.py +++ b/lib/sqlalchemy/util/__init__.py @@ -58,6 +58,7 @@ from .compat import decode_backslashreplace as decode_backslashreplace from .compat import dottedgetter as dottedgetter from .compat import has_refcount_gc as has_refcount_gc from .compat import inspect_getfullargspec as inspect_getfullargspec +from .compat import is64bit as is64bit from .compat import local_dataclass_fields as local_dataclass_fields from .compat import osx as osx from .compat import py310 as py310 diff --git a/lib/sqlalchemy/util/compat.py b/lib/sqlalchemy/util/compat.py index 2899b4258..24f9bcf10 100644 --- a/lib/sqlalchemy/util/compat.py +++ b/lib/sqlalchemy/util/compat.py @@ -41,6 +41,7 @@ cpython = platform.python_implementation() == "CPython" win32 = sys.platform.startswith("win") osx = sys.platform.startswith("darwin") arm = "aarch" in platform.machine().lower() +is64bit = platform.architecture()[0] == "64bit" has_refcount_gc = bool(cpython) diff --git a/test/dialect/postgresql/test_types.py b/test/dialect/postgresql/test_types.py index 83cea8f15..39e7d7317 100644 --- a/test/dialect/postgresql/test_types.py +++ b/test/dialect/postgresql/test_types.py @@ -1528,8 +1528,9 @@ class ArrayTest(AssertsCompiledSQL, fixtures.TestBase): return "MYTYPE" with expect_raises_message( - NotImplementedError, - r"Don't know how to literal-quote value \[1, 2, 3\]", + exc.CompileError, + r"No literal value renderer is available for literal " + r"value \"\[1, 2, 3\]\" with datatype ARRAY", ): self.assert_compile( select(literal([1, 2, 3], ARRAY(MyType()))), diff --git a/test/orm/test_cache_key.py b/test/orm/test_cache_key.py index 3106a71ad..a0bf8b598 100644 --- a/test/orm/test_cache_key.py +++ b/test/orm/test_cache_key.py @@ -1092,7 +1092,11 @@ class EmbeddedSubqTest(_RemoveListeners, DeclarativeMappedTest): "concrete": True, } - @testing.combinations("tuples", "memory", argnames="assert_on") + Base.registry.configure() + + @testing.combinations( + "tuples", ("memory", testing.requires.is64bit), argnames="assert_on" + ) def test_cache_key_gen(self, assert_on): Employee = self.classes.Employee diff --git a/test/sql/test_compiler.py b/test/sql/test_compiler.py index 97b1b9124..4eea11795 100644 --- a/test/sql/test_compiler.py +++ b/test/sql/test_compiler.py @@ -98,6 +98,7 @@ from sqlalchemy.testing import is_true from sqlalchemy.testing import mock from sqlalchemy.testing import ne_ from sqlalchemy.testing.schema import pep435_enum +from sqlalchemy.types import UserDefinedType table1 = table( "mytable", @@ -4609,6 +4610,51 @@ class BindParameterTest(AssertsCompiledSQL, fixtures.TestBase): "OR mytable.myid = :myid_2 OR mytable.myid = :myid_3", ) + @testing.combinations("plain", "expanding", argnames="exprtype") + def test_literal_bind_typeerror(self, exprtype): + """test #8800""" + + if exprtype == "expanding": + stmt = select(table1).where( + table1.c.myid.in_([("tuple",), ("tuple",)]) + ) + elif exprtype == "plain": + stmt = select(table1).where(table1.c.myid == ("tuple",)) + else: + assert False + + with expect_raises_message( + exc.CompileError, + r"Could not render literal value \"\(\'tuple\',\)\" " + r"with datatype INTEGER; see parent " + r"stack trace for more detail.", + ): + stmt.compile(compile_kwargs={"literal_binds": True}) + + @testing.combinations("plain", "expanding", argnames="exprtype") + def test_literal_bind_dont_know_how_to_quote(self, exprtype): + """test #8800""" + + class MyType(UserDefinedType): + def get_col_spec(self, **kw): + return "MYTYPE" + + col = column("x", MyType()) + + if exprtype == "expanding": + stmt = select(table1).where(col.in_([("tuple",), ("tuple",)])) + elif exprtype == "plain": + stmt = select(table1).where(col == ("tuple",)) + else: + assert False + + with expect_raises_message( + exc.CompileError, + r"No literal value renderer is available for literal " + r"value \"\('tuple',\)\" with datatype MYTYPE", + ): + stmt.compile(compile_kwargs={"literal_binds": True}) + @testing.fixture def ansi_compiler_fixture(self): dialect = default.DefaultDialect() diff --git a/test/sql/test_types.py b/test/sql/test_types.py index a608d0040..3b1df3498 100644 --- a/test/sql/test_types.py +++ b/test/sql/test_types.py @@ -3125,8 +3125,9 @@ class ArrayTest(AssertsCompiledSQL, fixtures.TestBase): return "MYTYPE" with expect_raises_message( - NotImplementedError, - r"Don't know how to literal-quote value \[1, 2, 3\]", + exc.CompileError, + r"No literal value renderer is available for literal value " + r"\"\[1, 2, 3\]\" with datatype ARRAY", ): self.assert_compile( select(literal([1, 2, 3], ARRAY(MyType()))), @@ -3629,7 +3630,8 @@ class CompileTest(fixtures.TestBase, AssertsCompiledSQL): def test_compile_err_formatting(self): with expect_raises_message( exc.CompileError, - r"Don't know how to render literal SQL value: \(1, 2, 3\)", + r"No literal value renderer is available for literal " + r"value \"\(1, 2, 3\)\" with datatype NULL", ): func.foo((1, 2, 3)).compile(compile_kwargs={"literal_binds": True}) diff --git a/test/sql/test_values.py b/test/sql/test_values.py index d14de9aee..b943c4701 100644 --- a/test/sql/test_values.py +++ b/test/sql/test_values.py @@ -277,7 +277,8 @@ class ValuesTest(fixtures.TablesTest, AssertsCompiledSQL): with expect_raises_message( exc.CompileError, - "Don't know how to render literal SQL value: 'textA'", + r"No literal value renderer is available for literal " + r"value \"'textA'\" with datatype NULL", ): str(stmt) |