diff options
author | Mike Bayer <mike_mp@zzzcomputing.com> | 2019-06-13 12:37:22 -0400 |
---|---|---|
committer | Mike Bayer <mike_mp@zzzcomputing.com> | 2019-07-06 13:02:22 -0400 |
commit | ef7ff058eb67d73ebeac7b125ab2a7806e14629c (patch) | |
tree | 9a09162961f7bcdb6d16837adacabb99f10b4410 /lib/sqlalchemy | |
parent | 1ce98ca83a4b2da12e52aa0f4ab181c83063abc2 (diff) | |
download | sqlalchemy-ef7ff058eb67d73ebeac7b125ab2a7806e14629c.tar.gz |
SelectBase no longer a FromClause
As part of the SQLAlchemy 2.0 migration project, a conceptual change has
been made to the role of the :class:`.SelectBase` class hierarchy,
which is the root of all "SELECT" statement constructs, in that they no
longer serve directly as FROM clauses, that is, they no longer subclass
:class:`.FromClause`. For end users, the change mostly means that any
placement of a :func:`.select` construct in the FROM clause of another
:func:`.select` requires first that it be wrapped in a subquery first,
which historically is through the use of the :meth:`.SelectBase.alias`
method, and is now also available through the use of
:meth:`.SelectBase.subquery`. This was usually a requirement in any
case since several databases don't accept unnamed SELECT subqueries
in their FROM clause in any case.
See the documentation in this change for lots more detail.
Fixes: #4617
Change-Id: I0f6174ee24b9a1a4529168e52e855e12abd60667
Diffstat (limited to 'lib/sqlalchemy')
-rw-r--r-- | lib/sqlalchemy/dialects/mssql/base.py | 2 | ||||
-rw-r--r-- | lib/sqlalchemy/dialects/oracle/base.py | 14 | ||||
-rw-r--r-- | lib/sqlalchemy/engine/result.py | 6 | ||||
-rw-r--r-- | lib/sqlalchemy/orm/query.py | 34 | ||||
-rw-r--r-- | lib/sqlalchemy/orm/strategies.py | 1 | ||||
-rw-r--r-- | lib/sqlalchemy/orm/util.py | 4 | ||||
-rw-r--r-- | lib/sqlalchemy/sql/__init__.py | 1 | ||||
-rw-r--r-- | lib/sqlalchemy/sql/base.py | 144 | ||||
-rw-r--r-- | lib/sqlalchemy/sql/clause_compare.py | 2 | ||||
-rw-r--r-- | lib/sqlalchemy/sql/coercions.py | 126 | ||||
-rw-r--r-- | lib/sqlalchemy/sql/compiler.py | 100 | ||||
-rw-r--r-- | lib/sqlalchemy/sql/dml.py | 4 | ||||
-rw-r--r-- | lib/sqlalchemy/sql/elements.py | 109 | ||||
-rw-r--r-- | lib/sqlalchemy/sql/expression.py | 5 | ||||
-rw-r--r-- | lib/sqlalchemy/sql/roles.py | 10 | ||||
-rw-r--r-- | lib/sqlalchemy/sql/selectable.py | 1012 | ||||
-rw-r--r-- | lib/sqlalchemy/sql/util.py | 2 | ||||
-rw-r--r-- | lib/sqlalchemy/testing/suite/__init__.py | 1 | ||||
-rw-r--r-- | lib/sqlalchemy/testing/suite/test_deprecations.py | 144 | ||||
-rw-r--r-- | lib/sqlalchemy/testing/suite/test_select.py | 24 |
20 files changed, 1193 insertions, 552 deletions
diff --git a/lib/sqlalchemy/dialects/mssql/base.py b/lib/sqlalchemy/dialects/mssql/base.py index b2bcc280c..86116e6a1 100644 --- a/lib/sqlalchemy/dialects/mssql/base.py +++ b/lib/sqlalchemy/dialects/mssql/base.py @@ -1760,7 +1760,7 @@ class MSSQLCompiler(compiler.SQLCompiler): select, column, asfrom ) - def for_update_clause(self, select): + def for_update_clause(self, select, **kw): # "FOR UPDATE" is only allowed on "DECLARE CURSOR" which # SQLAlchemy doesn't use return "" diff --git a/lib/sqlalchemy/dialects/oracle/base.py b/lib/sqlalchemy/dialects/oracle/base.py index 33f9c8659..842730c5b 100644 --- a/lib/sqlalchemy/dialects/oracle/base.py +++ b/lib/sqlalchemy/dialects/oracle/base.py @@ -886,7 +886,8 @@ class OracleCompiler(compiler.SQLCompiler): [ c for c in inner_subquery.c - if orig_select.corresponding_column(c) is not None + if orig_select.selected_columns.corresponding_column(c) + is not None ] ) if ( @@ -939,17 +940,22 @@ class OracleCompiler(compiler.SQLCompiler): limitselect._is_wrapper = True if for_update is not None and for_update.of: - + limitselect_cols = limitselect.selected_columns for elem in for_update.of: - if limitselect.corresponding_column(elem) is None: + if ( + limitselect_cols.corresponding_column(elem) + is None + ): limitselect = limitselect.column(elem) limit_subquery = limitselect.alias() + origselect_cols = orig_select.selected_columns offsetselect = sql.select( [ c for c in limit_subquery.c - if orig_select.corresponding_column(c) is not None + if origselect_cols.corresponding_column(c) + is not None ] ) diff --git a/lib/sqlalchemy/engine/result.py b/lib/sqlalchemy/engine/result.py index 740565f5e..480d086ff 100644 --- a/lib/sqlalchemy/engine/result.py +++ b/lib/sqlalchemy/engine/result.py @@ -300,7 +300,7 @@ class ResultMetaData(object): # if we did a pure positional match, then reset the # original "expression element" back to the "unambiguous" # entry. This is a new behavior in 1.1 which impacts - # TextAsFrom but also straight compiled SQL constructs. + # TextualSelect but also straight compiled SQL constructs. if not self.matched_on_name: self._keymap.update( [ @@ -363,7 +363,7 @@ class ResultMetaData(object): The remaining fairly common case is that of the textual SQL that includes at least partial column information; this is when - we use a :class:`.TextAsFrom` construct. This construct may have + we use a :class:`.TextualSelect` construct. This construct may have unordered or ordered column information. In the ordered case, we merge the cursor.description and the compiled construct's information positionally, and warn if there are additional description names @@ -386,7 +386,7 @@ class ResultMetaData(object): SQLAlchemy for all cases up through te 0.9 series. Positional matching for compiled SQL expressions was introduced in 1.0 as a major performance feature, and positional matching for textual - :class:`.TextAsFrom` objects in 1.1. As name matching is no longer + :class:`.TextualSelect` objects in 1.1. As name matching is no longer a common case, it was acceptable to factor it into smaller generator- oriented methods that are easier to understand, but incur slightly more performance overhead. diff --git a/lib/sqlalchemy/orm/query.py b/lib/sqlalchemy/orm/query.py index c73a8147c..13402e7f4 100644 --- a/lib/sqlalchemy/orm/query.py +++ b/lib/sqlalchemy/orm/query.py @@ -253,11 +253,6 @@ class Query(object): "expected when the base alias is being set." ) fa.append(info.selectable) - elif not info.is_clause_element or not info._is_from_clause: - raise sa_exc.ArgumentError( - "argument is not a mapped class, mapper, " - "aliased(), or FromClause instance." - ) else: from_obj = coercions.expect( roles.StrictFromClauseRole, from_obj, allow_select=True @@ -271,7 +266,9 @@ class Query(object): if ( set_base_alias and len(self._from_obj) == 1 - and isinstance(select_from_alias, expression.Alias) + and isinstance( + select_from_alias, sql.selectable.AliasedReturnsRows + ) ): equivs = self.__all_equivs() self._from_obj_alias = sql_util.ColumnAdapter( @@ -2302,7 +2299,14 @@ class Query(object): if ( len(keys) == 2 and isinstance( - keys[0], (expression.FromClause, type, AliasedClass) + keys[0], + ( + # note this would be FromClause once + # coercion of SELECT is removed + expression.Selectable, + type, + AliasedClass, + ), ) and isinstance( keys[1], @@ -2761,7 +2765,9 @@ class Query(object): # if the destination selectable is a plain select(), # turn it into an alias(). if isinstance(right_selectable, expression.SelectBase): - right_selectable = right_selectable.alias() + right_selectable = coercions.expect( + roles.FromClauseRole, right_selectable + ) need_adapter = True # make the right hand side target into an ORM entity @@ -2781,7 +2787,8 @@ class Query(object): and ( right_mapper.with_polymorphic and isinstance( - right_mapper._with_polymorphic_selectable, expression.Alias + right_mapper._with_polymorphic_selectable, + expression.AliasedReturnsRows, ) or overlap # test for overlap: @@ -3201,13 +3208,10 @@ class Query(object): """ statement = coercions.expect(roles.SelectStatementRole, statement) - if not isinstance( + # TODO: coercions above should have this handled + assert isinstance( statement, (expression.TextClause, expression.SelectBase) - ): - raise sa_exc.ArgumentError( - "from_statement accepts text(), select(), " - "and union() objects only." - ) + ) self._statement = statement diff --git a/lib/sqlalchemy/orm/strategies.py b/lib/sqlalchemy/orm/strategies.py index b0dffe5dd..7e150414b 100644 --- a/lib/sqlalchemy/orm/strategies.py +++ b/lib/sqlalchemy/orm/strategies.py @@ -1629,7 +1629,6 @@ class JoinedLoader(AbstractRelationshipLoader): # the object becomes shared among threads. this prevents # races for column identities. inspect(to_adapt).selectable.c - self._aliased_class_pool.append(to_adapt) return self._aliased_class_pool[idx] diff --git a/lib/sqlalchemy/orm/util.py b/lib/sqlalchemy/orm/util.py index 80d4b26b2..4b4fa4052 100644 --- a/lib/sqlalchemy/orm/util.py +++ b/lib/sqlalchemy/orm/util.py @@ -825,7 +825,9 @@ def aliased(element, alias=None, name=None, flat=False, adapt_on_names=False): raise sa_exc.ArgumentError( "adapt_on_names only applies to ORM elements" ) - return element._anonymous_fromclause(name=name, flat=flat) + return coercions.expect( + roles.AnonymizedFromClauseRole, element, name=name, flat=flat + ) else: return AliasedClass( element, diff --git a/lib/sqlalchemy/sql/__init__.py b/lib/sqlalchemy/sql/__init__.py index 00cafd8ff..bf5468aa5 100644 --- a/lib/sqlalchemy/sql/__init__.py +++ b/lib/sqlalchemy/sql/__init__.py @@ -58,6 +58,7 @@ from .expression import quoted_name # noqa from .expression import Select # noqa from .expression import select # noqa from .expression import Selectable # noqa +from .expression import Subquery # noqa from .expression import subquery # noqa from .expression import table # noqa from .expression import TableClause # noqa diff --git a/lib/sqlalchemy/sql/base.py b/lib/sqlalchemy/sql/base.py index 9df0c932f..a84843c4b 100644 --- a/lib/sqlalchemy/sql/base.py +++ b/lib/sqlalchemy/sql/base.py @@ -11,6 +11,7 @@ import itertools +import operator import re from .visitors import ClauseVisitor @@ -51,6 +52,38 @@ def _generative(fn, *args, **kw): return self +def _clone(element, **kw): + return element._clone() + + +def _expand_cloned(elements): + """expand the given set of ClauseElements to be the set of all 'cloned' + predecessors. + + """ + return itertools.chain(*[x._cloned_set for x in elements]) + + +def _cloned_intersection(a, b): + """return the intersection of sets a and b, counting + any overlap between 'cloned' predecessors. + + The returned set is in terms of the entities present within 'a'. + + """ + all_overlap = set(_expand_cloned(a)).intersection(_expand_cloned(b)) + return set( + elem for elem in a if all_overlap.intersection(elem._cloned_set) + ) + + +def _cloned_difference(a, b): + all_overlap = set(_expand_cloned(a)).intersection(_expand_cloned(b)) + return set( + elem for elem in a if not all_overlap.intersection(elem._cloned_set) + ) + + class _DialectArgView(util.collections_abc.MutableMapping): """A dictionary view of dialect-level arguments in the form <dialectname>_<argument_name>. @@ -486,6 +519,97 @@ class ColumnCollection(util.OrderedProperties): def __str__(self): return repr([str(c) for c in self]) + def corresponding_column(self, column, require_embedded=False): + """Given a :class:`.ColumnElement`, return the exported + :class:`.ColumnElement` object from this :class:`.ColumnCollection` + which corresponds to that original :class:`.ColumnElement` via a common + ancestor column. + + :param column: the target :class:`.ColumnElement` to be matched + + :param require_embedded: only return corresponding columns for + the given :class:`.ColumnElement`, if the given + :class:`.ColumnElement` is actually present within a sub-element + of this :class:`.Selectable`. Normally the column will match if + it merely shares a common ancestor with one of the exported + columns of this :class:`.Selectable`. + + .. seealso:: + + :meth:`.Selectable.corresponding_column` - invokes this method + against the collection returned by + :attr:`.Selectable.exported_columns`. + + .. versionchanged:: 1.4 the implementation for ``corresponding_column`` + was moved onto the :class:`.ColumnCollection` itself. + + """ + + def embedded(expanded_proxy_set, target_set): + for t in target_set.difference(expanded_proxy_set): + if not set(_expand_cloned([t])).intersection( + expanded_proxy_set + ): + return False + return True + + # don't dig around if the column is locally present + if self.contains_column(column): + return column + col, intersect = None, None + target_set = column.proxy_set + cols = self._all_columns + for c in cols: + expanded_proxy_set = set(_expand_cloned(c.proxy_set)) + i = target_set.intersection(expanded_proxy_set) + if i and ( + not require_embedded + or embedded(expanded_proxy_set, target_set) + ): + if col is None: + + # no corresponding column yet, pick this one. + + col, intersect = c, i + elif len(i) > len(intersect): + + # 'c' has a larger field of correspondence than + # 'col'. i.e. selectable.c.a1_x->a1.c.x->table.c.x + # matches a1.c.x->table.c.x better than + # selectable.c.x->table.c.x does. + + col, intersect = c, i + elif i == intersect: + # they have the same field of correspondence. see + # which proxy_set has fewer columns in it, which + # indicates a closer relationship with the root + # column. Also take into account the "weight" + # attribute which CompoundSelect() uses to give + # higher precedence to columns based on vertical + # position in the compound statement, and discard + # columns that have no reference to the target + # column (also occurs with CompoundSelect) + + col_distance = util.reduce( + operator.add, + [ + sc._annotations.get("weight", 1) + for sc in col._uncached_proxy_set() + if sc.shares_lineage(column) + ], + ) + c_distance = util.reduce( + operator.add, + [ + sc._annotations.get("weight", 1) + for sc in c._uncached_proxy_set() + if sc.shares_lineage(column) + ], + ) + if c_distance < col_distance: + col, intersect = c, i + return col + def replace(self, column): """add the given column to this collection, removing unaliased versions of this column as well as existing columns with the @@ -619,6 +743,26 @@ class ColumnCollection(util.OrderedProperties): return ImmutableColumnCollection(self._data, self._all_columns) +class SeparateKeyColumnCollection(ColumnCollection): + """Column collection that maintains a string name separate from the + column itself""" + + def __init__(self, cols_plus_names=None): + super(ColumnCollection, self).__init__() + object.__setattr__(self, "_all_columns", []) + if cols_plus_names: + self.update(cols_plus_names) + + def replace(self, column): + raise NotImplementedError() + + def add(self, column): + raise NotImplementedError() + + def remove(self, column): + raise NotImplementedError() + + class ImmutableColumnCollection(util.ImmutableProperties, ColumnCollection): def __init__(self, data, all_columns): util.ImmutableProperties.__init__(self, data) diff --git a/lib/sqlalchemy/sql/clause_compare.py b/lib/sqlalchemy/sql/clause_compare.py index 0ea981f1e..50b1df99e 100644 --- a/lib/sqlalchemy/sql/clause_compare.py +++ b/lib/sqlalchemy/sql/clause_compare.py @@ -291,7 +291,7 @@ class StructureComparatorStrategy(object): return True - def compare_text_as_from(self, left, right, **kw): + def compare_textual_select(self, left, right, **kw): self.compare_stack.extendleft( util.zip_longest(left.column_args, right.column_args) ) diff --git a/lib/sqlalchemy/sql/coercions.py b/lib/sqlalchemy/sql/coercions.py index 39e6628e4..64d9f0f96 100644 --- a/lib/sqlalchemy/sql/coercions.py +++ b/lib/sqlalchemy/sql/coercions.py @@ -124,18 +124,24 @@ class RoleImpl(object): return element def _implicit_coercions(self, element, resolved, argname=None, **kw): - self._raise_for_expected(element, argname) + self._raise_for_expected(element, argname, resolved) - def _raise_for_expected(self, element, argname=None): + def _raise_for_expected( + self, element, argname=None, resolved=None, advice=None, code=None + ): if argname: - raise exc.ArgumentError( - "%s expected for argument %r; got %r." - % (self.name, argname, element) + msg = "%s expected for argument %r; got %r." % ( + self.name, + argname, + element, ) else: - raise exc.ArgumentError( - "%s expected, got %r." % (self.name, element) - ) + msg = "%s expected, got %r." % (self.name, element) + + if advice: + msg += " " + advice + + raise exc.ArgumentError(msg, code=code) class _StringOnly(object): @@ -150,7 +156,7 @@ class _ReturnsStringKey(object): if isinstance(original_element, util.string_types): return original_element else: - self._raise_for_expected(original_element, argname) + self._raise_for_expected(original_element, argname, resolved) def _literal_coercion(self, element, **kw): return element @@ -172,15 +178,13 @@ class _ColumnCoercions(object): if resolved._is_select_statement: self._warn_for_scalar_subquery_coercion() return resolved.scalar_subquery() - elif ( - resolved._is_from_clause - and isinstance(resolved, selectable.Alias) - and resolved.element._is_select_statement + elif resolved._is_from_clause and isinstance( + resolved, selectable.Subquery ): self._warn_for_scalar_subquery_coercion() return resolved.element.scalar_subquery() else: - self._raise_for_expected(original_element, argname) + self._raise_for_expected(original_element, argname, resolved) def _no_text_coercion( @@ -236,6 +240,30 @@ class _CoerceLiterals(object): self._raise_for_expected(element, argname) +class _SelectIsNotFrom(object): + def _raise_for_expected(self, element, argname=None, resolved=None, **kw): + if isinstance(element, roles.SelectStatementRole) or isinstance( + resolved, roles.SelectStatementRole + ): + advice = ( + "To create a " + "FROM clause from a %s object, use the .subquery() method." + % (element.__class__) + ) + code = "89ve" + else: + advice = code = None + + return super(_SelectIsNotFrom, self)._raise_for_expected( + element, + argname=argname, + resolved=resolved, + advice=advice, + code=code, + **kw + ) + + class ExpressionElementImpl( _ColumnCoercions, RoleImpl, roles.ExpressionElementRole ): @@ -287,7 +315,7 @@ class InElementImpl(RoleImpl, roles.InElementRole): else: return resolved.select() else: - self._raise_for_expected(original_element, argname) + self._raise_for_expected(original_element, argname, resolved) def _literal_coercion(self, element, expr, operator, **kw): if isinstance(element, collections_abc.Iterable) and not isinstance( @@ -412,7 +440,7 @@ class TruncatedLabelImpl(_StringOnly, RoleImpl, roles.TruncatedLabelRole): if isinstance(original_element, util.string_types): return resolved else: - self._raise_for_expected(original_element, argname) + self._raise_for_expected(original_element, argname, resolved) def _literal_coercion(self, element, argname=None, **kw): """coerce the given value to :class:`._truncated_label`. @@ -447,7 +475,7 @@ class LimitOffsetImpl(RoleImpl, roles.LimitOffsetRole): if resolved is None: return None else: - self._raise_for_expected(element, argname) + self._raise_for_expected(element, argname, resolved) def _literal_coercion(self, element, name, type_, **kw): if element is None: @@ -474,10 +502,12 @@ class LabeledColumnExprImpl( if isinstance(new, roles.ExpressionElementRole): return new.label(None) else: - self._raise_for_expected(original_element, argname) + self._raise_for_expected(original_element, argname, resolved) -class ColumnsClauseImpl(_CoerceLiterals, RoleImpl, roles.ColumnsClauseRole): +class ColumnsClauseImpl( + _SelectIsNotFrom, _CoerceLiterals, RoleImpl, roles.ColumnsClauseRole +): _coerce_consts = True _coerce_numerics = True @@ -526,21 +556,40 @@ class SelectStatementImpl( if resolved._is_text_clause: return resolved.columns() else: - self._raise_for_expected(original_element, argname) + self._raise_for_expected(original_element, argname, resolved) class HasCTEImpl(ReturnsRowsImpl, roles.HasCTERole): pass -class FromClauseImpl(_NoTextCoercion, RoleImpl, roles.FromClauseRole): +class FromClauseImpl( + _SelectIsNotFrom, _NoTextCoercion, RoleImpl, roles.FromClauseRole +): def _implicit_coercions( - self, original_element, resolved, argname=None, **kw + self, + original_element, + resolved, + argname=None, + explicit_subquery=False, + allow_select=True, + **kw ): - if resolved._is_text_clause: + if resolved._is_select_statement: + if explicit_subquery: + return resolved.subquery() + elif allow_select: + util.warn_deprecated( + "Implicit coercion of SELECT and textual SELECT " + "constructs into FROM clauses is deprecated; please call " + ".subquery() on any Core select or ORM Query object in " + "order to produce a subquery object." + ) + return resolved._implicit_subquery + elif resolved._is_text_clause: return resolved else: - self._raise_for_expected(original_element, argname) + self._raise_for_expected(original_element, argname, resolved) class StrictFromClauseImpl(FromClauseImpl, roles.StrictFromClauseRole): @@ -559,16 +608,16 @@ class StrictFromClauseImpl(FromClauseImpl, roles.StrictFromClauseRole): "on any Core select or ORM Query object in order to produce a " "subquery object." ) - return resolved.subquery() + return resolved._implicit_subquery else: - self._raise_for_expected(original_element, argname) + self._raise_for_expected(original_element, argname, resolved) class AnonymizedFromClauseImpl( StrictFromClauseImpl, roles.AnonymizedFromClauseRole ): - def _post_coercion(self, element, flat=False, **kw): - return element.alias(flat=flat) + def _post_coercion(self, element, flat=False, name=None, **kw): + return element.alias(name=name, flat=flat) class DMLSelectImpl(_NoTextCoercion, RoleImpl, roles.DMLSelectRole): @@ -584,17 +633,28 @@ class DMLSelectImpl(_NoTextCoercion, RoleImpl, roles.DMLSelectRole): else: return resolved.select() else: - self._raise_for_expected(original_element, argname) + self._raise_for_expected(original_element, argname, resolved) class CompoundElementImpl( _NoTextCoercion, RoleImpl, roles.CompoundElementRole ): - def _implicit_coercions(self, original_element, resolved, argname=None): - if resolved._is_from_clause: - return resolved + def _raise_for_expected(self, element, argname=None, resolved=None, **kw): + if isinstance(element, roles.FromClauseRole): + if element._is_subquery: + advice = ( + "Use the plain select() object without " + "calling .subquery() or .alias()." + ) + else: + advice = ( + "To SELECT from any FROM clause, use the .select() method." + ) else: - self._raise_for_expected(original_element, argname) + advice = None + return super(CompoundElementImpl, self)._raise_for_expected( + element, argname=argname, resolved=resolved, advice=advice, **kw + ) _impl_lookup = {} diff --git a/lib/sqlalchemy/sql/compiler.py b/lib/sqlalchemy/sql/compiler.py index 7922054f8..13219ee68 100644 --- a/lib/sqlalchemy/sql/compiler.py +++ b/lib/sqlalchemy/sql/compiler.py @@ -488,7 +488,7 @@ class SQLCompiler(Compiled): """ if False, means we can't be sure the list of entries in _result_columns is actually the rendered order. Usually - True unless using an unordered TextAsFrom. + True unless using an unordered TextualSelect. """ _numeric_binds = False @@ -916,8 +916,8 @@ class SQLCompiler(Compiled): ), ) - def visit_text_as_from( - self, taf, compound_index=None, asfrom=False, parens=True, **kw + def visit_textual_select( + self, taf, compound_index=None, asfrom=False, **kw ): toplevel = not self.stack @@ -943,10 +943,7 @@ class SQLCompiler(Compiled): add_to_result_map=self._add_to_result_map, ) - text = self.process(taf.element, **kw) - if asfrom and parens: - text = "(%s)" % text - return text + return self.process(taf.element, **kw) def visit_null(self, expr, **kw): return "NULL" @@ -1120,7 +1117,7 @@ class SQLCompiler(Compiled): return func.clause_expr._compiler_dispatch(self, **kwargs) def visit_compound_select( - self, cs, asfrom=False, parens=True, compound_index=0, **kwargs + self, cs, asfrom=False, compound_index=0, **kwargs ): toplevel = not self.stack entry = self._default_stack_entry if toplevel else self.stack[-1] @@ -1143,16 +1140,13 @@ class SQLCompiler(Compiled): text = (" " + keyword + " ").join( ( c._compiler_dispatch( - self, - asfrom=asfrom, - parens=False, - compound_index=i, - **kwargs + self, asfrom=asfrom, compound_index=i, **kwargs ) for i, c in enumerate(cs.selects) ) ) + kwargs["include_table"] = False text += self.group_by_clause(cs, **dict(asfrom=asfrom, **kwargs)) text += self.order_by_clause(cs, **kwargs) text += ( @@ -1165,10 +1159,7 @@ class SQLCompiler(Compiled): text = self._render_cte_clause() + text self.stack.pop(-1) - if asfrom and parens: - return "(" + text + ")" - else: - return text + return text def _get_operator_dispatch(self, operator_, qualifier1, qualifier2): attrname = "visit_%s_%s%s" % ( @@ -1682,8 +1673,11 @@ class SQLCompiler(Compiled): if self.positional: kwargs["positional_names"] = self.cte_positional[cte] = [] - text += " AS \n" + cte.element._compiler_dispatch( - self, asfrom=True, **kwargs + assert kwargs.get("subquery", False) is False + text += " AS \n(%s)" % ( + cte.element._compiler_dispatch( + self, asfrom=True, **kwargs + ), ) if cte._suffixes: @@ -1713,8 +1707,28 @@ class SQLCompiler(Compiled): ashint=False, iscrud=False, fromhints=None, + subquery=False, + lateral=False, + enclosing_alias=None, **kwargs ): + if enclosing_alias is not None and enclosing_alias.element is alias: + inner = alias.element._compiler_dispatch( + self, + asfrom=asfrom, + ashint=ashint, + iscrud=iscrud, + fromhints=fromhints, + lateral=lateral, + enclosing_alias=alias, + **kwargs + ) + if subquery and (asfrom or lateral): + inner = "(%s)" % (inner,) + return inner + else: + enclosing_alias = kwargs["enclosing_alias"] = alias + if asfrom or ashint: if isinstance(alias.name, elements._truncated_label): alias_name = self._truncated_identifier("alias", alias.name) @@ -1724,12 +1738,15 @@ class SQLCompiler(Compiled): if ashint: return self.preparer.format_alias(alias, alias_name) elif asfrom: - ret = alias.element._compiler_dispatch( - self, asfrom=True, **kwargs - ) + self.get_render_as_alias_suffix( - self.preparer.format_alias(alias, alias_name) + inner = alias.element._compiler_dispatch( + self, asfrom=True, lateral=lateral, **kwargs ) + if subquery: + inner = "(%s)" % (inner,) + ret = inner + self.get_render_as_alias_suffix( + self.preparer.format_alias(alias, alias_name) + ) if fromhints and alias in fromhints: ret = self.format_from_hint_text( ret, alias, fromhints[alias], iscrud @@ -1737,7 +1754,14 @@ class SQLCompiler(Compiled): return ret else: - return alias.element._compiler_dispatch(self, **kwargs) + # note we cancel the "subquery" flag here as well + return alias.element._compiler_dispatch( + self, lateral=lateral, **kwargs + ) + + def visit_subquery(self, subquery, **kw): + kw["subquery"] = True + return self.visit_alias(subquery, **kw) def visit_lateral(self, lateral, **kw): kw["lateral"] = True @@ -2004,7 +2028,6 @@ class SQLCompiler(Compiled): self, select, asfrom=False, - parens=True, fromhints=None, compound_index=0, nested_join_translation=False, @@ -2027,7 +2050,6 @@ class SQLCompiler(Compiled): text = self.visit_select( transformed_select, asfrom=asfrom, - parens=parens, fromhints=fromhints, compound_index=compound_index, nested_join_translation=True, @@ -2138,10 +2160,7 @@ class SQLCompiler(Compiled): self.stack.pop(-1) - if (asfrom or lateral) and parens: - return "(" + text + ")" - else: - return text + return text def _setup_select_hints(self, select): byfrom = dict( @@ -2371,7 +2390,7 @@ class SQLCompiler(Compiled): ) return dialect_hints, table_text - def visit_insert(self, insert_stmt, asfrom=False, **kw): + def visit_insert(self, insert_stmt, **kw): toplevel = not self.stack self.stack.append( @@ -2475,10 +2494,7 @@ class SQLCompiler(Compiled): self.stack.pop(-1) - if asfrom: - return "(" + text + ")" - else: - return text + return text def update_limit_clause(self, update_stmt): """Provide a hook for MySQL to add LIMIT to the UPDATE""" @@ -2508,7 +2524,7 @@ class SQLCompiler(Compiled): "criteria within UPDATE" ) - def visit_update(self, update_stmt, asfrom=False, **kw): + def visit_update(self, update_stmt, **kw): toplevel = not self.stack extra_froms = update_stmt._extra_froms @@ -2605,10 +2621,7 @@ class SQLCompiler(Compiled): self.stack.pop(-1) - if asfrom: - return "(" + text + ")" - else: - return text + return text @util.memoized_property def _key_getters_for_crud_column(self): @@ -2633,7 +2646,7 @@ class SQLCompiler(Compiled): def delete_table_clause(self, delete_stmt, from_table, extra_froms): return from_table._compiler_dispatch(self, asfrom=True, iscrud=True) - def visit_delete(self, delete_stmt, asfrom=False, **kw): + def visit_delete(self, delete_stmt, **kw): toplevel = not self.stack crud._setup_crud_params(self, delete_stmt, crud.ISDELETE, **kw) @@ -2702,10 +2715,7 @@ class SQLCompiler(Compiled): self.stack.pop(-1) - if asfrom: - return "(" + text + ")" - else: - return text + return text def visit_savepoint(self, savepoint_stmt): return "SAVEPOINT %s" % self.preparer.format_savepoint(savepoint_stmt) diff --git a/lib/sqlalchemy/sql/dml.py b/lib/sqlalchemy/sql/dml.py index c7d83fc12..5a9be7c62 100644 --- a/lib/sqlalchemy/sql/dml.py +++ b/lib/sqlalchemy/sql/dml.py @@ -47,6 +47,10 @@ class UpdateBase( _prefixes = () named_with_column = False + def _generate_fromclause_column_proxies(self, fromclause): + for col in self._returning: + col._make_proxy(fromclause) + def _process_colparams(self, parameters): def process_single(p): if isinstance(p, (list, tuple)): diff --git a/lib/sqlalchemy/sql/elements.py b/lib/sqlalchemy/sql/elements.py index b902ef4b4..5b4442222 100644 --- a/lib/sqlalchemy/sql/elements.py +++ b/lib/sqlalchemy/sql/elements.py @@ -22,6 +22,7 @@ from . import operators from . import roles from . import type_api from .annotation import Annotated +from .base import _clone from .base import _generative from .base import Executable from .base import Immutable @@ -36,10 +37,6 @@ from .. import inspection from .. import util -def _clone(element, **kw): - return element._clone() - - def collate(expression, collation): """Return the clause ``expression COLLATE collation``. @@ -415,6 +412,11 @@ class ClauseElement(roles.SQLRole, Visitable): """ return self + def _ungroup(self): + """Return this :class:`.ClauseElement` without any groupings.""" + + return self + @util.dependencies("sqlalchemy.engine.default") def compile(self, default, bind=None, dialect=None, **kw): """Compile this SQL expression. @@ -821,6 +823,16 @@ class ColumnElement( and other.name == self.name ) + @util.memoized_property + def _proxy_key(self): + if self.key: + return self.key + else: + try: + return str(self) + except exc.UnsupportedCompilationError: + return self.anon_label + def _make_proxy( self, selectable, name=None, name_is_truncatable=False, **kw ): @@ -831,14 +843,7 @@ class ColumnElement( """ if name is None: name = self.anon_label - if self.key: - key = self.key - else: - try: - key = str(self) - except exc.UnsupportedCompilationError: - key = self.anon_label - + key = self._proxy_key else: key = name co = ColumnClause( @@ -1619,8 +1624,14 @@ class TextClause( @util.dependencies("sqlalchemy.sql.selectable") def columns(self, selectable, *cols, **types): - """Turn this :class:`.TextClause` object into a :class:`.TextAsFrom` - object that can be embedded into another statement. + r"""Turn this :class:`.TextClause` object into a + :class:`.TextualSelect` object that serves the same role as a SELECT + statement. + + The :class:`.TextualSelect` is part of the :class:`.SelectBase` + hierarchy and can be embedded into another statement by using the + :meth:`.TextualSelect.subquery` method to produce a :class:`.Subquery` + object, which can then be SELECTed from. This function essentially bridges the gap between an entirely textual SELECT statement and the SQL expression language concept @@ -1629,7 +1640,7 @@ class TextClause( from sqlalchemy.sql import column, text stmt = text("SELECT id, name FROM some_table") - stmt = stmt.columns(column('id'), column('name')).alias('st') + stmt = stmt.columns(column('id'), column('name')).subquery('st') stmt = select([mytable]).\ select_from( @@ -1638,8 +1649,10 @@ class TextClause( Above, we pass a series of :func:`.column` elements to the :meth:`.TextClause.columns` method positionally. These :func:`.column` - elements now become first class elements upon the :attr:`.TextAsFrom.c` - column collection, just like any other selectable. + elements now become first class elements upon the + :attr:`.TextualSelect.selected_columns` column collection, which then + become part of the :attr:`.Subquery.c` collection after + :meth:`.TextualSelect.subquery` is invoked. The column expressions we pass to :meth:`.TextClause.columns` may also be typed; when we do so, these :class:`.TypeEngine` objects become @@ -1697,17 +1710,22 @@ class TextClause( the column expressions are passed purely positionally. The :meth:`.TextClause.columns` method provides a direct - route to calling :meth:`.FromClause.alias` as well as + route to calling :meth:`.FromClause.subquery` as well as :meth:`.SelectBase.cte` against a textual SELECT statement:: stmt = stmt.columns(id=Integer, name=String).cte('st') stmt = select([sometable]).where(sometable.c.id == stmt.c.id) - .. versionadded:: 0.9.0 :func:`.text` can now be converted into a - fully featured "selectable" construct using the - :meth:`.TextClause.columns` method. + :param \*cols: A series of :class:`.ColumnElement` objects, typically + :class:`.Column` objects from a :class:`.Table` or ORM level + column-mapped attributes, representing a set of columns that this + textual string will SELECT from. + :param \**types: A mapping of string names to :class:`.TypeEngine` + type objects indicating the datatypes to use for names that are + SELECTed from the textual string. Prefer to use the ``\*cols`` + argument as it also indicates positional ordering. """ positional_input_cols = [ @@ -1720,7 +1738,7 @@ class TextClause( ColumnClause(key, type_) for key, type_ in types.items() ] - return selectable.TextAsFrom( + return selectable.TextualSelect( self, positional_input_cols + keyed_input_cols, positional=bool(positional_input_cols) and not keyed_input_cols, @@ -3291,19 +3309,26 @@ class IndexExpression(BinaryExpression): pass -class Grouping(ColumnElement): - """Represent a grouping within a column expression""" +class GroupedElement(ClauseElement): + """Represent any parenthesized expression""" __visit_name__ = "grouping" - def __init__(self, element): - self.element = element - self.type = getattr(element, "type", type_api.NULLTYPE) - def self_group(self, against=None): # type: (Optional[Any]) -> ClauseElement return self + def _ungroup(self): + return self.element._ungroup() + + +class Grouping(GroupedElement, ColumnElement): + """Represent a grouping within a column expression""" + + def __init__(self, element): + self.element = element + self.type = getattr(element, "type", type_api.NULLTYPE) + @util.memoized_property def _is_implicitly_boolean(self): return self.element._is_implicitly_boolean @@ -4351,14 +4376,6 @@ class quoted_name(util.MemoizedSlots, util.text_type): return "'%s'" % backslashed -def _expand_cloned(elements): - """expand the given set of ClauseElements to be the set of all 'cloned' - predecessors. - - """ - return itertools.chain(*[x._cloned_set for x in elements]) - - def _select_iterables(elements): """expand tables into individual columns in the given list of column expressions. @@ -4367,26 +4384,6 @@ def _select_iterables(elements): return itertools.chain(*[c._select_iterable for c in elements]) -def _cloned_intersection(a, b): - """return the intersection of sets a and b, counting - any overlap between 'cloned' predecessors. - - The returned set is in terms of the entities present within 'a'. - - """ - all_overlap = set(_expand_cloned(a)).intersection(_expand_cloned(b)) - return set( - elem for elem in a if all_overlap.intersection(elem._cloned_set) - ) - - -def _cloned_difference(a, b): - all_overlap = set(_expand_cloned(a)).intersection(_expand_cloned(b)) - return set( - elem for elem in a if not all_overlap.intersection(elem._cloned_set) - ) - - def _find_columns(clause): """locate Column objects within the given expression.""" diff --git a/lib/sqlalchemy/sql/expression.py b/lib/sqlalchemy/sql/expression.py index b04355cf5..7ce822669 100644 --- a/lib/sqlalchemy/sql/expression.py +++ b/lib/sqlalchemy/sql/expression.py @@ -16,6 +16,7 @@ class. __all__ = [ "Alias", + "AliasedReturnsRows", "any_", "all_", "ClauseElement", @@ -76,6 +77,7 @@ __all__ = [ "union_all", "update", "within_group", + "Subquery", "TableSample", "tablesample", ] @@ -132,6 +134,7 @@ from .functions import Function # noqa from .functions import FunctionElement # noqa from .functions import modifier # noqa from .selectable import Alias # noqa +from .selectable import AliasedReturnsRows # noqa from .selectable import CompoundSelect # noqa from .selectable import CTE # noqa from .selectable import Exists # noqa @@ -148,10 +151,12 @@ from .selectable import ScalarSelect # noqa from .selectable import Select # noqa from .selectable import Selectable # noqa from .selectable import SelectBase # noqa +from .selectable import Subquery # noqa from .selectable import subquery # noqa from .selectable import TableClause # noqa from .selectable import TableSample # noqa from .selectable import TextAsFrom # noqa +from .selectable import TextualSelect # noqa from .visitors import Visitable # noqa from ..util.langhelpers import public_factory # noqa diff --git a/lib/sqlalchemy/sql/roles.py b/lib/sqlalchemy/sql/roles.py index 053bd7146..55c52d401 100644 --- a/lib/sqlalchemy/sql/roles.py +++ b/lib/sqlalchemy/sql/roles.py @@ -95,6 +95,8 @@ class InElementRole(SQLRole): class FromClauseRole(ColumnsClauseRole): _role_name = "FROM expression, such as a Table or alias() object" + _is_subquery = False + @property def _hide_froms(self): raise NotImplementedError() @@ -134,7 +136,7 @@ class StatementRole(CoerceTextStatementRole): class ReturnsRowsRole(StatementRole): _role_name = ( - "Row returning expression such as a SELECT, or an " + "Row returning expression such as a SELECT, a FROM clause, or an " "INSERT/UPDATE/DELETE with RETURNING" ) @@ -142,6 +144,12 @@ class ReturnsRowsRole(StatementRole): class SelectStatementRole(ReturnsRowsRole): _role_name = "SELECT construct or equivalent text() construct" + def subquery(self): + raise NotImplementedError( + "All SelectStatementRole objects should implement a " + ".subquery() method." + ) + class HasCTERole(ReturnsRowsRole): pass diff --git a/lib/sqlalchemy/sql/selectable.py b/lib/sqlalchemy/sql/selectable.py index 591086a46..2263073c4 100644 --- a/lib/sqlalchemy/sql/selectable.py +++ b/lib/sqlalchemy/sql/selectable.py @@ -12,7 +12,6 @@ SQL tables and derived rowsets. import collections import itertools -import operator from operator import attrgetter from . import coercions @@ -20,6 +19,10 @@ from . import operators from . import roles from . import type_api from .annotation import Annotated +from .base import _clone +from .base import _cloned_difference +from .base import _cloned_intersection +from .base import _expand_cloned from .base import _from_objects from .base import _generative from .base import ColumnCollection @@ -27,17 +30,15 @@ from .base import ColumnSet from .base import Executable from .base import Generative from .base import Immutable +from .base import SeparateKeyColumnCollection from .coercions import _document_text_coercion from .elements import _anonymous_label -from .elements import _clone -from .elements import _cloned_difference -from .elements import _cloned_intersection -from .elements import _expand_cloned from .elements import _select_iterables from .elements import and_ from .elements import BindParameter from .elements import ClauseElement from .elements import ClauseList +from .elements import GroupedElement from .elements import Grouping from .elements import literal_column from .elements import True_ @@ -52,20 +53,22 @@ class _OffsetLimitParam(BindParameter): return self.effective_value +@util.deprecated( + "1.4", + "The standalone :func:`.subquery` function is deprecated " + "and will be removed in a future release. Use select().subquery().", +) def subquery(alias, *args, **kwargs): - r"""Return an :class:`.Alias` object derived + r"""Return an :class:`.Subquery` object derived from a :class:`.Select`. - name - alias name + :param name: the alias name for the subquery - \*args, \**kwargs - - all other arguments are delivered to the - :func:`select` function. + :param \*args, \**kwargs: all other arguments are passed through to the + :func:`.select` function. """ - return Select(*args, **kwargs).alias(alias) + return Select(*args, **kwargs).subquery(alias) class ReturnsRows(roles.ReturnsRowsRole, ClauseElement): @@ -90,19 +93,12 @@ class ReturnsRows(roles.ReturnsRowsRole, ClauseElement): @property def selectable(self): - raise NotImplementedError( - "This object is a base ReturnsRows object, but is not a " - "FromClause so has no .c. collection." - ) + raise NotImplementedError() class Selectable(ReturnsRows): """mark a class as being selectable. - This class is legacy as of 1.4 as the concept of a SQL construct which - "returns rows" is more generalized than one which can be the subject - of a SELECT. - """ __visit_name__ = "selectable" @@ -113,6 +109,93 @@ class Selectable(ReturnsRows): def selectable(self): return self + @property + def exported_columns(self): + """A :class:`.ColumnCollection` that represents the "exported" + columns of this :class:`.Selectable`. + + The "exported" columns represent the collection of + :class:`.ColumnElement` expressions that are rendered by this SQL + construct. There are two primary varieties which are the + "FROM clause columns" of a FROM clause, such as a table, join, + or subquery, and the "SELECTed columns", which are the columns in + the "columns clause" of a SELECT statement. + + .. versionadded:: 1.4 + + .. seealso: + + :attr:`.FromClause.exported_columns` + + :attr:`.SelectBase.exported_columns` + """ + + raise NotImplementedError() + + def _refresh_for_new_column(self, column): + raise NotImplementedError() + + def lateral(self, name=None): + """Return a LATERAL alias of this :class:`.Selectable`. + + The return value is the :class:`.Lateral` construct also + provided by the top-level :func:`~.expression.lateral` function. + + .. versionadded:: 1.1 + + .. seealso:: + + :ref:`lateral_selects` - overview of usage. + + """ + return Lateral._construct(self, name) + + @util.deprecated( + "1.4", + message="The :meth:`.Selectable.replace_selectable` method is " + "deprecated, and will be removed in a future release. Similar " + "functionality is available via the sqlalchemy.sql.visitors module.", + ) + @util.dependencies("sqlalchemy.sql.util") + def replace_selectable(self, sqlutil, old, alias): + """replace all occurrences of FromClause 'old' with the given Alias + object, returning a copy of this :class:`.FromClause`. + + """ + + return sqlutil.ClauseAdapter(alias).traverse(self) + + def corresponding_column(self, column, require_embedded=False): + """Given a :class:`.ColumnElement`, return the exported + :class:`.ColumnElement` object from the + :attr:`.Selectable.exported_columns` + collection of this :class:`.Selectable` which corresponds to that + original :class:`.ColumnElement` via a common ancestor + column. + + :param column: the target :class:`.ColumnElement` to be matched + + :param require_embedded: only return corresponding columns for + the given :class:`.ColumnElement`, if the given + :class:`.ColumnElement` is actually present within a sub-element + of this :class:`.Selectable`. Normally the column will match if + it merely shares a common ancestor with one of the exported + columns of this :class:`.Selectable`. + + .. seealso:: + + :attr:`.Selectable.exported_columns` - the + :class:`.ColumnCollection` that is used for the operation + + :meth:`.ColumnCollection.corresponding_column` - implementation + method. + + """ + + return self.exported_columns.corresponding_column( + column, require_embedded + ) + class HasPrefixes(object): _prefixes = () @@ -211,7 +294,7 @@ class HasSuffixes(object): ) -class FromClause(roles.FromClauseRole, Selectable): +class FromClause(roles.AnonymizedFromClauseRole, Selectable): """Represent an element that can be used within the ``FROM`` clause of a ``SELECT`` statement. @@ -243,9 +326,8 @@ class FromClause(roles.FromClauseRole, Selectable): """ - is_selectable = has_selectable = True + is_selectable = True _is_from_clause = True - _is_text_as_from = False _is_join = False def _translate_schema(self, effective_schema, map_): @@ -400,21 +482,6 @@ class FromClause(roles.FromClauseRole, Selectable): return Alias._construct(self, name) - def lateral(self, name=None): - """Return a LATERAL alias of this :class:`.FromClause`. - - The return value is the :class:`.Lateral` construct also - provided by the top-level :func:`~.expression.lateral` function. - - .. versionadded:: 1.1 - - .. seealso:: - - :ref:`lateral_selects` - overview of usage. - - """ - return Lateral._construct(self, name) - def tablesample(self, sampling, name=None, seed=None): """Return a TABLESAMPLE alias of this :class:`.FromClause`. @@ -452,125 +519,43 @@ class FromClause(roles.FromClauseRole, Selectable): """ return self._cloned_set.intersection(other._cloned_set) - @util.dependencies("sqlalchemy.sql.util") - def replace_selectable(self, sqlutil, old, alias): - """replace all occurrences of FromClause 'old' with the given Alias - object, returning a copy of this :class:`.FromClause`. - - """ - - return sqlutil.ClauseAdapter(alias).traverse(self) + @property + def description(self): + """a brief description of this FromClause. - def correspond_on_equivalents(self, column, equivalents): - """Return corresponding_column for the given column, or if None - search for a match in the given dictionary. + Used primarily for error message formatting. """ - col = self.corresponding_column(column, require_embedded=True) - if col is None and col in equivalents: - for equiv in equivalents[col]: - nc = self.corresponding_column(equiv, require_embedded=True) - if nc: - return nc - return col + return getattr(self, "name", self.__class__.__name__ + " object") - def corresponding_column(self, column, require_embedded=False): - """Given a :class:`.ColumnElement`, return the exported - :class:`.ColumnElement` object from this :class:`.Selectable` - which corresponds to that original - :class:`~sqlalchemy.schema.Column` via a common ancestor - column. + def _reset_exported(self): + """delete memoized collections when a FromClause is cloned.""" - :param column: the target :class:`.ColumnElement` to be matched + self._memoized_property.expire_instance(self) - :param require_embedded: only return corresponding columns for - the given :class:`.ColumnElement`, if the given - :class:`.ColumnElement` is actually present within a sub-element - of this :class:`.FromClause`. Normally the column will match if - it merely shares a common ancestor with one of the exported - columns of this :class:`.FromClause`. + def _generate_fromclause_column_proxies(self, fromclause): + for col in self.c: + col._make_proxy(fromclause) - """ + @property + def exported_columns(self): + """A :class:`.ColumnCollection` that represents the "exported" + columns of this :class:`.Selectable`. - def embedded(expanded_proxy_set, target_set): - for t in target_set.difference(expanded_proxy_set): - if not set(_expand_cloned([t])).intersection( - expanded_proxy_set - ): - return False - return True + The "exported" columns for a :class:`.FromClause` object are synonymous + with the :attr:`.FromClause.columns` collection. - # don't dig around if the column is locally present - if self.c.contains_column(column): - return column - col, intersect = None, None - target_set = column.proxy_set - cols = self.c._all_columns - for c in cols: - expanded_proxy_set = set(_expand_cloned(c.proxy_set)) - i = target_set.intersection(expanded_proxy_set) - if i and ( - not require_embedded - or embedded(expanded_proxy_set, target_set) - ): - if col is None: - - # no corresponding column yet, pick this one. - - col, intersect = c, i - elif len(i) > len(intersect): - - # 'c' has a larger field of correspondence than - # 'col'. i.e. selectable.c.a1_x->a1.c.x->table.c.x - # matches a1.c.x->table.c.x better than - # selectable.c.x->table.c.x does. - - col, intersect = c, i - elif i == intersect: - - # they have the same field of correspondence. see - # which proxy_set has fewer columns in it, which - # indicates a closer relationship with the root - # column. Also take into account the "weight" - # attribute which CompoundSelect() uses to give - # higher precedence to columns based on vertical - # position in the compound statement, and discard - # columns that have no reference to the target - # column (also occurs with CompoundSelect) - - col_distance = util.reduce( - operator.add, - [ - sc._annotations.get("weight", 1) - for sc in col._uncached_proxy_set() - if sc.shares_lineage(column) - ], - ) - c_distance = util.reduce( - operator.add, - [ - sc._annotations.get("weight", 1) - for sc in c._uncached_proxy_set() - if sc.shares_lineage(column) - ], - ) - if c_distance < col_distance: - col, intersect = c, i - return col + .. versionadded:: 1.4 - @property - def description(self): - """a brief description of this FromClause. + .. seealso: - Used primarily for error message formatting. + :attr:`.Selectable.exported_columns` - """ - return getattr(self, "name", self.__class__.__name__ + " object") + :attr:`.SelectBase.exported_columns` - def _reset_exported(self): - """delete memoized collections when a FromClause is cloned.""" - self._memoized_property.expire_instance(self) + """ + return self.columns @_memoized_property def columns(self): @@ -660,15 +645,10 @@ class FromClause(roles.FromClauseRole, Selectable): derivations. """ - if not self._cols_populated: - return None - elif column.key in self.columns and self.columns[column.key] is column: - return column - else: - return None + self._reset_exported() -class Join(roles.AnonymizedFromClauseRole, FromClause): +class Join(FromClause): """represent a ``JOIN`` construct between two :class:`.FromClause` elements. @@ -811,23 +791,15 @@ class Join(roles.AnonymizedFromClauseRole, FromClause): (c for c in columns if c.primary_key), self.onclause ) ) - self._columns.update((col._label, col) for col in columns) + self._columns.update((col._key_label, col) for col in columns) self.foreign_keys.update( itertools.chain(*[col.foreign_keys for col in columns]) ) def _refresh_for_new_column(self, column): - col = self.left._refresh_for_new_column(column) - if col is None: - col = self.right._refresh_for_new_column(column) - if col is not None: - if self._cols_populated: - self._columns[col._label] = col - self.foreign_keys.update(col.foreign_keys) - if col.primary_key: - self.primary_key.add(col) - return col - return None + super(Join, self)._refresh_for_new_column(column) + self.left._refresh_for_new_column(column) + self.right._refresh_for_new_column(column) def _copy_internals(self, clone=_clone, **kw): self._reset_exported() @@ -947,6 +919,9 @@ class Join(roles.AnonymizedFromClauseRole, FromClause): def _joincond_scan_left_right( cls, a, a_subset, b, consider_as_foreign_keys ): + a = coercions.expect(roles.FromClauseRole, a) + b = coercions.expect(roles.FromClauseRole, b) + constraints = collections.defaultdict(list) for left in (a_subset, a): @@ -1186,23 +1161,20 @@ class Join(roles.AnonymizedFromClauseRole, FromClause): ) -class Alias(roles.AnonymizedFromClauseRole, FromClause): - """Represents an table or selectable alias (AS). - - Represents an alias, as typically applied to any table or - sub-select within a SQL statement using the ``AS`` keyword (or - without the keyword on certain databases such as Oracle). - - This object is constructed from the :func:`~.expression.alias` module - level function as well as the :meth:`.FromClause.alias` method available - on all :class:`.FromClause` subclasses. - - """ - - __visit_name__ = "alias" - named_with_column = True +# FromClause -> +# AliasedReturnsRows +# -> Alias only for FromClause +# -> Subquery only for SelectBase +# -> CTE only for HasCTE -> SelectBase, DML +# -> Lateral -> FromClause, but we accept SelectBase +# w/ non-deprecated coercion +# -> TableSample -> only for FromClause +class AliasedReturnsRows(FromClause): + """Base class of aliases against tables, subqueries, and other + selectables.""" _is_from_container = True + named_with_column = True def __init__(self, *arg, **kw): raise NotImplementedError( @@ -1224,59 +1196,12 @@ class Alias(roles.AnonymizedFromClauseRole, FromClause): return obj @classmethod - def _factory(cls, selectable, name=None, flat=False): - """Return an :class:`.Alias` object. - - An :class:`.Alias` represents any :class:`.FromClause` - with an alternate name assigned within SQL, typically using the ``AS`` - clause when generated, e.g. ``SELECT * FROM table AS aliasname``. - - Similar functionality is available via the - :meth:`~.FromClause.alias` method - available on all :class:`.FromClause` subclasses. In terms of a - SELECT object as generated from the :func:`.select` function, the - :meth:`.SelectBase.alias` method returns an :class:`.Alias` or - similar object which represents a named, parenthesized subquery. - - When an :class:`.Alias` is created from a :class:`.Table` object, - this has the effect of the table being rendered - as ``tablename AS aliasname`` in a SELECT statement. - - For :func:`.select` objects, the effect is that of creating a named - subquery, i.e. ``(select ...) AS aliasname``. - - The ``name`` parameter is optional, and provides the name - to use in the rendered SQL. If blank, an "anonymous" name - will be deterministically generated at compile time. - Deterministic means the name is guaranteed to be unique against - other constructs used in the same statement, and will also be the - same name for each successive compilation of the same statement - object. - - :param selectable: any :class:`.FromClause` subclass, - such as a table, select statement, etc. - - :param name: string name to be assigned as the alias. - If ``None``, a name will be deterministically generated - at compile time. - - :param flat: Will be passed through to if the given selectable - is an instance of :class:`.Join` - see :meth:`.Join.alias` - for details. - - .. versionadded:: 0.9.0 - - """ - return coercions.expect(roles.FromClauseRole, selectable).alias( - name=name, flat=flat - ) + def _factory(cls, returnsrows, name=None): + """Base factory method. Subclasses need to provide this.""" + raise NotImplementedError() def _init(self, selectable, name=None): - self.wrapped = selectable - if isinstance(selectable, Alias): - selectable = selectable.element - assert not isinstance(selectable, Alias) - + self.element = selectable self.supports_execution = selectable.supports_execution if self.supports_execution: self._execution_options = selectable._execution_options @@ -1288,18 +1213,14 @@ class Alias(roles.AnonymizedFromClauseRole, FromClause): and selectable.named_with_column ): name = getattr(selectable, "name", None) + if isinstance(name, _anonymous_label): + name = None name = _anonymous_label("%%(%d %s)s" % (id(self), name or "anon")) self.name = name - def self_group(self, against=None): - if ( - isinstance(against, CompoundSelect) - and isinstance(self.element, Select) - and self.element._needs_parens_for_grouping() - ): - return FromGrouping(self) - - return super(Alias, self).self_group(against=against) + def _refresh_for_new_column(self, column): + super(AliasedReturnsRows, self)._refresh_for_new_column(column) + self.element._refresh_for_new_column(column) @property def description(self): @@ -1319,18 +1240,7 @@ class Alias(roles.AnonymizedFromClauseRole, FromClause): return self.element.is_derived_from(fromclause) def _populate_column_collection(self): - for col in self.wrapped.columns._all_columns: - col._make_proxy(self) - - def _refresh_for_new_column(self, column): - col = self.wrapped._refresh_for_new_column(column) - if col is not None: - if not self._cols_populated: - return None - else: - return col._make_proxy(self) - else: - return None + self.element._generate_fromclause_column_proxies(self) def _copy_internals(self, clone=_clone, **kw): # don't apply anything to an aliased Table @@ -1339,17 +1249,13 @@ class Alias(roles.AnonymizedFromClauseRole, FromClause): if isinstance(self.element, TableClause): return self._reset_exported() - self.wrapped = clone(self.wrapped, **kw) - if isinstance(self.wrapped, Alias): - self.element = self.wrapped.element - else: - self.element = self.wrapped + self.element = clone(self.element, **kw) def get_children(self, column_collections=True, **kw): if column_collections: for c in self.c: yield c - yield self.wrapped + yield self.element def _cache_key(self, **kw): return (self.__class__, self.element._cache_key(**kw), self._orig_name) @@ -1363,7 +1269,71 @@ class Alias(roles.AnonymizedFromClauseRole, FromClause): return self.element.bind -class Lateral(Alias): +class Alias(AliasedReturnsRows): + """Represents an table or selectable alias (AS). + + Represents an alias, as typically applied to any table or + sub-select within a SQL statement using the ``AS`` keyword (or + without the keyword on certain databases such as Oracle). + + This object is constructed from the :func:`~.expression.alias` module + level function as well as the :meth:`.FromClause.alias` method available + on all :class:`.FromClause` subclasses. + + """ + + __visit_name__ = "alias" + + @classmethod + def _factory(cls, selectable, name=None, flat=False): + """Return an :class:`.Alias` object. + + An :class:`.Alias` represents any :class:`.FromClause` + with an alternate name assigned within SQL, typically using the ``AS`` + clause when generated, e.g. ``SELECT * FROM table AS aliasname``. + + Similar functionality is available via the + :meth:`~.FromClause.alias` method + available on all :class:`.FromClause` subclasses. In terms of a + SELECT object as generated from the :func:`.select` function, the + :meth:`.SelectBase.alias` method returns an :class:`.Alias` or + similar object which represents a named, parenthesized subquery. + + When an :class:`.Alias` is created from a :class:`.Table` object, + this has the effect of the table being rendered + as ``tablename AS aliasname`` in a SELECT statement. + + For :func:`.select` objects, the effect is that of creating a named + subquery, i.e. ``(select ...) AS aliasname``. + + The ``name`` parameter is optional, and provides the name + to use in the rendered SQL. If blank, an "anonymous" name + will be deterministically generated at compile time. + Deterministic means the name is guaranteed to be unique against + other constructs used in the same statement, and will also be the + same name for each successive compilation of the same statement + object. + + :param selectable: any :class:`.FromClause` subclass, + such as a table, select statement, etc. + + :param name: string name to be assigned as the alias. + If ``None``, a name will be deterministically generated + at compile time. + + :param flat: Will be passed through to if the given selectable + is an instance of :class:`.Join` - see :meth:`.Join.alias` + for details. + + .. versionadded:: 0.9.0 + + """ + return coercions.expect( + roles.FromClauseRole, selectable, allow_select=True + ).alias(name=name, flat=flat) + + +class Lateral(AliasedReturnsRows): """Represent a LATERAL subquery. This object is constructed from the :func:`~.expression.lateral` module @@ -1404,12 +1374,12 @@ class Lateral(Alias): :ref:`lateral_selects` - overview of usage. """ - return coercions.expect(roles.FromClauseRole, selectable).lateral( - name=name - ) + return coercions.expect( + roles.FromClauseRole, selectable, explicit_subquery=True + ).lateral(name=name) -class TableSample(Alias): +class TableSample(AliasedReturnsRows): """Represent a TABLESAMPLE clause. This object is constructed from the :func:`~.expression.tablesample` module @@ -1485,7 +1455,7 @@ class TableSample(Alias): return functions.func.system(self.sampling) -class CTE(Generative, HasSuffixes, Alias): +class CTE(Generative, HasSuffixes, AliasedReturnsRows): """Represent a Common Table Expression. The :class:`.CTE` object is obtained using the @@ -1531,15 +1501,6 @@ class CTE(Generative, HasSuffixes, Alias): [clone(elem, **kw) for elem in self._restates] ) - @util.dependencies("sqlalchemy.sql.dml") - def _populate_column_collection(self, dml): - if isinstance(self.element, dml.UpdateBase): - for col in self.element._returning: - col._make_proxy(self) - else: - for col in self.element.columns._all_columns: - col._make_proxy(self) - def alias(self, name=None, flat=False): """Return an :class:`.Alias` of this :class:`.CTE`. @@ -1748,13 +1709,26 @@ class HasCTE(roles.HasCTERole): return CTE._construct(self, name=name, recursive=recursive) -class FromGrouping(FromClause): - """Represent a grouping of a FROM clause""" +class Subquery(AliasedReturnsRows): + __visit_name__ = "subquery" - __visit_name__ = "grouping" + _is_subquery = True + + @classmethod + def _factory(cls, selectable, name=None): + """Return a :class:`.Subquery` object. + + """ + return coercions.expect( + roles.SelectStatementRole, selectable + ).subquery(name=name) + + +class FromGrouping(GroupedElement, FromClause): + """Represent a grouping of a FROM clause""" def __init__(self, element): - self.element = element + self.element = coercions.expect(roles.FromClauseRole, element) def _init_collections(self): pass @@ -1794,9 +1768,6 @@ class FromGrouping(FromClause): def _from_objects(self): return self.element._from_objects - def __getattr__(self, attr): - return getattr(self.element, attr) - def __getstate__(self): return {"element": self.element} @@ -1804,7 +1775,7 @@ class FromGrouping(FromClause): self.element = state["element"] -class TableClause(roles.AnonymizedFromClauseRole, Immutable, FromClause): +class TableClause(Immutable, FromClause): """Represents a minimal "table" construct. This is a lightweight table object that has only a name and a @@ -1870,6 +1841,9 @@ class TableClause(roles.AnonymizedFromClauseRole, Immutable, FromClause): for c in columns: self.append_column(c) + def _refresh_for_new_column(self, column): + pass + def _init_collections(self): pass @@ -2065,19 +2039,122 @@ class SelectBase( roles.InElementRole, HasCTE, Executable, - FromClause, + Selectable, ): """Base class for SELECT statements. This includes :class:`.Select`, :class:`.CompoundSelect` and - :class:`.TextAsFrom`. + :class:`.TextualSelect`. """ _is_select_statement = True + _memoized_property = util.group_expirable_memoized_property() + + def _reset_memoizations(self): + self._memoized_property.expire_instance(self) + + def _generate_fromclause_column_proxies(self, fromclause): + # type: (FromClause) + raise NotImplementedError() + + def _refresh_for_new_column(self, column): + self._reset_memoizations() + + @property + def selected_columns(self): + """A :class:`.ColumnCollection` representing the columns that + this SELECT statement or similar construct returns in its result set. + + This collection differs from the :attr:`.FromClause.columns` collection + of a :class:`.FromClause` in that the columns within this collection + cannot be directly nested inside another SELECT statement; a subquery + must be applied first which provides for the necessary parenthesization + required by SQL. + + .. versionadded:: 1.4 + + """ + raise NotImplementedError() + + @property + def exported_columns(self): + """A :class:`.ColumnCollection` that represents the "exported" + columns of this :class:`.Selectable`. + + The "exported" columns for a :class:`.SelectBase` object are synonymous + with the :attr:`.SelectBase.selected_columns` collection. + + .. versionadded:: 1.4 + + .. seealso: + + :attr:`.Selectable.exported_columns` + + :attr:`.FromClause.exported_columns` + + + """ + return self.selected_columns + + @property + @util.deprecated( + "1.4", + "The :attr:`.SelectBase.c` and :attr:`.SelectBase.columns` attributes " + "are deprecated and will be removed in a future release; these " + "attributes implicitly create a subquery that should be explicit. " + "Please call :meth:`.SelectBase.subquery` first in order to create " + "a subquery, which then contains this attribute. To access the " + "columns that this SELECT object SELECTs " + "from, use the :attr:`.SelectBase.selected_columns` attribute.", + ) + def c(self): + return self._implicit_subquery.columns + + @property + def columns(self): + return self.c + + @util.deprecated( + "1.4", + "The :meth:`.SelectBase.select` method is deprecated " + "and will be removed in a future release; this method implicitly " + "creates a subquery that should be explicit. " + "Please call :meth:`.SelectBase.subquery` first in order to create " + "a subquery, which then can be seleted.", + ) + def select(self, *arg, **kw): + return self._implicit_subquery.select(*arg, **kw) + + @util.deprecated( + "1.4", + "The :meth:`.SelectBase.join` method is deprecated " + "and will be removed in a future release; this method implicitly " + "creates a subquery that should be explicit. " + "Please call :meth:`.SelectBase.subquery` first in order to create " + "a subquery, which then can be seleted.", + ) + def join(self, *arg, **kw): + return self._implicit_subquery.join(*arg, **kw) + + @util.deprecated( + "1.4", + "The :meth:`.SelectBase.outerjoin` method is deprecated " + "and will be removed in a future release; this method implicitly " + "creates a subquery that should be explicit. " + "Please call :meth:`.SelectBase.subquery` first in order to create " + "a subquery, which then can be seleted.", + ) + def outerjoin(self, *arg, **kw): + return self._implicit_subquery.outerjoin(*arg, **kw) + + @_memoized_property + def _implicit_subquery(self): + return self.subquery() + @util.deprecated( "1.4", "The :meth:`.SelectBase.as_scalar` method is deprecated and will be " @@ -2117,6 +2194,21 @@ class SelectBase( """ return self.scalar_subquery().label(name) + def lateral(self, name=None): + """Return a LATERAL alias of this :class:`.Selectable`. + + The return value is the :class:`.Lateral` construct also + provided by the top-level :func:`~.expression.lateral` function. + + .. versionadded:: 1.1 + + .. seealso:: + + :ref:`lateral_selects` - overview of usage. + + """ + return Lateral._factory(self, name) + @_generative @util.deprecated( "0.6", @@ -2141,7 +2233,7 @@ class SelectBase( s = self.__class__.__new__(self.__class__) s.__dict__ = self.__dict__.copy() - s._reset_exported() + s._reset_memoizations() return s @property @@ -2182,7 +2274,78 @@ class SelectBase( .. versionadded:: 1.4 """ - return self.alias() + return Subquery._construct(self, name) + + def alias(self, name=None, flat=False): + """Return a named subquery against this :class:`.SelectBase`. + + For a :class:`.SelectBase` (as opposed to a :class:`.FromClause`), + this returns a :class:`.Subquery` object which behaves mostly the + same as the :class:`.Alias` object that is used with a + :class:`.FromClause`. + + .. versionchanged:: 1.4 The :meth:`.SelectBase.alias` method is now + a synonym for the :meth:`.SelectBase.subquery` method. + + """ + return self.subquery(name=name) + + +class SelectStatementGrouping(GroupedElement, SelectBase): + """Represent a grouping of a :class:`.SelectBase`. + + This differs from :class:`.Subquery` in that we are still + an "inner" SELECT statement, this is strictly for grouping inside of + compound selects. + + """ + + __visit_name__ = "grouping" + + def __init__(self, element): + # type: (SelectBase) + self.element = coercions.expect(roles.SelectStatementRole, element) + + @property + def select_statement(self): + return self.element + + def get_children(self, **kwargs): + return (self.element,) + + def self_group(self, against=None): + # type: (Optional[Any]) -> FromClause + return self + + def _generate_fromclause_column_proxies(self, subquery): + self.element._generate_fromclause_column_proxies(subquery) + + def _generate_proxy_for_new_column(self, column, subquery): + return self.element._generate_proxy_for_new_column(subquery) + + @property + def selected_columns(self): + """A :class:`.ColumnCollection` representing the columns that + the embedded SELECT statement returns in its result set. + + .. versionadded:: 1.4 + + .. seealso:: + + :ref:`.SelectBase.selected_columns` + + """ + return self.element.selected_columns + + def _copy_internals(self, clone=_clone, **kw): + self.element = clone(self.element, **kw) + + def _cache_key(self, **kw): + return (SelectStatementGrouping, self.element._cache_key(**kw)) + + @property + def _from_objects(self): + return self.element._from_objects class GenerativeSelect(SelectBase): @@ -2191,7 +2354,7 @@ class GenerativeSelect(SelectBase): This serves as the base for :class:`.Select` and :class:`.CompoundSelect` where elements such as ORDER BY, GROUP BY can be added and column - rendering can be controlled. Compare to :class:`.TextAsFrom`, which, + rendering can be controlled. Compare to :class:`.TextualSelect`, which, while it subclasses :class:`.SelectBase` and is also a SELECT construct, represents a fixed textual string which cannot be altered at this level, only wrapped as a subquery. @@ -2199,7 +2362,7 @@ class GenerativeSelect(SelectBase): .. versionadded:: 0.9.0 :class:`.GenerativeSelect` was added to provide functionality specific to :class:`.Select` and :class:`.CompoundSelect` while allowing :class:`.SelectBase` to be - used for other SELECT-like objects, e.g. :class:`.TextAsFrom`. + used for other SELECT-like objects, e.g. :class:`.TextualSelect`. """ @@ -2591,8 +2754,8 @@ class CompoundSelect(GenerativeSelect): s = coercions.expect(roles.CompoundElementRole, s) if not numcols: - numcols = len(s.c._all_columns) - elif len(s.c._all_columns) != numcols: + numcols = len(s.selected_columns) + elif len(s.selected_columns) != numcols: raise exc.ArgumentError( "All selectables passed to " "CompoundSelect must have identical numbers of " @@ -2600,9 +2763,9 @@ class CompoundSelect(GenerativeSelect): "#%d has %d" % ( 1, - len(self.selects[0].c._all_columns), + len(self.selects[0].selected_columns), n + 1, - len(s.c._all_columns), + len(s.selected_columns), ) ) @@ -2610,9 +2773,12 @@ class CompoundSelect(GenerativeSelect): GenerativeSelect.__init__(self, **kwargs) - @property + @SelectBase._memoized_property def _label_resolve_dict(self): - d = dict((c.key, c) for c in self.c) + # TODO: this is hacky and slow + hacky_subquery = self.subquery() + hacky_subquery.named_with_column = False + d = dict((c.key, c) for c in hacky_subquery.c) return d, d, d @classmethod @@ -2727,7 +2893,8 @@ class CompoundSelect(GenerativeSelect): return self.selects[0]._scalar_type() def self_group(self, against=None): - return FromGrouping(self) + # type: (Optional[Any]) -> FromClause + return SelectStatementGrouping(self) def is_derived_from(self, fromclause): for s in self.selects: @@ -2735,50 +2902,59 @@ class CompoundSelect(GenerativeSelect): return True return False - def _populate_column_collection(self): - for cols in zip(*[s.c._all_columns for s in self.selects]): - - # this is a slightly hacky thing - the union exports a - # column that resembles just that of the *first* selectable. - # to get at a "composite" column, particularly foreign keys, - # you have to dig through the proxies collection which we - # generate below. We may want to improve upon this, such as - # perhaps _make_proxy can accept a list of other columns - # that are "shared" - schema.column can then copy all the - # ForeignKeys in. this would allow the union() to have all - # those fks too. - - proxy = cols[0]._make_proxy( - self, - name=cols[0]._label if self.use_labels else None, - key=cols[0]._key_label if self.use_labels else None, - ) - - # hand-construct the "_proxies" collection to include all - # derived columns place a 'weight' annotation corresponding - # to how low in the list of select()s the column occurs, so - # that the corresponding_column() operation can resolve - # conflicts - proxy._proxies = [ - c._annotate({"weight": i + 1}) for (i, c) in enumerate(cols) + def _generate_fromclause_column_proxies(self, subquery): + + # this is a slightly hacky thing - the union exports a + # column that resembles just that of the *first* selectable. + # to get at a "composite" column, particularly foreign keys, + # you have to dig through the proxies collection which we + # generate below. We may want to improve upon this, such as + # perhaps _make_proxy can accept a list of other columns + # that are "shared" - schema.column can then copy all the + # ForeignKeys in. this would allow the union() to have all + # those fks too. + select_0 = self.selects[0] + if self.use_labels: + select_0 = select_0.apply_labels() + select_0._generate_fromclause_column_proxies(subquery) + + # hand-construct the "_proxies" collection to include all + # derived columns place a 'weight' annotation corresponding + # to how low in the list of select()s the column occurs, so + # that the corresponding_column() operation can resolve + # conflicts + for subq_col, select_cols in zip( + subquery.c._all_columns, + zip(*[s.selected_columns for s in self.selects]), + ): + subq_col._proxies = [ + c._annotate({"weight": i + 1}) + for (i, c) in enumerate(select_cols) ] def _refresh_for_new_column(self, column): - for s in self.selects: - s._refresh_for_new_column(column) + super(CompoundSelect, self)._refresh_for_new_column(column) + for select in self.selects: + select._refresh_for_new_column(column) - if not self._cols_populated: - return None + @property + def selected_columns(self): + """A :class:`.ColumnCollection` representing the columns that + this SELECT statement or similar construct returns in its result set. - raise NotImplementedError( - "CompoundSelect constructs don't support " - "addition of columns to underlying " - "selectables" - ) + For a :class:`.CompoundSelect`, the + :attr:`.CompoundSelect.selected_columns` attribute returns the selected + columns of the first SELECT statement contined within the series of + statements within the set operation. + + .. versionadded:: 1.4 + + """ + return self.selects[0].selected_columns def _copy_internals(self, clone=_clone, **kw): super(CompoundSelect, self)._copy_internals(clone, **kw) - self._reset_exported() + self._reset_memoizations() self.selects = [clone(s, **kw) for s in self.selects] if hasattr(self, "_col_map"): del self._col_map @@ -2790,11 +2966,9 @@ class CompoundSelect(GenerativeSelect): if getattr(self, attr) is not None: setattr(self, attr, clone(getattr(self, attr), **kw)) - def get_children(self, column_collections=True, **kwargs): - return ( - (column_collections and list(self.c) or []) - + [self._order_by_clause, self._group_by_clause] - + list(self.selects) + def get_children(self, **kwargs): + return [self._order_by_clause, self._group_by_clause] + list( + self.selects ) def _cache_key(self, **kw): @@ -3142,7 +3316,7 @@ class Select(HasPrefixes, HasSuffixes, GenerativeSelect): else (), self._from_obj, ): - if item is self: + if item._is_subquery and item.element is self: raise exc.InvalidRequestError( "select() construct refers to itself as a FROM" ) @@ -3415,16 +3589,15 @@ class Select(HasPrefixes, HasSuffixes, GenerativeSelect): if getattr(self, attr) is not None: setattr(self, attr, clone(getattr(self, attr), **kw)) - # erase exported column list, _froms collection, + # erase _froms collection, # etc. - self._reset_exported() + self._reset_memoizations() - def get_children(self, column_collections=True, **kwargs): + def get_children(self, **kwargs): """return child elements as per the ClauseElement specification.""" return ( - (column_collections and list(self.columns) or []) - + self._raw_columns + self._raw_columns + list(self._froms) + [ x @@ -3594,7 +3767,7 @@ class Select(HasPrefixes, HasSuffixes, GenerativeSelect): asked to select both from ``table1`` as well as itself. """ - self._reset_exported() + self._reset_memoizations() rc = [] for c in columns: c = coercions.expect(roles.ColumnsClauseRole, c) @@ -3789,7 +3962,7 @@ class Select(HasPrefixes, HasSuffixes, GenerativeSelect): :class:`.Select` object. """ - self._reset_exported() + self._reset_memoizations() column = coercions.expect(roles.ColumnsClauseRole, column) if isinstance(column, ScalarSelect): @@ -3821,7 +3994,7 @@ class Select(HasPrefixes, HasSuffixes, GenerativeSelect): """ - self._reset_exported() + self._reset_memoizations() self._whereclause = and_(True_._ifnone(self._whereclause), whereclause) def append_having(self, having): @@ -3835,7 +4008,7 @@ class Select(HasPrefixes, HasSuffixes, GenerativeSelect): :term:`method chaining`. """ - self._reset_exported() + self._reset_memoizations() self._having = and_(True_._ifnone(self._having), having) def append_from(self, fromclause): @@ -3847,11 +4020,61 @@ class Select(HasPrefixes, HasSuffixes, GenerativeSelect): standard :term:`method chaining`. """ - self._reset_exported() + self._reset_memoizations() fromclause = coercions.expect(roles.FromClauseRole, fromclause) self._from_obj = self._from_obj.union([fromclause]) @_memoized_property + def selected_columns(self): + """A :class:`.ColumnCollection` representing the columns that + this SELECT statement or similar construct returns in its result set. + + This collection differs from the :attr:`.FromClause.columns` collection + of a :class:`.FromClause` in that the columns within this collection + cannot be directly nested inside another SELECT statement; a subquery + must be applied first which provides for the necessary parenthesization + required by SQL. + + For a :func:`.select` construct, the collection here is exactly what + would be rendered inside the "SELECT" statement, and the + :class:`.ColumnElement` objects are directly present as they were + given, e.g.:: + + col1 = column('q', Integer) + col2 = column('p', Integer) + stmt = select([col1, col2]) + + Above, ``stmt.selected_columns`` would be a collection that contains + the ``col1`` and ``col2`` objects directly. For a statement that is + against a :class:`.Table` or other :class:`.FromClause`, the collection + will use the :class:`.ColumnElement` objects that are in the + :attr:`.FromClause.c` collection of the from element. + + .. versionadded:: 1.4 + + """ + names = set() + + def name_for_col(c): + # we use key_label since this name is intended for targeting + # within the ColumnCollection only, it's not related to SQL + # rendering which always uses column name for SQL label names + if self.use_labels: + name = c._key_label + else: + name = c._proxy_key + if name in names: + name = c.anon_label + else: + names.add(name) + return name + + return SeparateKeyColumnCollection( + (name_for_col(c), c) + for c in util.unique_list(_select_iterables(self._raw_columns)) + ).as_immutable() + + @_memoized_property def _columns_plus_names(self): if self.use_labels: names = set() @@ -3877,7 +4100,9 @@ class Select(HasPrefixes, HasSuffixes, GenerativeSelect): for c in util.unique_list(_select_iterables(self._raw_columns)) ] - def _populate_column_collection(self): + def _generate_fromclause_column_proxies(self, subquery): + keys_seen = set() + for name, c in self._columns_plus_names: if not hasattr(c, "_make_proxy"): continue @@ -3885,27 +4110,15 @@ class Select(HasPrefixes, HasSuffixes, GenerativeSelect): key = None elif self.use_labels: key = c._key_label - if key is not None and key in self.c: + if key is not None and key in keys_seen: key = c.anon_label + keys_seen.add(key) else: key = None - c._make_proxy(self, key=key, name=name, name_is_truncatable=True) - def _refresh_for_new_column(self, column): - for fromclause in self._froms: - col = fromclause._refresh_for_new_column(column) - if col is not None: - if col in self.inner_columns and self._cols_populated: - our_label = col._key_label if self.use_labels else col.key - if our_label not in self.c: - return col._make_proxy( - self, - name=col._label if self.use_labels else None, - key=col._key_label if self.use_labels else None, - name_is_truncatable=True, - ) - return None - return None + c._make_proxy( + subquery, key=key, name=name, name_is_truncatable=True + ) def _needs_parens_for_grouping(self): return ( @@ -3928,7 +4141,8 @@ class Select(HasPrefixes, HasSuffixes, GenerativeSelect): and not self._needs_parens_for_grouping() ): return self - return FromGrouping(self) + else: + return SelectStatementGrouping(self) def union(self, other, **kwargs): """return a SQL UNION of this select() construct against the given @@ -4030,7 +4244,6 @@ class Exists(UnaryExpression): """ - __visit_name__ = UnaryExpression.__visit_name__ _from_objects = [] def __init__(self, *args, **kwargs): @@ -4056,7 +4269,7 @@ class Exists(UnaryExpression): else: if not args: args = ([literal_column("*")],) - s = Select(*args, **kwargs).scalar_subquery().self_group() + s = Select(*args, **kwargs).scalar_subquery() UnaryExpression.__init__( self, @@ -4066,17 +4279,26 @@ class Exists(UnaryExpression): wraps_column_expression=True, ) + def _regroup(self, fn): + element = self.element._ungroup() + element = fn(element) + return element.self_group(against=operators.exists) + def select(self, whereclause=None, **params): return Select([self], whereclause, **params) def correlate(self, *fromclause): e = self._clone() - e.element = self.element.correlate(*fromclause).self_group() + e.element = self._regroup( + lambda element: element.correlate(*fromclause) + ) return e def correlate_except(self, *fromclause): e = self._clone() - e.element = self.element.correlate_except(*fromclause).self_group() + e.element = self._regroup( + lambda element: element.correlate_except(*fromclause) + ) return e def select_from(self, clause): @@ -4086,7 +4308,7 @@ class Exists(UnaryExpression): """ e = self._clone() - e.element = self.element.select_from(clause).self_group() + e.element = self._regroup(lambda element: element.select_from(clause)) return e def where(self, clause): @@ -4095,12 +4317,11 @@ class Exists(UnaryExpression): """ e = self._clone() - e.element = self.element.where(clause).self_group() + e.element = self._regroup(lambda element: element.where(clause)) return e -# TODO: rename to TextualSelect, this is not a FROM clause -class TextAsFrom(SelectBase): +class TextualSelect(SelectBase): """Wrap a :class:`.TextClause` construct within a :class:`.SelectBase` interface. @@ -4108,20 +4329,22 @@ class TextAsFrom(SelectBase): and other FROM-like capabilities such as :meth:`.FromClause.alias`, :meth:`.SelectBase.cte`, etc. - The :class:`.TextAsFrom` construct is produced via the + The :class:`.TextualSelect` construct is produced via the :meth:`.TextClause.columns` method - see that method for details. - .. versionadded:: 0.9.0 + .. versionchanged:: 1.4 the :class:`.TextualSelect` class was renamed + from ``TextAsFrom``, to more correctly suit its role as a + SELECT-oriented object and not a FROM clause. .. seealso:: :func:`.text` - :meth:`.TextClause.columns` + :meth:`.TextClause.columns` - primary creation interface. """ - __visit_name__ = "text_as_from" + __visit_name__ = "textual_select" _is_textual = True @@ -4130,6 +4353,26 @@ class TextAsFrom(SelectBase): self.column_args = columns self.positional = positional + @SelectBase._memoized_property + def selected_columns(self): + """A :class:`.ColumnCollection` representing the columns that + this SELECT statement or similar construct returns in its result set. + + This collection differs from the :attr:`.FromClause.columns` collection + of a :class:`.FromClause` in that the columns within this collection + cannot be directly nested inside another SELECT statement; a subquery + must be applied first which provides for the necessary parenthesization + required by SQL. + + For a :class:`.TextualSelect` construct, the collection contains the + :class:`.ColumnElement` objects that were passed to the constructor, + typically via the :meth:`.TextClause.columns` method. + + .. versionadded:: 1.4 + + """ + return ColumnCollection(*self.column_args).as_immutable() + @property def _bind(self): return self.element._bind @@ -4138,22 +4381,19 @@ class TextAsFrom(SelectBase): def bindparams(self, *binds, **bind_as_values): self.element = self.element.bindparams(*binds, **bind_as_values) - def _populate_column_collection(self): + def _generate_fromclause_column_proxies(self, fromclause): for c in self.column_args: - c._make_proxy(self) + c._make_proxy(fromclause) def _copy_internals(self, clone=_clone, **kw): - self._reset_exported() + self._reset_memoizations() self.element = clone(self.element, **kw) - def get_children(self, column_collections=True, **kw): - if column_collections: - for c in self.column_args: - yield c - yield self.element + def get_children(self, **kw): + return [self.element] def _cache_key(self, **kw): - return (TextAsFrom, self.element._cache_key(**kw)) + tuple( + return (TextualSelect, self.element._cache_key(**kw)) + tuple( col._cache_key(**kw) for col in self.column_args ) @@ -4161,6 +4401,10 @@ class TextAsFrom(SelectBase): return self.column_args[0].type +TextAsFrom = TextualSelect +"""Backwards compatibility with the previous name""" + + class AnnotatedFromClause(Annotated): def __init__(self, element, values): # force FromClause to generate their internal diff --git a/lib/sqlalchemy/sql/util.py b/lib/sqlalchemy/sql/util.py index d90b3f158..dd2c7c1fb 100644 --- a/lib/sqlalchemy/sql/util.py +++ b/lib/sqlalchemy/sql/util.py @@ -17,10 +17,10 @@ from . import visitors from .annotation import _deep_annotate # noqa from .annotation import _deep_deannotate # noqa from .annotation import _shallow_annotate # noqa +from .base import _expand_cloned from .base import _from_objects from .base import ColumnSet from .ddl import sort_tables # noqa -from .elements import _expand_cloned from .elements import _find_columns # noqa from .elements import _label_reference from .elements import _textual_label_reference diff --git a/lib/sqlalchemy/testing/suite/__init__.py b/lib/sqlalchemy/testing/suite/__init__.py index 465a6cfc5..4c71157cd 100644 --- a/lib/sqlalchemy/testing/suite/__init__.py +++ b/lib/sqlalchemy/testing/suite/__init__.py @@ -1,5 +1,6 @@ from .test_cte import * # noqa from .test_ddl import * # noqa +from .test_deprecations import * # noqa from .test_dialect import * # noqa from .test_insert import * # noqa from .test_reflection import * # noqa diff --git a/lib/sqlalchemy/testing/suite/test_deprecations.py b/lib/sqlalchemy/testing/suite/test_deprecations.py new file mode 100644 index 000000000..d0202a0a9 --- /dev/null +++ b/lib/sqlalchemy/testing/suite/test_deprecations.py @@ -0,0 +1,144 @@ +from .. import config +from .. import fixtures +from ..assertions import eq_ +from ..schema import Column +from ..schema import Table +from ... import Integer +from ... import select +from ... import testing +from ... import union + + +class DeprecatedCompoundSelectTest(fixtures.TablesTest): + __backend__ = True + + @classmethod + def define_tables(cls, metadata): + Table( + "some_table", + metadata, + Column("id", Integer, primary_key=True), + Column("x", Integer), + Column("y", Integer), + ) + + @classmethod + def insert_data(cls): + config.db.execute( + cls.tables.some_table.insert(), + [ + {"id": 1, "x": 1, "y": 2}, + {"id": 2, "x": 2, "y": 3}, + {"id": 3, "x": 3, "y": 4}, + {"id": 4, "x": 4, "y": 5}, + ], + ) + + def _assert_result(self, select, result, params=()): + eq_(config.db.execute(select, params).fetchall(), result) + + def test_plain_union(self): + table = self.tables.some_table + s1 = select([table]).where(table.c.id == 2) + s2 = select([table]).where(table.c.id == 3) + + u1 = union(s1, s2) + with testing.expect_deprecated( + "The SelectBase.c and SelectBase.columns " + "attributes are deprecated" + ): + self._assert_result(u1.order_by(u1.c.id), [(2, 2, 3), (3, 3, 4)]) + + # note we've had to remove one use case entirely, which is this + # one. the Select gets its FROMS from the WHERE clause and the + # columns clause, but not the ORDER BY, which means the old ".c" system + # allowed you to "order_by(s.c.foo)" to get an unnamed column in the + # ORDER BY without adding the SELECT into the FROM and breaking the + # query. Users will have to adjust for this use case if they were doing + # it before. + def _dont_test_select_from_plain_union(self): + table = self.tables.some_table + s1 = select([table]).where(table.c.id == 2) + s2 = select([table]).where(table.c.id == 3) + + u1 = union(s1, s2).alias().select() + with testing.expect_deprecated( + "The SelectBase.c and SelectBase.columns " + "attributes are deprecated" + ): + self._assert_result(u1.order_by(u1.c.id), [(2, 2, 3), (3, 3, 4)]) + + @testing.requires.order_by_col_from_union + @testing.requires.parens_in_union_contained_select_w_limit_offset + def test_limit_offset_selectable_in_unions(self): + table = self.tables.some_table + s1 = ( + select([table]) + .where(table.c.id == 2) + .limit(1) + .order_by(table.c.id) + ) + s2 = ( + select([table]) + .where(table.c.id == 3) + .limit(1) + .order_by(table.c.id) + ) + + u1 = union(s1, s2).limit(2) + with testing.expect_deprecated( + "The SelectBase.c and SelectBase.columns " + "attributes are deprecated" + ): + self._assert_result(u1.order_by(u1.c.id), [(2, 2, 3), (3, 3, 4)]) + + @testing.requires.parens_in_union_contained_select_wo_limit_offset + def test_order_by_selectable_in_unions(self): + table = self.tables.some_table + s1 = select([table]).where(table.c.id == 2).order_by(table.c.id) + s2 = select([table]).where(table.c.id == 3).order_by(table.c.id) + + u1 = union(s1, s2).limit(2) + with testing.expect_deprecated( + "The SelectBase.c and SelectBase.columns " + "attributes are deprecated" + ): + self._assert_result(u1.order_by(u1.c.id), [(2, 2, 3), (3, 3, 4)]) + + def test_distinct_selectable_in_unions(self): + table = self.tables.some_table + s1 = select([table]).where(table.c.id == 2).distinct() + s2 = select([table]).where(table.c.id == 3).distinct() + + u1 = union(s1, s2).limit(2) + with testing.expect_deprecated( + "The SelectBase.c and SelectBase.columns " + "attributes are deprecated" + ): + self._assert_result(u1.order_by(u1.c.id), [(2, 2, 3), (3, 3, 4)]) + + def test_limit_offset_aliased_selectable_in_unions(self): + table = self.tables.some_table + s1 = ( + select([table]) + .where(table.c.id == 2) + .limit(1) + .order_by(table.c.id) + .alias() + .select() + ) + s2 = ( + select([table]) + .where(table.c.id == 3) + .limit(1) + .order_by(table.c.id) + .alias() + .select() + ) + + u1 = union(s1, s2).limit(2) + with testing.expect_deprecated( + "The SelectBase.c and SelectBase.columns " + "attributes are deprecated" + ): + self._assert_result(u1.order_by(u1.c.id), [(2, 2, 3), (3, 3, 4)]) diff --git a/lib/sqlalchemy/testing/suite/test_select.py b/lib/sqlalchemy/testing/suite/test_select.py index d26585141..dabb30f9b 100644 --- a/lib/sqlalchemy/testing/suite/test_select.py +++ b/lib/sqlalchemy/testing/suite/test_select.py @@ -268,7 +268,9 @@ class CompoundSelectTest(fixtures.TablesTest): s2 = select([table]).where(table.c.id == 3) u1 = union(s1, s2) - self._assert_result(u1.order_by(u1.c.id), [(2, 2, 3), (3, 3, 4)]) + self._assert_result( + u1.order_by(u1.selected_columns.id), [(2, 2, 3), (3, 3, 4)] + ) def test_select_from_plain_union(self): table = self.tables.some_table @@ -276,7 +278,9 @@ class CompoundSelectTest(fixtures.TablesTest): s2 = select([table]).where(table.c.id == 3) u1 = union(s1, s2).alias().select() - self._assert_result(u1.order_by(u1.c.id), [(2, 2, 3), (3, 3, 4)]) + self._assert_result( + u1.order_by(u1.selected_columns.id), [(2, 2, 3), (3, 3, 4)] + ) @testing.requires.order_by_col_from_union @testing.requires.parens_in_union_contained_select_w_limit_offset @@ -296,7 +300,9 @@ class CompoundSelectTest(fixtures.TablesTest): ) u1 = union(s1, s2).limit(2) - self._assert_result(u1.order_by(u1.c.id), [(2, 2, 3), (3, 3, 4)]) + self._assert_result( + u1.order_by(u1.selected_columns.id), [(2, 2, 3), (3, 3, 4)] + ) @testing.requires.parens_in_union_contained_select_wo_limit_offset def test_order_by_selectable_in_unions(self): @@ -305,7 +311,9 @@ class CompoundSelectTest(fixtures.TablesTest): s2 = select([table]).where(table.c.id == 3).order_by(table.c.id) u1 = union(s1, s2).limit(2) - self._assert_result(u1.order_by(u1.c.id), [(2, 2, 3), (3, 3, 4)]) + self._assert_result( + u1.order_by(u1.selected_columns.id), [(2, 2, 3), (3, 3, 4)] + ) def test_distinct_selectable_in_unions(self): table = self.tables.some_table @@ -313,7 +321,9 @@ class CompoundSelectTest(fixtures.TablesTest): s2 = select([table]).where(table.c.id == 3).distinct() u1 = union(s1, s2).limit(2) - self._assert_result(u1.order_by(u1.c.id), [(2, 2, 3), (3, 3, 4)]) + self._assert_result( + u1.order_by(u1.selected_columns.id), [(2, 2, 3), (3, 3, 4)] + ) @testing.requires.parens_in_union_contained_select_w_limit_offset def test_limit_offset_in_unions_from_alias(self): @@ -357,7 +367,9 @@ class CompoundSelectTest(fixtures.TablesTest): ) u1 = union(s1, s2).limit(2) - self._assert_result(u1.order_by(u1.c.id), [(2, 2, 3), (3, 3, 4)]) + self._assert_result( + u1.order_by(u1.selected_columns.id), [(2, 2, 3), (3, 3, 4)] + ) class ExpandingBoundInTest(fixtures.TablesTest): |