From b0cfa7379cf8513a821a3dbe3028c4965d9f85bd Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Sat, 6 Jun 2020 20:40:43 -0400 Subject: Turn on caching everywhere, add logging A variety of caching issues found by running all tests with statement caching turned on. The cache system now has a more conservative approach where any subclass of a SQL element will by default invalidate the cache key unless it adds the flag inherit_cache=True at the class level, or if it implements its own caching. Add working caching to a few elements that were omitted previously; fix some caching implementations to suit lesser used edge cases such as json casts and array slices. Refine the way BaseCursorResult and CursorMetaData interact with caching; to suit cases like Alembic modifying table structures, don't cache the cursor metadata if it were created against a cursor.description using non-positional matching, e.g. "select *". if a table re-ordered its columns or added/removed, now that data is obsolete. Additionally we have to adapt the cursor metadata _keymap regardless of if we just processed cursor.description, because if we ran against a cached SQLCompiler we won't have the right columns in _keymap. Other refinements to how and when we do this adaption as some weird cases were exposed in the Postgresql dialect, a text() construct that names just one column that is not actually in the statement. Fixed that also as it looks like a cut-and-paste artifact that doesn't actually affect anything. Various issues with re-use of compiled result maps and cursor metadata in conjunction with tables being changed, such as change in order of columns. mappers can be cleared but the class remains, meaning a mapper has to use itself as the cache key not the class. lots of bound parameter / literal issues, due to Alembic creating a straight subclass of bindparam that renders inline directly. While we can update Alembic to not do this, we have to assume other people might be doing this, so bindparam() implements the inherit_cache=True logic as well that was a bit involved. turn on cache stats in logging. Includes a fix to subqueryloader which moves all setup to the create_row_processor() phase and elminates any storage within the compiled context. This includes some changes to create_row_processor() signature and a revising of the technique used to determine if the loader can participate in polymorphic queries, which is also applied to selectinloading. DML update.values() and ordered_values() now coerces the keys as we have tests that pass an arbitrary class here which only includes __clause_element__(), so the key can't be cached unless it is coerced. this in turn changed how composite attributes support bulk update to use the standard approach of ClauseElement with annotations that are parsed in the ORM context. memory profiling successfully caught that the Session from Query was getting passed into _statement_20() so that was a big win for that test suite. Apparently Compiler had .execute() and .scalar() methods stuck on it, these date back to version 0.4 and there was a single test in the PostgreSQL dialect tests that exercised it for no apparent reason. Removed these methods as well as the concept of a Compiler holding onto a "bind". Fixes: #5386 Change-Id: I990b43aab96b42665af1b2187ad6020bee778784 --- lib/sqlalchemy/sql/functions.py | 39 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 36 insertions(+), 3 deletions(-) (limited to 'lib/sqlalchemy/sql/functions.py') diff --git a/lib/sqlalchemy/sql/functions.py b/lib/sqlalchemy/sql/functions.py index 6b1172eba..7b723f371 100644 --- a/lib/sqlalchemy/sql/functions.py +++ b/lib/sqlalchemy/sql/functions.py @@ -744,6 +744,7 @@ class GenericFunction(util.with_metaclass(_GenericMeta, Function)): coerce_arguments = True _register = False + inherit_cache = True def __init__(self, *args, **kwargs): parsed_args = kwargs.pop("_parsed_args", None) @@ -808,6 +809,8 @@ class next_value(GenericFunction): class AnsiFunction(GenericFunction): + inherit_cache = True + def __init__(self, *args, **kwargs): GenericFunction.__init__(self, *args, **kwargs) @@ -815,6 +818,8 @@ class AnsiFunction(GenericFunction): class ReturnTypeFromArgs(GenericFunction): """Define a function whose return type is the same as its arguments.""" + inherit_cache = True + def __init__(self, *args, **kwargs): args = [ coercions.expect( @@ -832,30 +837,34 @@ class ReturnTypeFromArgs(GenericFunction): class coalesce(ReturnTypeFromArgs): _has_args = True + inherit_cache = True class max(ReturnTypeFromArgs): # noqa - pass + inherit_cache = True class min(ReturnTypeFromArgs): # noqa - pass + inherit_cache = True class sum(ReturnTypeFromArgs): # noqa - pass + inherit_cache = True class now(GenericFunction): # noqa type = sqltypes.DateTime + inherit_cache = True class concat(GenericFunction): type = sqltypes.String + inherit_cache = True class char_length(GenericFunction): type = sqltypes.Integer + inherit_cache = True def __init__(self, arg, **kwargs): GenericFunction.__init__(self, arg, **kwargs) @@ -863,6 +872,7 @@ class char_length(GenericFunction): class random(GenericFunction): _has_args = True + inherit_cache = True class count(GenericFunction): @@ -887,6 +897,7 @@ class count(GenericFunction): """ type = sqltypes.Integer + inherit_cache = True def __init__(self, expression=None, **kwargs): if expression is None: @@ -896,38 +907,47 @@ class count(GenericFunction): class current_date(AnsiFunction): type = sqltypes.Date + inherit_cache = True class current_time(AnsiFunction): type = sqltypes.Time + inherit_cache = True class current_timestamp(AnsiFunction): type = sqltypes.DateTime + inherit_cache = True class current_user(AnsiFunction): type = sqltypes.String + inherit_cache = True class localtime(AnsiFunction): type = sqltypes.DateTime + inherit_cache = True class localtimestamp(AnsiFunction): type = sqltypes.DateTime + inherit_cache = True class session_user(AnsiFunction): type = sqltypes.String + inherit_cache = True class sysdate(AnsiFunction): type = sqltypes.DateTime + inherit_cache = True class user(AnsiFunction): type = sqltypes.String + inherit_cache = True class array_agg(GenericFunction): @@ -951,6 +971,7 @@ class array_agg(GenericFunction): """ type = sqltypes.ARRAY + inherit_cache = True def __init__(self, *args, **kwargs): args = [ @@ -978,6 +999,7 @@ class OrderedSetAgg(GenericFunction): :meth:`.FunctionElement.within_group` method.""" array_for_multi_clause = False + inherit_cache = True def within_group_type(self, within_group): func_clauses = self.clause_expr.element @@ -1000,6 +1022,8 @@ class mode(OrderedSetAgg): """ + inherit_cache = True + class percentile_cont(OrderedSetAgg): """implement the ``percentile_cont`` ordered-set aggregate function. @@ -1016,6 +1040,7 @@ class percentile_cont(OrderedSetAgg): """ array_for_multi_clause = True + inherit_cache = True class percentile_disc(OrderedSetAgg): @@ -1033,6 +1058,7 @@ class percentile_disc(OrderedSetAgg): """ array_for_multi_clause = True + inherit_cache = True class rank(GenericFunction): @@ -1048,6 +1074,7 @@ class rank(GenericFunction): """ type = sqltypes.Integer() + inherit_cache = True class dense_rank(GenericFunction): @@ -1063,6 +1090,7 @@ class dense_rank(GenericFunction): """ type = sqltypes.Integer() + inherit_cache = True class percent_rank(GenericFunction): @@ -1078,6 +1106,7 @@ class percent_rank(GenericFunction): """ type = sqltypes.Numeric() + inherit_cache = True class cume_dist(GenericFunction): @@ -1093,6 +1122,7 @@ class cume_dist(GenericFunction): """ type = sqltypes.Numeric() + inherit_cache = True class cube(GenericFunction): @@ -1109,6 +1139,7 @@ class cube(GenericFunction): """ _has_args = True + inherit_cache = True class rollup(GenericFunction): @@ -1125,6 +1156,7 @@ class rollup(GenericFunction): """ _has_args = True + inherit_cache = True class grouping_sets(GenericFunction): @@ -1158,3 +1190,4 @@ class grouping_sets(GenericFunction): """ _has_args = True + inherit_cache = True -- cgit v1.2.1