From 92cc232726a01dd3beff762ebccd326a9659e8b9 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Tue, 13 Jan 2015 14:33:33 -0500 Subject: - The multi-values version of :meth:`.Insert.values` has been repaired to work more usefully with tables that have Python- side default values and/or functions, as well as server-side defaults. The feature will now work with a dialect that uses "positional" parameters; a Python callable will also be invoked individually for each row just as is the case with an "executemany" style invocation; a server- side default column will no longer implicitly receive the value explicitly specified for the first row, instead refusing to invoke without an explicit value. fixes #3288 --- lib/sqlalchemy/engine/default.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'lib/sqlalchemy/engine/default.py') diff --git a/lib/sqlalchemy/engine/default.py b/lib/sqlalchemy/engine/default.py index a5af6ff19..c5b5deece 100644 --- a/lib/sqlalchemy/engine/default.py +++ b/lib/sqlalchemy/engine/default.py @@ -956,14 +956,17 @@ class DefaultExecutionContext(interfaces.ExecutionContext): def _process_executesingle_defaults(self): key_getter = self.compiled._key_getters_for_crud_column[2] - prefetch = self.compiled.prefetch self.current_parameters = compiled_parameters = \ self.compiled_parameters[0] for c in prefetch: if self.isinsert: - val = self.get_insert_default(c) + if c.default and \ + not c.default.is_sequence and c.default.is_scalar: + val = c.default.arg + else: + val = self.get_insert_default(c) else: val = self.get_update_default(c) @@ -972,6 +975,4 @@ class DefaultExecutionContext(interfaces.ExecutionContext): del self.current_parameters - - DefaultDialect.execution_ctx_cls = DefaultExecutionContext -- cgit v1.2.1 From a33c250da273ba9b1c62b5ba6d99914870155faf Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Sun, 25 Jan 2015 17:53:41 -0500 Subject: - remove context-specific post-crud logic from Connection and inline post-crud logic to some degree in DefaultExecutionContext. In particular we are removing post_insert() which doesn't appear to be used based on a survey of prominent third party dialects. Callcounts aren't added to existing execute profiling tests and inserts might be a little better. - simplify the execution_options join in DEC. Callcounts don't appear affected. --- lib/sqlalchemy/engine/default.py | 127 +++++++++++++++++++++++---------------- 1 file changed, 76 insertions(+), 51 deletions(-) (limited to 'lib/sqlalchemy/engine/default.py') diff --git a/lib/sqlalchemy/engine/default.py b/lib/sqlalchemy/engine/default.py index c5b5deece..f6c2263b3 100644 --- a/lib/sqlalchemy/engine/default.py +++ b/lib/sqlalchemy/engine/default.py @@ -452,14 +452,12 @@ class DefaultExecutionContext(interfaces.ExecutionContext): isinsert = False isupdate = False isdelete = False + is_crud = False isddl = False executemany = False result_map = None compiled = None statement = None - postfetch_cols = None - prefetch_cols = None - returning_cols = None _is_implicit_returning = False _is_explicit_returning = False @@ -515,10 +513,8 @@ class DefaultExecutionContext(interfaces.ExecutionContext): if not compiled.can_execute: raise exc.ArgumentError("Not an executable clause") - self.execution_options = compiled.statement._execution_options - if connection._execution_options: - self.execution_options = dict(self.execution_options) - self.execution_options.update(connection._execution_options) + self.execution_options = compiled.statement._execution_options.union( + connection._execution_options) # compiled clauseelement. process bind params, process table defaults, # track collections used by ResultProxy to target and process results @@ -548,6 +544,7 @@ class DefaultExecutionContext(interfaces.ExecutionContext): self.cursor = self.create_cursor() if self.isinsert or self.isupdate or self.isdelete: + self.is_crud = True self._is_explicit_returning = bool(compiled.statement._returning) self._is_implicit_returning = bool( compiled.returning and not compiled.statement._returning) @@ -680,10 +677,6 @@ class DefaultExecutionContext(interfaces.ExecutionContext): def no_parameters(self): return self.execution_options.get("no_parameters", False) - @util.memoized_property - def is_crud(self): - return self.isinsert or self.isupdate or self.isdelete - @util.memoized_property def should_autocommit(self): autocommit = self.execution_options.get('autocommit', @@ -799,52 +792,84 @@ class DefaultExecutionContext(interfaces.ExecutionContext): def supports_sane_multi_rowcount(self): return self.dialect.supports_sane_multi_rowcount - def post_insert(self): - + def _setup_crud_result_proxy(self): + if self.isinsert and \ + not self.executemany: + if not self._is_implicit_returning and \ + not self.compiled.inline and \ + self.dialect.postfetch_lastrowid: + + self._setup_ins_pk_from_lastrowid() + + elif not self._is_implicit_returning: + self._setup_ins_pk_from_empty() + + result = self.get_result_proxy() + + if self.isinsert: + if self._is_implicit_returning: + row = result.fetchone() + self.returned_defaults = row + self._setup_ins_pk_from_implicit_returning(row) + result.close(_autoclose_connection=False) + result._metadata = None + elif not self._is_explicit_returning: + result.close(_autoclose_connection=False) + result._metadata = None + elif self.isupdate and self._is_implicit_returning: + row = result.fetchone() + self.returned_defaults = row + result.close(_autoclose_connection=False) + result._metadata = None + + elif result._metadata is None: + # no results, get rowcount + # (which requires open cursor on some drivers + # such as kintersbasdb, mxodbc) + result.rowcount + result.close(_autoclose_connection=False) + return result + + def _setup_ins_pk_from_lastrowid(self): key_getter = self.compiled._key_getters_for_crud_column[2] table = self.compiled.statement.table + compiled_params = self.compiled_parameters[0] + + lastrowid = self.get_lastrowid() + autoinc_col = table._autoincrement_column + if autoinc_col is not None: + # apply type post processors to the lastrowid + proc = autoinc_col.type._cached_result_processor( + self.dialect, None) + if proc is not None: + lastrowid = proc(lastrowid) + self.inserted_primary_key = [ + lastrowid if c is autoinc_col else + compiled_params.get(key_getter(c), None) + for c in table.primary_key + ] - if not self._is_implicit_returning and \ - not self._is_explicit_returning and \ - not self.compiled.inline and \ - self.dialect.postfetch_lastrowid: - - lastrowid = self.get_lastrowid() - autoinc_col = table._autoincrement_column - if autoinc_col is not None: - # apply type post processors to the lastrowid - proc = autoinc_col.type._cached_result_processor( - self.dialect, None) - if proc is not None: - lastrowid = proc(lastrowid) - self.inserted_primary_key = [ - lastrowid if c is autoinc_col else - self.compiled_parameters[0].get(key_getter(c), None) - for c in table.primary_key - ] - else: - self.inserted_primary_key = [ - self.compiled_parameters[0].get(key_getter(c), None) - for c in table.primary_key - ] - - def _fetch_implicit_returning(self, resultproxy): + def _setup_ins_pk_from_empty(self): + key_getter = self.compiled._key_getters_for_crud_column[2] table = self.compiled.statement.table - row = resultproxy.fetchone() - - ipk = [] - for c, v in zip(table.primary_key, self.inserted_primary_key): - if v is not None: - ipk.append(v) - else: - ipk.append(row[c]) + compiled_params = self.compiled_parameters[0] + self.inserted_primary_key = [ + compiled_params.get(key_getter(c), None) + for c in table.primary_key + ] - self.inserted_primary_key = ipk - self.returned_defaults = row + def _setup_ins_pk_from_implicit_returning(self, row): + key_getter = self.compiled._key_getters_for_crud_column[2] + table = self.compiled.statement.table + compiled_params = self.compiled_parameters[0] - def _fetch_implicit_update_returning(self, resultproxy): - row = resultproxy.fetchone() - self.returned_defaults = row + self.inserted_primary_key = [ + row[col] if value is None else value + for col, value in [ + (col, compiled_params.get(key_getter(col), None)) + for col in table.primary_key + ] + ] def lastrow_has_defaults(self): return (self.isinsert or self.isupdate) and \ -- cgit v1.2.1 From 9cbe235810b7c0c24d2556b4bb581b0207812e2d Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Wed, 4 Feb 2015 17:07:15 -0500 Subject: - A warning is emitted if the ``isolation_level`` parameter is used with :meth:`.Connection.execution_options` when a :class:`.Transaction` is in play; DBAPIs and/or SQLAlchemy dialects such as psycopg2, MySQLdb may implicitly rollback or commit the transaction, or not change the setting til next transaction, so this is never safe. - Added new parameter :paramref:`.Session.connection.execution_options` which may be used to set up execution options on a :class:`.Connection` when it is first checked out, before the transaction has begun. This is used to set up options such as isolation level on the connection before the transaction starts. - added new documentation section detailing best practices for setting transaction isolation with sessions. fixes #3296 --- lib/sqlalchemy/engine/default.py | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'lib/sqlalchemy/engine/default.py') diff --git a/lib/sqlalchemy/engine/default.py b/lib/sqlalchemy/engine/default.py index f6c2263b3..17d2e2531 100644 --- a/lib/sqlalchemy/engine/default.py +++ b/lib/sqlalchemy/engine/default.py @@ -395,6 +395,12 @@ class DefaultDialect(interfaces.Dialect): self._set_connection_isolation(connection, opts['isolation_level']) def _set_connection_isolation(self, connection, level): + if connection.in_transaction(): + util.warn( + "Connection is already established with a Transaction; " + "setting isolation_level may implicitly rollback or commit " + "the existing transaction, or have no effect until " + "next transaction") self.set_isolation_level(connection.connection, level) connection.connection._connection_record.\ finalize_callback.append(self.reset_isolation_level) -- cgit v1.2.1 From b3d3795de0d45fe4adda7393881f0f955409a45d Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Sat, 7 Mar 2015 12:48:13 -0500 Subject: - The SQL compiler now generates the mapping of expected columns such that they are matched to the received result set positionally, rather than by name. Originally, this was seen as a way to handle cases where we had columns returned with difficult-to-predict names, though in modern use that issue has been overcome by anonymous labeling. In this version, the approach basically reduces function call count per-result by a few dozen calls, or more for larger sets of result columns. The approach still degrades into a modern version of the old approach if textual elements modify the result map, or if any discrepancy in size exists between the compiled set of columns versus what was received, so there's no issue for partially or fully textual compilation scenarios where these lists might not line up. fixes #918 - callcounts still need to be adjusted down for this so zoomark tests won't pass at the moment --- lib/sqlalchemy/engine/default.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) (limited to 'lib/sqlalchemy/engine/default.py') diff --git a/lib/sqlalchemy/engine/default.py b/lib/sqlalchemy/engine/default.py index 17d2e2531..9d2bbfb15 100644 --- a/lib/sqlalchemy/engine/default.py +++ b/lib/sqlalchemy/engine/default.py @@ -461,9 +461,9 @@ class DefaultExecutionContext(interfaces.ExecutionContext): is_crud = False isddl = False executemany = False - result_map = None compiled = None statement = None + _result_columns = None _is_implicit_returning = False _is_explicit_returning = False @@ -525,7 +525,7 @@ class DefaultExecutionContext(interfaces.ExecutionContext): # compiled clauseelement. process bind params, process table defaults, # track collections used by ResultProxy to target and process results - self.result_map = compiled.result_map + self._result_columns = compiled._result_columns self.unicode_statement = util.text_type(compiled) if not dialect.supports_unicode_statements: @@ -663,6 +663,13 @@ class DefaultExecutionContext(interfaces.ExecutionContext): self.cursor = self.create_cursor() return self + @property + def result_map(self): + if self._result_columns: + return self.compiled.result_map + else: + return None + @util.memoized_property def engine(self): return self.root_connection.engine -- cgit v1.2.1 From 17b2fd3fba8eef5bdd4040f2d71c75e4aeb4e215 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Sat, 7 Mar 2015 22:51:12 -0500 Subject: - the change for #918 was of course not nearly that simple. The "wrapping" employed by the mssql and oracle dialects using the "iswrapper" argument was not being used intelligently by the compiler, and the result map was being written incorrectly, using *more* columns in the result map than were actually returned by the statement, due to "row number" columns that are inside the subquery. The compiler now writes out result map on the "top level" select in all cases fully, and for the mssql/oracle wrapping case extracts out the "proxied" columns in a second step, which only includes those columns that are proxied outwards to the top level. This change might have implications for 3rd party dialects that might be imitating oracle's approach. They can safely continue to use the "iswrapper" kw which is now ignored, but they may need to also add the _select_wraps argument as well. --- lib/sqlalchemy/engine/default.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/sqlalchemy/engine/default.py') diff --git a/lib/sqlalchemy/engine/default.py b/lib/sqlalchemy/engine/default.py index 9d2bbfb15..62469d720 100644 --- a/lib/sqlalchemy/engine/default.py +++ b/lib/sqlalchemy/engine/default.py @@ -663,7 +663,7 @@ class DefaultExecutionContext(interfaces.ExecutionContext): self.cursor = self.create_cursor() return self - @property + @util.memoized_property def result_map(self): if self._result_columns: return self.compiled.result_map -- cgit v1.2.1 From 9854114f578833aee49b65e440319a0ec26daab6 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Sun, 8 Mar 2015 11:29:10 -0400 Subject: foo --- lib/sqlalchemy/engine/default.py | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) (limited to 'lib/sqlalchemy/engine/default.py') diff --git a/lib/sqlalchemy/engine/default.py b/lib/sqlalchemy/engine/default.py index 62469d720..bdd9d2715 100644 --- a/lib/sqlalchemy/engine/default.py +++ b/lib/sqlalchemy/engine/default.py @@ -463,7 +463,7 @@ class DefaultExecutionContext(interfaces.ExecutionContext): executemany = False compiled = None statement = None - _result_columns = None + result_column_struct = None _is_implicit_returning = False _is_explicit_returning = False @@ -522,10 +522,8 @@ class DefaultExecutionContext(interfaces.ExecutionContext): self.execution_options = compiled.statement._execution_options.union( connection._execution_options) - # compiled clauseelement. process bind params, process table defaults, - # track collections used by ResultProxy to target and process results - - self._result_columns = compiled._result_columns + self.result_column_struct = ( + compiled._result_columns, compiled._ordered_columns) self.unicode_statement = util.text_type(compiled) if not dialect.supports_unicode_statements: @@ -663,13 +661,6 @@ class DefaultExecutionContext(interfaces.ExecutionContext): self.cursor = self.create_cursor() return self - @util.memoized_property - def result_map(self): - if self._result_columns: - return self.compiled.result_map - else: - return None - @util.memoized_property def engine(self): return self.root_connection.engine -- cgit v1.2.1 From 50866d2f857dd45bb2d186a0fa076768437d62a3 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Tue, 10 Mar 2015 15:24:28 -0400 Subject: - copyright 2015 --- lib/sqlalchemy/engine/default.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/sqlalchemy/engine/default.py') diff --git a/lib/sqlalchemy/engine/default.py b/lib/sqlalchemy/engine/default.py index bdd9d2715..b46acb650 100644 --- a/lib/sqlalchemy/engine/default.py +++ b/lib/sqlalchemy/engine/default.py @@ -1,5 +1,5 @@ # engine/default.py -# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors +# Copyright (C) 2005-2015 the SQLAlchemy authors and contributors # # # This module is part of SQLAlchemy and is released under -- cgit v1.2.1 From b7e151ac5cf5a0c13b9a30bc6841ed0cfe322536 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Tue, 17 Mar 2015 12:32:33 -0400 Subject: - The "auto close" for :class:`.ResultProxy` is now a "soft" close. That is, after exhausing all rows using the fetch methods, the DBAPI cursor is released as before and the object may be safely discarded, but the fetch methods may continue to be called for which they will return an end-of-result object (None for fetchone, empty list for fetchmany and fetchall). Only if :meth:`.ResultProxy.close` is called explicitly will these methods raise the "result is closed" error. fixes #3330 fixes #3329 --- lib/sqlalchemy/engine/default.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'lib/sqlalchemy/engine/default.py') diff --git a/lib/sqlalchemy/engine/default.py b/lib/sqlalchemy/engine/default.py index b46acb650..3eebc6c06 100644 --- a/lib/sqlalchemy/engine/default.py +++ b/lib/sqlalchemy/engine/default.py @@ -815,15 +815,15 @@ class DefaultExecutionContext(interfaces.ExecutionContext): row = result.fetchone() self.returned_defaults = row self._setup_ins_pk_from_implicit_returning(row) - result.close(_autoclose_connection=False) + result._soft_close(_autoclose_connection=False) result._metadata = None elif not self._is_explicit_returning: - result.close(_autoclose_connection=False) + result._soft_close(_autoclose_connection=False) result._metadata = None elif self.isupdate and self._is_implicit_returning: row = result.fetchone() self.returned_defaults = row - result.close(_autoclose_connection=False) + result._soft_close(_autoclose_connection=False) result._metadata = None elif result._metadata is None: @@ -831,7 +831,7 @@ class DefaultExecutionContext(interfaces.ExecutionContext): # (which requires open cursor on some drivers # such as kintersbasdb, mxodbc) result.rowcount - result.close(_autoclose_connection=False) + result._soft_close(_autoclose_connection=False) return result def _setup_ins_pk_from_lastrowid(self): -- cgit v1.2.1