implement deferred scalarobject history load

Modified the approach used for history tracking of scalar object
relationships that are not many-to-one, i.e. one-to-one relationships that
would otherwise be one-to-many. When replacing a one-to-one value, the
"old" value that would be replaced is no longer loaded immediately, and is
instead handled during the flush process. This eliminates an historically
troublesome lazy load that otherwise often occurs when assigning to a
one-to-one attribute, and is particularly troublesome when using
"lazy='raise'" as well as asyncio use cases.

This change does cause a behavioral change within the
:meth:`_orm.AttributeEvents.set` event, which is nonetheless currently
documented, which is that the event applied to such a one-to-one attribute
will no longer receive the "old" parameter if it is unloaded and the
:paramref:`_orm.relationship.active_history` flag is not set. As is
documented in :meth:`_orm.AttributeEvents.set`, if the event handler needs
to receive the "old" value when the event fires off, the active_history
flag must be established either with the event listener or with the
relationship. This is already the behavior with other kinds of attributes
such as many-to-one and column value references.

The change additionally will defer updating a backref on the "old" value
in the less common case that the "old" value is locally present in the
session, but isn't loaded on the relationship in question, until the
next flush occurs.  If this causes an issue, again the normal
:paramref:`_orm.relationship.active_history` flag can be set to ``True``
on the relationship.

A private flag which restores the old value is retained for now,
as support within relevant test suites to exercise the old and
new behaviors together.  This is so that if the behavioral change
produces problems we have test harnesses set up to further examine these
behaviors.   The "legacy" style can go away in 2.0 or in a much later
1.4 release.

Fixes: #6708
Change-Id: Id7f72fc39dcbec9119b665e528667a9919bb73b4

1 files changed, 53 insertions, 19 deletions

diff --git a/lib/sqlalchemy/orm/attributes.py b/lib/sqlalchemy/orm/attributes.py
index 105a9cfd2..9ba05395a 100644
--- a/lib/sqlalchemy/orm/attributes.py
+++ b/lib/sqlalchemy/orm/attributes.py
@@ -22,6 +22,7 @@ from . import interfaces
 from .base import ATTR_EMPTY
 from .base import ATTR_WAS_SET
 from .base import CALLABLES_OK
+from .base import DEFERRED_HISTORY_LOAD
 from .base import INIT_OK
 from .base import instance_dict
 from .base import instance_state
@@ -768,6 +769,9 @@ class AttributeImpl(object):
         else:
             self.accepts_scalar_loader = self.default_accepts_scalar_loader
 
+        _deferred_history = kwargs.pop("_deferred_history", False)
+        self._deferred_history = _deferred_history
+
         if active_history:
             self.dispatch._active_history = True
 
@@ -786,6 +790,7 @@ class AttributeImpl(object):
         "load_on_unexpire",
         "_modified_token",
         "accepts_scalar_loader",
+        "_deferred_history",
     )
 
     def __str__(self):
@@ -918,19 +923,7 @@ class AttributeImpl(object):
                 if not passive & CALLABLES_OK:
                     return PASSIVE_NO_RESULT
 
-                if (
-                    self.accepts_scalar_loader
-                    and self.load_on_unexpire
-                    and key in state.expired_attributes
-                ):
-                    value = state._load_expired(state, passive)
-                elif key in state.callables:
-                    callable_ = state.callables[key]
-                    value = callable_(state, passive)
-                elif self.callable_:
-                    value = self.callable_(state, passive)
-                else:
-                    value = ATTR_EMPTY
+                value = self._fire_loader_callables(state, key, passive)
 
                 if value is PASSIVE_NO_RESULT or value is NO_VALUE:
                     return value
@@ -955,6 +948,21 @@ class AttributeImpl(object):
             else:
                 return self._default_value(state, dict_)
 
+    def _fire_loader_callables(self, state, key, passive):
+        if (
+            self.accepts_scalar_loader
+            and self.load_on_unexpire
+            and key in state.expired_attributes
+        ):
+            return state._load_expired(state, passive)
+        elif key in state.callables:
+            callable_ = state.callables[key]
+            return callable_(state, passive)
+        elif self.callable_:
+            return self.callable_(state, passive)
+        else:
+            return ATTR_EMPTY
+
     def append(self, state, dict_, value, initiator, passive=PASSIVE_OFF):
         self.set(state, dict_, value, initiator, passive=passive)
 
@@ -1142,15 +1150,33 @@ class ScalarObjectAttributeImpl(ScalarAttributeImpl):
 
     def get_history(self, state, dict_, passive=PASSIVE_OFF):
         if self.key in dict_:
-            return History.from_object_attribute(self, state, dict_[self.key])
+            current = dict_[self.key]
         else:
             if passive & INIT_OK:
                 passive ^= INIT_OK
             current = self.get(state, dict_, passive=passive)
             if current is PASSIVE_NO_RESULT:
                 return HISTORY_BLANK
-            else:
-                return History.from_object_attribute(self, state, current)
+
+        if not self._deferred_history:
+            return History.from_object_attribute(self, state, current)
+        else:
+            original = state.committed_state.get(self.key, _NO_HISTORY)
+            if original is PASSIVE_NO_RESULT:
+
+                loader_passive = passive | (
+                    PASSIVE_ONLY_PERSISTENT
+                    | NO_AUTOFLUSH
+                    | LOAD_AGAINST_COMMITTED
+                    | NO_RAISE
+                    | DEFERRED_HISTORY_LOAD
+                )
+                original = self._fire_loader_callables(
+                    state, self.key, loader_passive
+                )
+            return History.from_object_attribute(
+                self, state, current, original=original
+            )
 
     def get_all_pending(self, state, dict_, passive=PASSIVE_NO_INITIALIZE):
         if self.key in dict_:
@@ -1193,6 +1219,7 @@ class ScalarObjectAttributeImpl(ScalarAttributeImpl):
         pop=False,
     ):
         """Set a value on the given InstanceState."""
+
         if self.dispatch._active_history:
             old = self.get(
                 state,
@@ -1227,7 +1254,11 @@ class ScalarObjectAttributeImpl(ScalarAttributeImpl):
         dict_[self.key] = value
 
     def fire_remove_event(self, state, dict_, value, initiator):
-        if self.trackparent and value is not None:
+        if self.trackparent and value not in (
+            None,
+            PASSIVE_NO_RESULT,
+            NO_VALUE,
+        ):
             self.sethasparent(instance_state(value), state, False)
 
         for fn in self.dispatch.remove:
@@ -1930,8 +1961,11 @@ class History(util.namedtuple("History", ["added", "unchanged", "deleted"])):
                 return cls([current], (), deleted)
 
     @classmethod
-    def from_object_attribute(cls, attribute, state, current):
-        original = state.committed_state.get(attribute.key, _NO_HISTORY)
+    def from_object_attribute(
+        cls, attribute, state, current, original=_NO_HISTORY
+    ):
+        if original is _NO_HISTORY:
+            original = state.committed_state.get(attribute.key, _NO_HISTORY)
 
         if original is _NO_HISTORY:
             if current is NO_VALUE: