Fix Object Movement allocation in GC

When moving Objects between size pools we have to assign a new shape.

This happened during updating references - we tried to create a new shape
tree that mirrored the existing tree, but based on the root shape of the
new size pool.

This causes allocations to happen if the new tree doesn't already exist,
potentially triggering a GC, during GC.

This commit changes object movement to look for a pre-existing new tree
during object movement, and if that tree does not exist, we don't move
the object to the new pool.

This allows us to remove the shape allocation from update references.

Co-Authored-By: Peter Zhu <peter@peterzhu.ca>

1 files changed, 36 insertions, 0 deletions

diff --git a/shape.c b/shape.c
index ac9a016c00..c0f1bc6e8f 100644
--- a/shape.c
+++ b/shape.c
@@ -445,6 +445,42 @@ rb_shape_id_offset(void)
 }
 
 rb_shape_t *
+rb_shape_traverse_from_new_root(rb_shape_t *initial_shape, rb_shape_t *dest_shape)
+{
+    RUBY_ASSERT(initial_shape->type == SHAPE_T_OBJECT);
+    rb_shape_t *next_shape = initial_shape;
+
+    if (dest_shape->type != initial_shape->type) {
+        next_shape = rb_shape_traverse_from_new_root(initial_shape, rb_shape_get_parent(dest_shape));
+        if (!next_shape) {
+            return NULL;
+        }
+    }
+
+    switch ((enum shape_type)dest_shape->type) {
+      case SHAPE_IVAR:
+        if (!next_shape->edges) {
+            return NULL;
+        }
+        if (!rb_id_table_lookup(next_shape->edges, dest_shape->edge_name, (VALUE *)&next_shape)) {
+            return NULL;
+        }
+        break;
+      case SHAPE_ROOT:
+      case SHAPE_FROZEN:
+      case SHAPE_CAPACITY_CHANGE:
+      case SHAPE_INITIAL_CAPACITY:
+      case SHAPE_T_OBJECT:
+        break;
+      case SHAPE_OBJ_TOO_COMPLEX:
+        rb_bug("Unreachable\n");
+        break;
+    }
+
+    return next_shape;
+}
+
+rb_shape_t *
 rb_shape_rebuild_shape(rb_shape_t * initial_shape, rb_shape_t * dest_shape)
 {
     rb_shape_t * midway_shape;