BUG: Avoid errors on NULL during deepcopy (#21996)

Addresses concerns raised in the two linked issues by:

Casting NULL objects to Py_None within _deepcopy_call to avoid issues within CPython's deepcopy
Setting _deepcopy_call to return an int code for error/success, which is then checked in array_deepcopy

Closes gh-8706, gh-21883

2 files changed, 47 insertions, 10 deletions

diff --git a/numpy/core/src/multiarray/methods.c b/numpy/core/src/multiarray/methods.c
index b738c1d44..f10f68ea5 100644
--- a/numpy/core/src/multiarray/methods.c
+++ b/numpy/core/src/multiarray/methods.c
@@ -1602,17 +1602,17 @@ array_searchsorted(PyArrayObject *self,
     return PyArray_Return((PyArrayObject *)PyArray_SearchSorted(self, keys, side, sorter));
 }
 
-static void
+static int
 _deepcopy_call(char *iptr, char *optr, PyArray_Descr *dtype,
                PyObject *deepcopy, PyObject *visit)
 {
     if (!PyDataType_REFCHK(dtype)) {
-        return;
+        return 0;
     }
     else if (PyDataType_HASFIELDS(dtype)) {
         PyObject *key, *value, *title = NULL;
         PyArray_Descr *new;
-        int offset;
+        int offset, res;
         Py_ssize_t pos = 0;
         while (PyDict_Next(dtype->fields, &pos, &key, &value)) {
             if (NPY_TITLE_KEY(key, value)) {
@@ -1620,10 +1620,13 @@ _deepcopy_call(char *iptr, char *optr, PyArray_Descr *dtype,
             }
             if (!PyArg_ParseTuple(value, "Oi|O", &new, &offset,
                                   &title)) {
-                return;
+                return -1;
+            }
+            res = _deepcopy_call(iptr + offset, optr + offset, new,
+                                 deepcopy, visit);
+            if (res < 0) {
+                return -1;
             }
-            _deepcopy_call(iptr + offset, optr + offset, new,
-                           deepcopy, visit);
         }
     }
     else {
@@ -1631,13 +1634,20 @@ _deepcopy_call(char *iptr, char *optr, PyArray_Descr *dtype,
         PyObject *res;
         memcpy(&itemp, iptr, sizeof(itemp));
         memcpy(&otemp, optr, sizeof(otemp));
-        Py_XINCREF(itemp);
+        if (itemp == NULL) {
+            itemp = Py_None;
+        }
+        Py_INCREF(itemp);
         /* call deepcopy on this argument */
         res = PyObject_CallFunctionObjArgs(deepcopy, itemp, visit, NULL);
-        Py_XDECREF(itemp);
+        Py_DECREF(itemp);
+        if (res == NULL) {
+            return -1;
+        }
         Py_XDECREF(otemp);
         memcpy(optr, &res, sizeof(res));
     }
+    return 0;
 
 }
 
@@ -1654,6 +1664,7 @@ array_deepcopy(PyArrayObject *self, PyObject *args)
     npy_intp *strideptr, *innersizeptr;
     npy_intp stride, count;
     PyObject *copy, *deepcopy;
+    int deepcopy_res;
 
     if (!PyArg_ParseTuple(args, "O:__deepcopy__", &visit)) {
         return NULL;
@@ -1704,8 +1715,12 @@ array_deepcopy(PyArrayObject *self, PyObject *args)
                 stride = *strideptr;
                 count = *innersizeptr;
                 while (count--) {
-                    _deepcopy_call(data, data, PyArray_DESCR(copied_array),
-                                   deepcopy, visit);
+                    deepcopy_res = _deepcopy_call(data, data, PyArray_DESCR(copied_array),
+                                                  deepcopy, visit);
+                    if (deepcopy_res == -1) {
+                        return NULL;
+                    }
+
                     data += stride;
                 }
             } while (iternext(iter));
diff --git a/numpy/core/tests/test_multiarray.py b/numpy/core/tests/test_multiarray.py
index ec89f2b37..a7bdf335b 100644
--- a/numpy/core/tests/test_multiarray.py
+++ b/numpy/core/tests/test_multiarray.py
@@ -2207,6 +2207,28 @@ class TestMethods:
         assert_c(a.copy('C'))
         assert_fortran(a.copy('F'))
         assert_c(a.copy('A'))
+    
+    @pytest.mark.parametrize("dtype", ['O', np.int32, 'i,O'])
+    def test__deepcopy__(self, dtype):
+        # Force the entry of NULLs into array
+        a = np.empty(4, dtype=dtype)
+        ctypes.memset(a.ctypes.data, 0, a.nbytes)
+
+        # Ensure no error is raised, see gh-21833
+        b = a.__deepcopy__({})
+
+        a[0] = 42
+        with pytest.raises(AssertionError):
+            assert_array_equal(a, b)
+
+    def test__deepcopy__catches_failure(self):
+        class MyObj:
+            def __deepcopy__(self, *args, **kwargs):
+                raise RuntimeError
+
+        arr = np.array([1, MyObj(), 3], dtype='O')
+        with pytest.raises(RuntimeError):
+            arr.__deepcopy__({})
 
     def test_sort_order(self):
         # Test sorting an array with fields