BUG: check return value of _buffer_format_string (#11754)

test, fix for not checking return value from call to _buffer_format_string, exposed when datetime64 dtype used in a field. Also fixes some variable types to be Py_ssize_t. 

* BUG: check return value of _buffer_format_string

* fix from review, document _buffer_format_string

* BUG: Fix type in declaration of old_offset.

Type should be Py_ssize_t.

* MAINT: Explicitly return -1 for error.

* MAINT: Fix typo and style nit.

2 files changed, 23 insertions, 6 deletions

diff --git a/numpy/core/src/multiarray/buffer.c b/numpy/core/src/multiarray/buffer.c
index 0325f3c6a..d549a3029 100644
--- a/numpy/core/src/multiarray/buffer.c
+++ b/numpy/core/src/multiarray/buffer.c
@@ -175,6 +175,14 @@ _is_natively_aligned_at(PyArray_Descr *descr,
     return 1;
 }
 
+/*
+ * Fill in str with an appropriate PEP 3118 format string, based on
+ * descr. For structured dtypes, calls itself recursively. Each call extends
+ * str at offset then updates offset, and uses  descr->byteorder, (and
+ * possibly the byte order in obj) to determine the byte-order char.
+ *
+ * Returns 0 for success, -1 for failure
+ */
 static int
 _buffer_format_string(PyArray_Descr *descr, _tmp_string_t *str,
                       PyObject* obj, Py_ssize_t *offset,
@@ -195,8 +203,8 @@ _buffer_format_string(PyArray_Descr *descr, _tmp_string_t *str,
         PyObject *item, *subarray_tuple;
         Py_ssize_t total_count = 1;
         Py_ssize_t dim_size;
+        Py_ssize_t old_offset;
         char buf[128];
-        int old_offset;
         int ret;
 
         if (PyTuple_Check(descr->subarray->shape)) {
@@ -230,15 +238,15 @@ _buffer_format_string(PyArray_Descr *descr, _tmp_string_t *str,
         return ret;
     }
     else if (PyDataType_HASFIELDS(descr)) {
-        int base_offset = *offset;
+        Py_ssize_t base_offset = *offset;
 
         _append_str(str, "T{");
         for (k = 0; k < PyTuple_GET_SIZE(descr->names); ++k) {
             PyObject *name, *item, *offset_obj, *tmp;
             PyArray_Descr *child;
             char *p;
-            Py_ssize_t len;
-            int new_offset;
+            Py_ssize_t len, new_offset;
+            int ret;
 
             name = PyTuple_GET_ITEM(descr->names, k);
             item = PyDict_GetItem(descr->fields, name);
@@ -266,8 +274,11 @@ _buffer_format_string(PyArray_Descr *descr, _tmp_string_t *str,
             }
 
             /* Insert child item */
-            _buffer_format_string(child, str, obj, offset,
+            ret = _buffer_format_string(child, str, obj, offset,
                                   active_byteorder);
+            if (ret < 0) {
+                return -1;
+            }
 
             /* Insert field name */
 #if defined(NPY_PY3K)
diff --git a/numpy/core/tests/test_scalarbuffer.py b/numpy/core/tests/test_scalarbuffer.py
index 6d57a5014..9e75fec3e 100644
--- a/numpy/core/tests/test_scalarbuffer.py
+++ b/numpy/core/tests/test_scalarbuffer.py
@@ -5,7 +5,7 @@ import sys
 import numpy as np
 import pytest
 
-from numpy.testing import assert_, assert_equal
+from numpy.testing import assert_, assert_equal, assert_raises
 
 # PEP3118 format strings for native (standard alignment and byteorder) types
 scalars_and_codes = [
@@ -77,3 +77,9 @@ class TestScalarPEP3118(object):
         mv_a = memoryview(a)
         assert_equal(mv_x.itemsize, mv_a.itemsize)
         assert_equal(mv_x.format, mv_a.format)
+
+    def test_invalid_buffer_format(self):
+        # datetime64 cannot be used in a buffer yet
+        dt = np.dtype([('a', int), ('b', 'M8[s]')])
+        a = np.empty(1, dt)
+        assert_raises((ValueError, BufferError), memoryview, a[0])