Handle additions to "_DataclassParams" signature in Py3.12 dataclasses (GH-5368)

Fixes https://github.com/cython/cython/issues/5346

Uses a potentially slightly inefficient helper function
to inspect the signature of dataclasses.field and
dataclasses._DataclassParams and drop any arguments that are
unsupported (i.e. ones that we're passing because they apply to
later Python versions).

3 files changed, 137 insertions, 16 deletions

diff --git a/Cython/Compiler/Dataclass.py b/Cython/Compiler/Dataclass.py
index 7d6859170..e775e9182 100644
--- a/Cython/Compiler/Dataclass.py
+++ b/Cython/Compiler/Dataclass.py
@@ -34,7 +34,21 @@ def make_dataclasses_module_callnode(pos):
         args=[],
     )
 
-_INTERNAL_DEFAULTSHOLDER_NAME = EncodedString('__pyx_dataclass_defaults')
+def make_dataclass_call_helper(pos, callable, kwds):
+    utility_code = UtilityCode.load_cached("DataclassesCallHelper", "Dataclasses.c")
+    func_type = PyrexTypes.CFuncType(
+        PyrexTypes.py_object_type, [
+            PyrexTypes.CFuncTypeArg("callable", PyrexTypes.py_object_type, None),
+            PyrexTypes.CFuncTypeArg("kwds", PyrexTypes.py_object_type, None)
+        ],
+    )
+    return ExprNodes.PythonCapiCallNode(
+        pos,
+        function_name="__Pyx_DataclassesCallHelper",
+        func_type=func_type,
+        utility_code=utility_code,
+        args=[callable, kwds],
+    )
 
 
 class RemoveAssignmentsToNames(VisitorTransform, SkipDeclarations):
@@ -298,8 +312,7 @@ def handle_cclass_dataclass(node, dataclass_args, analyse_decs_transform):
                       "Arguments passed to cython.dataclasses.dataclass must be True or False")
             kwargs[k] = v.value
 
-    # remove everything that does not belong into _DataclassParams()
-    kw_only = kwargs.pop("kw_only")
+    kw_only = kwargs['kw_only']
 
     fields = process_class_get_fields(node)
 
@@ -314,11 +327,14 @@ def handle_cclass_dataclass(node, dataclass_args, analyse_decs_transform):
         node.pos,
         [ (ExprNodes.IdentifierStringNode(node.pos, value=EncodedString(k)),
            ExprNodes.BoolNode(node.pos, value=v))
-          for k, v in kwargs.items() ])
-    dataclass_params = ExprNodes.GeneralCallNode(node.pos,
-                                                 function = dataclass_params_func,
-                                                 positional_args = ExprNodes.TupleNode(node.pos, args=[]),
-                                                 keyword_args = dataclass_params_keywords)
+          for k, v in kwargs.items() ] +
+        [ (ExprNodes.IdentifierStringNode(node.pos, value=EncodedString(k)),
+           ExprNodes.BoolNode(node.pos, value=v))
+          for k, v in [('kw_only', kw_only), ('match_args', False),
+                       ('slots', False), ('weakref_slot', False)]
+        ])
+    dataclass_params = make_dataclass_call_helper(
+        node.pos, dataclass_params_func, dataclass_params_keywords)
     dataclass_params_assignment = Nodes.SingleAssignmentNode(
         node.pos,
         lhs = ExprNodes.NameNode(node.pos, name=EncodedString("__dataclass_params__")),
@@ -792,10 +808,9 @@ def _set_up_dataclass_fields(node, fields, dataclass_module):
               for k, v in field.iterate_record_node_arguments()]
 
         )
-        dc_field_call = ExprNodes.GeneralCallNode(
-            node.pos, function = field_func,
-            positional_args = ExprNodes.TupleNode(node.pos, args=[]),
-            keyword_args = dc_field_keywords)
+        dc_field_call = make_dataclass_call_helper(
+            node.pos, field_func, dc_field_keywords
+        )
         dc_fields.key_value_pairs.append(
             ExprNodes.DictItemNode(
                 node.pos,
diff --git a/Cython/Utility/Dataclasses.c b/Cython/Utility/Dataclasses.c
index 8640b6c0c..fc6a88d94 100644
--- a/Cython/Utility/Dataclasses.c
+++ b/Cython/Utility/Dataclasses.c
@@ -76,3 +76,103 @@ static PyObject* __Pyx_Load_{{cname}}_Module(void); /* proto */
 static PyObject* __Pyx_Load_{{cname}}_Module(void) {
     return __Pyx_LoadInternalModule("{{cname}}", {{py_code}});
 }
+
+//////////////////// DataclassesCallHelper.proto ////////////////////////
+
+static PyObject* __Pyx_DataclassesCallHelper(PyObject *callable, PyObject *kwds); /* proto */
+
+//////////////////// DataclassesCallHelper ////////////////////////
+//@substitute: naming
+
+// The signature of a few of the dataclasses module functions has
+// been expanded over the years. Cython always passes the full set
+// of arguments from the most recent version we know of, so needs
+// to remove any arguments that don't exist on earlier versions.
+
+#if PY_MAJOR_VERSION >= 3
+static int __Pyx_DataclassesCallHelper_FilterToDict(PyObject *callable, PyObject *kwds, PyObject *new_kwds, PyObject *args_list, int is_kwonly) {
+    Py_ssize_t size, i;
+    size = PySequence_Size(args_list);
+    if (size == -1) return -1;
+
+    for (i=0; i<size; ++i) {
+        PyObject *key, *value;
+        int setitem_result;
+        key = PySequence_GetItem(args_list, i);
+        if (!key) return -1;
+
+        if (PyUnicode_Check(key) && (
+                PyUnicode_CompareWithASCIIString(key, "self") == 0 ||
+                // namedtuple constructor in fallback code
+                PyUnicode_CompareWithASCIIString(key, "_cls") == 0)) {
+            Py_DECREF(key);
+            continue;
+        }
+
+        value = PyDict_GetItem(kwds, key);
+        if (!value) {
+            if (is_kwonly) {
+                Py_DECREF(key);
+                continue;
+            } else {
+                // The most likely reason for this is that Cython
+                // hasn't kept up to date with the Python dataclasses module.
+                // To be nice to our users, try not to fail, but ask them
+                // to report a bug so we can keep up to date.
+                value = Py_None;
+                if (PyErr_WarnFormat(
+                        PyExc_RuntimeWarning, 1,
+                        "Argument %S not passed to %R. This is likely a bug in Cython so please report it.",
+                        key, callable) == -1) {
+                    Py_DECREF(key);
+                    return -1;
+                }
+            }
+        }
+        Py_INCREF(value);
+        setitem_result = PyDict_SetItem(new_kwds, key, value);
+        Py_DECREF(key);
+        Py_DECREF(value);
+        if (setitem_result == -1) return -1;
+    }
+    return 0;
+}
+#endif
+
+static PyObject* __Pyx_DataclassesCallHelper(PyObject *callable, PyObject *kwds) {
+#if PY_MAJOR_VERSION < 3
+    // We're falling back to our full replacement anyway
+    return PyObject_Call(callable, $empty_tuple, kwds);
+#else
+    PyObject *new_kwds=NULL, *result=NULL;
+    PyObject *inspect;
+    PyObject *args_list=NULL, *kwonly_args_list=NULL, *getfullargspec_result=NULL;
+
+    // Going via inspect to work out what arguments to pass is unlikely to be the
+    // fastest thing ever. However, it is compatible, and only happens once
+    // at module-import time.
+    inspect = PyImport_ImportModule("inspect");
+    if (!inspect) goto bad;
+    getfullargspec_result = PyObject_CallMethodObjArgs(inspect, PYUNICODE("getfullargspec"), callable, NULL);
+    Py_DECREF(inspect);
+    if (!getfullargspec_result) goto bad;
+    args_list = PyObject_GetAttrString(getfullargspec_result, "args");
+    if (!args_list) goto bad;
+    kwonly_args_list = PyObject_GetAttrString(getfullargspec_result, "kwonlyargs");
+    if (!kwonly_args_list) goto bad;
+
+    new_kwds = PyDict_New();
+    if (!new_kwds) goto bad;
+
+    // copy over only those arguments that are in the specification
+    if (__Pyx_DataclassesCallHelper_FilterToDict(callable, kwds, new_kwds, args_list, 0) == -1) goto bad;
+    if (__Pyx_DataclassesCallHelper_FilterToDict(callable, kwds, new_kwds, kwonly_args_list, 1) == -1) goto bad;
+    result = PyObject_Call(callable, $empty_tuple, new_kwds);
+bad:
+    Py_XDECREF(getfullargspec_result);
+    Py_XDECREF(args_list);
+    Py_XDECREF(kwonly_args_list);
+    Py_XDECREF(new_kwds);
+    return result;
+#endif
+}
diff --git a/Cython/Utility/Dataclasses.py b/Cython/Utility/Dataclasses.py
index 2fb5fce78..2aa2d25a3 100644
--- a/Cython/Utility/Dataclasses.py
+++ b/Cython/Utility/Dataclasses.py
@@ -18,7 +18,8 @@ class _MISSING_TYPE(object):
 MISSING = _MISSING_TYPE()
 
 _DataclassParams = namedtuple('_DataclassParams',
-    ["init", "repr", "eq", "order", "unsafe_hash", "frozen"])
+    ["init", "repr", "eq", "order", "unsafe_hash", "frozen",
+     "match_args", "kw_only", "slots", "weakref_slot"])
 class Field(object):
     __slots__ = ('name',
                  'type',
@@ -29,11 +30,12 @@ class Field(object):
                  'init',
                  'compare',
                  'metadata',
+                 'kw_only',
                  '_field_type',  # Private: not to be used by user code.
                  )
 
     def __init__(self, default, default_factory, init, repr, hash, compare,
-                 metadata):
+                 metadata, kw_only):
         self.name = None
         self.type = None
         self.default = default
@@ -47,6 +49,7 @@ class Field(object):
         self.metadata = (MappingProxyType({})
                          if metadata is None else
                          MappingProxyType(metadata))
+        self.kw_only = kw_only
         self._field_type = None
 
     def __repr__(self):
@@ -60,10 +63,11 @@ class Field(object):
                 'hash={6!r},'
                 'compare={7!r},'
                 'metadata={8!r},'
+                'kwonly={9!r},'
                 ')'.format(self.name, self.type, self.default,
                            self.default_factory, self.init,
                            self.repr, self.hash, self.compare,
-                           self.metadata))
+                           self.metadata, self.kw_only))
 
 # A sentinel object for default values to signal that a default
 # factory will be used.  This is given a nice repr() which will appear
@@ -95,6 +99,7 @@ def field(*ignore, **kwds):
     hash = kwds.pop("hash", None)
     compare = kwds.pop("compare", True)
     metadata = kwds.pop("metadata", None)
+    kw_only = kwds.pop("kw_only", None)
 
     if kwds:
         raise ValueError("field received unexpected keyword arguments: %s"
@@ -103,4 +108,5 @@ def field(*ignore, **kwds):
         raise ValueError('cannot specify both default and default_factory')
     if ignore:
         raise ValueError("'field' does not take any positional arguments")
-    return Field(default, default_factory, init, repr, hash, compare, metadata)
+    return Field(default, default_factory, init,
+                 repr, hash, compare, metadata, kw_only)