From 6562977b6a026761071df890854ac71d446bb3b1 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 9 Jun 2016 16:30:29 +0300 Subject: Issue #26282: PyArg_ParseTupleAndKeywords() and Argument Clinic now support positional-only and keyword parameters in the same function. --- Python/getargs.c | 121 ++++++++++++++++++++++++++++++++++++------------------- 1 file changed, 79 insertions(+), 42 deletions(-) (limited to 'Python/getargs.c') diff --git a/Python/getargs.c b/Python/getargs.c index 9858bd560c..4418ebb664 100644 --- a/Python/getargs.c +++ b/Python/getargs.c @@ -1443,7 +1443,8 @@ vgetargskeywords(PyObject *args, PyObject *keywords, const char *format, const char *fname, *msg, *custom_msg, *keyword; int min = INT_MAX; int max = INT_MAX; - int i, len; + int i, pos, len; + int skip = 0; Py_ssize_t nargs, nkeywords; PyObject *current_arg; freelistentry_t static_entries[STATIC_FREELIST_ENTRIES]; @@ -1471,9 +1472,17 @@ vgetargskeywords(PyObject *args, PyObject *keywords, const char *format, custom_msg++; } + /* scan kwlist and count the number of positional-only parameters */ + for (pos = 0; kwlist[pos] && !*kwlist[pos]; pos++) { + } /* scan kwlist and get greatest possible nbr of args */ - for (len=0; kwlist[len]; len++) - continue; + for (len = pos; kwlist[len]; len++) { + if (!*kwlist[len]) { + PyErr_SetString(PyExc_SystemError, + "Empty keyword parameter name"); + return cleanreturn(0, &freelist); + } + } if (len > STATIC_FREELIST_ENTRIES) { freelist.entries = PyMem_NEW(freelistentry_t, len); @@ -1526,6 +1535,14 @@ vgetargskeywords(PyObject *args, PyObject *keywords, const char *format, max = i; format++; + if (max < pos) { + PyErr_SetString(PyExc_SystemError, + "Empty parameter name after $"); + return cleanreturn(0, &freelist); + } + if (skip) { + break; + } if (max < nargs) { PyErr_Format(PyExc_TypeError, "Function takes %s %d positional arguments" @@ -1541,48 +1558,59 @@ vgetargskeywords(PyObject *args, PyObject *keywords, const char *format, "format specifiers (%d)", len, i); return cleanreturn(0, &freelist); } - current_arg = NULL; - if (nkeywords) { - current_arg = PyDict_GetItemString(keywords, keyword); - } - if (current_arg) { - --nkeywords; - if (i < nargs) { - /* arg present in tuple and in dict */ - PyErr_Format(PyExc_TypeError, - "Argument given by name ('%s') " - "and position (%d)", - keyword, i+1); - return cleanreturn(0, &freelist); + if (!skip) { + current_arg = NULL; + if (nkeywords && i >= pos) { + current_arg = PyDict_GetItemString(keywords, keyword); + if (!current_arg && PyErr_Occurred()) { + return cleanreturn(0, &freelist); + } } - } - else if (nkeywords && PyErr_Occurred()) - return cleanreturn(0, &freelist); - else if (i < nargs) - current_arg = PyTuple_GET_ITEM(args, i); - - if (current_arg) { - msg = convertitem(current_arg, &format, p_va, flags, - levels, msgbuf, sizeof(msgbuf), &freelist); - if (msg) { - seterror(i+1, msg, levels, fname, custom_msg); - return cleanreturn(0, &freelist); + if (current_arg) { + --nkeywords; + if (i < nargs) { + /* arg present in tuple and in dict */ + PyErr_Format(PyExc_TypeError, + "Argument given by name ('%s') " + "and position (%d)", + keyword, i+1); + return cleanreturn(0, &freelist); + } + } + else if (i < nargs) + current_arg = PyTuple_GET_ITEM(args, i); + + if (current_arg) { + msg = convertitem(current_arg, &format, p_va, flags, + levels, msgbuf, sizeof(msgbuf), &freelist); + if (msg) { + seterror(i+1, msg, levels, fname, custom_msg); + return cleanreturn(0, &freelist); + } + continue; } - continue; - } - if (i < min) { - PyErr_Format(PyExc_TypeError, "Required argument " - "'%s' (pos %d) not found", - keyword, i+1); - return cleanreturn(0, &freelist); + if (i < min) { + if (i < pos) { + assert (min == INT_MAX); + assert (max == INT_MAX); + skip = 1; + } + else { + PyErr_Format(PyExc_TypeError, "Required argument " + "'%s' (pos %d) not found", + keyword, i+1); + return cleanreturn(0, &freelist); + } + } + /* current code reports success when all required args + * fulfilled and no keyword args left, with no further + * validation. XXX Maybe skip this in debug build ? + */ + if (!nkeywords && !skip) { + return cleanreturn(1, &freelist); + } } - /* current code reports success when all required args - * fulfilled and no keyword args left, with no further - * validation. XXX Maybe skip this in debug build ? - */ - if (!nkeywords) - return cleanreturn(1, &freelist); /* We are into optional args, skip thru to any remaining * keyword args */ @@ -1594,6 +1622,15 @@ vgetargskeywords(PyObject *args, PyObject *keywords, const char *format, } } + if (skip) { + PyErr_Format(PyExc_TypeError, + "Function takes %s %d positional arguments" + " (%d given)", + (Py_MIN(pos, min) < i) ? "at least" : "exactly", + Py_MIN(pos, min), nargs); + return cleanreturn(0, &freelist); + } + if (!IS_END_OF_FORMAT(*format) && (*format != '|') && (*format != '$')) { PyErr_Format(PyExc_SystemError, "more argument specifiers than keyword list entries " @@ -1613,7 +1650,7 @@ vgetargskeywords(PyObject *args, PyObject *keywords, const char *format, return cleanreturn(0, &freelist); } for (i = 0; i < len; i++) { - if (!PyUnicode_CompareWithASCIIString(key, kwlist[i])) { + if (*kwlist[i] && !PyUnicode_CompareWithASCIIString(key, kwlist[i])) { match = 1; break; } -- cgit v1.2.1 From 96d299b2b87dd2311bcc94cbab7b1cfa2c30ccba Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 14 Aug 2016 10:52:18 +0300 Subject: Issue #27574: Decreased an overhead of parsing keyword arguments in functions implemented with using Argument Clinic. --- Python/getargs.c | 449 +++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 437 insertions(+), 12 deletions(-) (limited to 'Python/getargs.c') diff --git a/Python/getargs.c b/Python/getargs.c index 4418ebb664..cf0ad26920 100644 --- a/Python/getargs.c +++ b/Python/getargs.c @@ -18,6 +18,11 @@ int PyArg_ParseTupleAndKeywords(PyObject *, PyObject *, int PyArg_VaParseTupleAndKeywords(PyObject *, PyObject *, const char *, char **, va_list); +int _PyArg_ParseTupleAndKeywordsFast(PyObject *, PyObject *, + struct _PyArg_Parser *, ...); +int _PyArg_VaParseTupleAndKeywordsFast(PyObject *, PyObject *, + struct _PyArg_Parser *, va_list); + #ifdef HAVE_DECLSPEC_DLL /* Export functions */ PyAPI_FUNC(int) _PyArg_Parse_SizeT(PyObject *, const char *, ...); @@ -28,6 +33,11 @@ PyAPI_FUNC(PyObject *) _Py_BuildValue_SizeT(const char *, ...); PyAPI_FUNC(int) _PyArg_VaParse_SizeT(PyObject *, const char *, va_list); PyAPI_FUNC(int) _PyArg_VaParseTupleAndKeywords_SizeT(PyObject *, PyObject *, const char *, char **, va_list); + +PyAPI_FUNC(int) _PyArg_ParseTupleAndKeywordsFast_SizeT(PyObject *, PyObject *, + struct _PyArg_Parser *, ...); +PyAPI_FUNC(int) _PyArg_VaParseTupleAndKeywordsFast_SizeT(PyObject *, PyObject *, + struct _PyArg_Parser *, va_list); #endif #define FLAG_COMPAT 1 @@ -67,6 +77,8 @@ static int getbuffer(PyObject *, Py_buffer *, const char**); static int vgetargskeywords(PyObject *, PyObject *, const char *, char **, va_list *, int); +static int vgetargskeywordsfast(PyObject *, PyObject *, + struct _PyArg_Parser *, va_list *, int); static const char *skipitem(const char **, va_list *, int); int @@ -1417,6 +1429,91 @@ _PyArg_VaParseTupleAndKeywords_SizeT(PyObject *args, return retval; } +int +_PyArg_ParseTupleAndKeywordsFast(PyObject *args, PyObject *keywords, + struct _PyArg_Parser *parser, ...) +{ + int retval; + va_list va; + + if ((args == NULL || !PyTuple_Check(args)) || + (keywords != NULL && !PyDict_Check(keywords)) || + parser == NULL) + { + PyErr_BadInternalCall(); + return 0; + } + + va_start(va, parser); + retval = vgetargskeywordsfast(args, keywords, parser, &va, 0); + va_end(va); + return retval; +} + +int +_PyArg_ParseTupleAndKeywordsFast_SizeT(PyObject *args, PyObject *keywords, + struct _PyArg_Parser *parser, ...) +{ + int retval; + va_list va; + + if ((args == NULL || !PyTuple_Check(args)) || + (keywords != NULL && !PyDict_Check(keywords)) || + parser == NULL) + { + PyErr_BadInternalCall(); + return 0; + } + + va_start(va, parser); + retval = vgetargskeywordsfast(args, keywords, parser, &va, FLAG_SIZE_T); + va_end(va); + return retval; +} + + +int +_PyArg_VaParseTupleAndKeywordsFast(PyObject *args, PyObject *keywords, + struct _PyArg_Parser *parser, va_list va) +{ + int retval; + va_list lva; + + if ((args == NULL || !PyTuple_Check(args)) || + (keywords != NULL && !PyDict_Check(keywords)) || + parser == NULL) + { + PyErr_BadInternalCall(); + return 0; + } + + Py_VA_COPY(lva, va); + + retval = vgetargskeywordsfast(args, keywords, parser, &lva, 0); + return retval; +} + +int +_PyArg_VaParseTupleAndKeywordsFast_SizeT(PyObject *args, PyObject *keywords, + struct _PyArg_Parser *parser, va_list va) +{ + int retval; + va_list lva; + + if ((args == NULL || !PyTuple_Check(args)) || + (keywords != NULL && !PyDict_Check(keywords)) || + parser == NULL) + { + PyErr_BadInternalCall(); + return 0; + } + + Py_VA_COPY(lva, va); + + retval = vgetargskeywordsfast(args, keywords, parser, &lva, FLAG_SIZE_T); + return retval; +} + int PyArg_ValidateKeywordArguments(PyObject *kwargs) { @@ -1541,6 +1638,9 @@ vgetargskeywords(PyObject *args, PyObject *keywords, const char *format, return cleanreturn(0, &freelist); } if (skip) { + /* Now we know the minimal and the maximal numbers of + * positional arguments and can raise an exception with + * informative message (see below). */ break; } if (max < nargs) { @@ -1595,6 +1695,10 @@ vgetargskeywords(PyObject *args, PyObject *keywords, const char *format, assert (min == INT_MAX); assert (max == INT_MAX); skip = 1; + /* At that moment we still don't know the minimal and + * the maximal numbers of positional arguments. Raising + * an exception is deferred until we encounter | and $ + * or the end of the format. */ } else { PyErr_Format(PyExc_TypeError, "Required argument " @@ -1669,6 +1773,300 @@ vgetargskeywords(PyObject *args, PyObject *keywords, const char *format, } +/* List of static parsers. */ +static struct _PyArg_Parser *static_arg_parsers = NULL; + +static int +parser_init(struct _PyArg_Parser *parser) +{ + const char * const *keywords; + const char *format, *msg; + int i, len, min, max, nkw; + PyObject *kwtuple; + + assert(parser->format != NULL); + assert(parser->keywords != NULL); + if (parser->kwtuple != NULL) { + return 1; + } + + /* grab the function name or custom error msg first (mutually exclusive) */ + parser->fname = strchr(parser->format, ':'); + if (parser->fname) { + parser->fname++; + parser->custom_msg = NULL; + } + else { + parser->custom_msg = strchr(parser->format,';'); + if (parser->custom_msg) + parser->custom_msg++; + } + + keywords = parser->keywords; + /* scan keywords and count the number of positional-only parameters */ + for (i = 0; keywords[i] && !*keywords[i]; i++) { + } + parser->pos = i; + /* scan keywords and get greatest possible nbr of args */ + for (; keywords[i]; i++) { + if (!*keywords[i]) { + PyErr_SetString(PyExc_SystemError, + "Empty keyword parameter name"); + return 0; + } + } + len = i; + + min = max = INT_MAX; + format = parser->format; + for (i = 0; i < len; i++) { + if (*format == '|') { + if (min != INT_MAX) { + PyErr_SetString(PyExc_SystemError, + "Invalid format string (| specified twice)"); + return 0; + } + if (max != INT_MAX) { + PyErr_SetString(PyExc_SystemError, + "Invalid format string ($ before |)"); + return 0; + } + min = i; + format++; + } + if (*format == '$') { + if (max != INT_MAX) { + PyErr_SetString(PyExc_SystemError, + "Invalid format string ($ specified twice)"); + return 0; + } + if (i < parser->pos) { + PyErr_SetString(PyExc_SystemError, + "Empty parameter name after $"); + return 0; + } + max = i; + format++; + } + if (IS_END_OF_FORMAT(*format)) { + PyErr_Format(PyExc_SystemError, + "More keyword list entries (%d) than " + "format specifiers (%d)", len, i); + return 0; + } + + msg = skipitem(&format, NULL, 0); + if (msg) { + PyErr_Format(PyExc_SystemError, "%s: '%s'", msg, + format); + return 0; + } + } + parser->min = Py_MIN(min, len); + parser->max = Py_MIN(max, len); + + if (!IS_END_OF_FORMAT(*format) && (*format != '|') && (*format != '$')) { + PyErr_Format(PyExc_SystemError, + "more argument specifiers than keyword list entries " + "(remaining format:'%s')", format); + return 0; + } + + nkw = len - parser->pos; + kwtuple = PyTuple_New(nkw); + if (kwtuple == NULL) { + return 0; + } + keywords = parser->keywords + parser->pos; + for (i = 0; i < nkw; i++) { + PyObject *str = PyUnicode_FromString(keywords[i]); + if (str == NULL) { + Py_DECREF(kwtuple); + return 0; + } + PyUnicode_InternInPlace(&str); + PyTuple_SET_ITEM(kwtuple, i, str); + } + parser->kwtuple = kwtuple; + + assert(parser->next == NULL); + parser->next = static_arg_parsers; + static_arg_parsers = parser; + return 1; +} + +static void +parser_clear(struct _PyArg_Parser *parser) +{ + Py_CLEAR(parser->kwtuple); +} + +static int +vgetargskeywordsfast(PyObject *args, PyObject *keywords, + struct _PyArg_Parser *parser, + va_list *p_va, int flags) +{ + PyObject *kwtuple; + char msgbuf[512]; + int levels[32]; + const char *format; + const char *msg; + PyObject *keyword; + int i, pos, len; + Py_ssize_t nargs, nkeywords; + PyObject *current_arg; + freelistentry_t static_entries[STATIC_FREELIST_ENTRIES]; + freelist_t freelist; + + freelist.entries = static_entries; + freelist.first_available = 0; + freelist.entries_malloced = 0; + + assert(args != NULL && PyTuple_Check(args)); + assert(keywords == NULL || PyDict_Check(keywords)); + assert(parser != NULL); + assert(p_va != NULL); + + if (!parser_init(parser)) { + return 0; + } + + kwtuple = parser->kwtuple; + pos = parser->pos; + len = pos + PyTuple_GET_SIZE(kwtuple); + + if (len > STATIC_FREELIST_ENTRIES) { + freelist.entries = PyMem_NEW(freelistentry_t, len); + if (freelist.entries == NULL) { + PyErr_NoMemory(); + return 0; + } + freelist.entries_malloced = 1; + } + + nargs = PyTuple_GET_SIZE(args); + nkeywords = (keywords == NULL) ? 0 : PyDict_Size(keywords); + if (nargs + nkeywords > len) { + PyErr_Format(PyExc_TypeError, + "%s%s takes at most %d argument%s (%zd given)", + (parser->fname == NULL) ? "function" : parser->fname, + (parser->fname == NULL) ? "" : "()", + len, + (len == 1) ? "" : "s", + nargs + nkeywords); + return cleanreturn(0, &freelist); + } + if (parser->max < nargs) { + PyErr_Format(PyExc_TypeError, + "Function takes %s %d positional arguments (%d given)", + (parser->min != INT_MAX) ? "at most" : "exactly", + parser->max, nargs); + return cleanreturn(0, &freelist); + } + + format = parser->format; + /* convert tuple args and keyword args in same loop, using kwtuple to drive process */ + for (i = 0; i < len; i++) { + keyword = (i >= pos) ? PyTuple_GET_ITEM(kwtuple, i - pos) : NULL; + if (*format == '|') { + format++; + } + if (*format == '$') { + format++; + } + assert(!IS_END_OF_FORMAT(*format)); + + current_arg = NULL; + if (nkeywords && i >= pos) { + current_arg = PyDict_GetItem(keywords, keyword); + if (!current_arg && PyErr_Occurred()) { + return cleanreturn(0, &freelist); + } + } + if (current_arg) { + --nkeywords; + if (i < nargs) { + /* arg present in tuple and in dict */ + PyErr_Format(PyExc_TypeError, + "Argument given by name ('%U') " + "and position (%d)", + keyword, i+1); + return cleanreturn(0, &freelist); + } + } + else if (i < nargs) + current_arg = PyTuple_GET_ITEM(args, i); + + if (current_arg) { + msg = convertitem(current_arg, &format, p_va, flags, + levels, msgbuf, sizeof(msgbuf), &freelist); + if (msg) { + seterror(i+1, msg, levels, parser->fname, parser->custom_msg); + return cleanreturn(0, &freelist); + } + continue; + } + + if (i < parser->min) { + /* Less arguments than required */ + if (i < pos) { + PyErr_Format(PyExc_TypeError, + "Function takes %s %d positional arguments" + " (%d given)", + (Py_MIN(pos, parser->min) < parser->max) ? "at least" : "exactly", + Py_MIN(pos, parser->min), nargs); + } + else { + PyErr_Format(PyExc_TypeError, "Required argument " + "'%U' (pos %d) not found", + keyword, i+1); + } + return cleanreturn(0, &freelist); + } + /* current code reports success when all required args + * fulfilled and no keyword args left, with no further + * validation. XXX Maybe skip this in debug build ? + */ + if (!nkeywords) { + return cleanreturn(1, &freelist); + } + + /* We are into optional args, skip thru to any remaining + * keyword args */ + msg = skipitem(&format, p_va, flags); + assert(msg == NULL); + } + + assert(IS_END_OF_FORMAT(*format) || (*format == '|') || (*format == '$')); + + /* make sure there are no extraneous keyword arguments */ + if (nkeywords > 0) { + PyObject *key, *value; + Py_ssize_t pos = 0; + while (PyDict_Next(keywords, &pos, &key, &value)) { + int match; + if (!PyUnicode_Check(key)) { + PyErr_SetString(PyExc_TypeError, + "keywords must be strings"); + return cleanreturn(0, &freelist); + } + match = PySequence_Contains(kwtuple, key); + if (match <= 0) { + if (!match) { + PyErr_Format(PyExc_TypeError, + "'%U' is an invalid keyword " + "argument for this function", + key); + } + return cleanreturn(0, &freelist); + } + } + } + + return cleanreturn(1, &freelist); +} + + static const char * skipitem(const char **p_format, va_list *p_va, int flags) { @@ -1705,7 +2103,9 @@ skipitem(const char **p_format, va_list *p_va, int flags) case 'Y': /* string object */ case 'U': /* unicode string object */ { - (void) va_arg(*p_va, void *); + if (p_va != NULL) { + (void) va_arg(*p_va, void *); + } break; } @@ -1713,7 +2113,9 @@ skipitem(const char **p_format, va_list *p_va, int flags) case 'e': /* string with encoding */ { - (void) va_arg(*p_va, const char *); + if (p_va != NULL) { + (void) va_arg(*p_va, const char *); + } if (!(*format == 's' || *format == 't')) /* after 'e', only 's' and 't' is allowed */ goto err; @@ -1728,12 +2130,16 @@ skipitem(const char **p_format, va_list *p_va, int flags) case 'Z': /* unicode string or None */ case 'w': /* buffer, read-write */ { - (void) va_arg(*p_va, char **); + if (p_va != NULL) { + (void) va_arg(*p_va, char **); + } if (*format == '#') { - if (flags & FLAG_SIZE_T) - (void) va_arg(*p_va, Py_ssize_t *); - else - (void) va_arg(*p_va, int *); + if (p_va != NULL) { + if (flags & FLAG_SIZE_T) + (void) va_arg(*p_va, Py_ssize_t *); + else + (void) va_arg(*p_va, int *); + } format++; } else if ((c == 's' || c == 'z' || c == 'y') && *format == '*') { format++; @@ -1745,17 +2151,23 @@ skipitem(const char **p_format, va_list *p_va, int flags) { if (*format == '!') { format++; - (void) va_arg(*p_va, PyTypeObject*); - (void) va_arg(*p_va, PyObject **); + if (p_va != NULL) { + (void) va_arg(*p_va, PyTypeObject*); + (void) va_arg(*p_va, PyObject **); + } } else if (*format == '&') { typedef int (*converter)(PyObject *, void *); - (void) va_arg(*p_va, converter); - (void) va_arg(*p_va, void *); + if (p_va != NULL) { + (void) va_arg(*p_va, converter); + (void) va_arg(*p_va, void *); + } format++; } else { - (void) va_arg(*p_va, PyObject **); + if (p_va != NULL) { + (void) va_arg(*p_va, PyObject **); + } } break; } @@ -1891,6 +2303,19 @@ _PyArg_NoPositional(const char *funcname, PyObject *args) return 0; } +void +_PyArg_Fini(void) +{ + struct _PyArg_Parser *tmp, *s = static_arg_parsers; + while (s) { + tmp = s->next; + s->next = NULL; + parser_clear(s); + s = tmp; + } + static_arg_parsers = NULL; +} + #ifdef __cplusplus }; #endif -- cgit v1.2.1 From 907ccded8b6fa68f6f572ae5de6760595df0d5a1 Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Mon, 5 Sep 2016 17:44:18 -0700 Subject: require a long long data type (closes #27961) --- Python/getargs.c | 4 ---- 1 file changed, 4 deletions(-) (limited to 'Python/getargs.c') diff --git a/Python/getargs.c b/Python/getargs.c index cf0ad26920..008a4346fb 100644 --- a/Python/getargs.c +++ b/Python/getargs.c @@ -769,7 +769,6 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, break; } -#ifdef HAVE_LONG_LONG case 'L': {/* PY_LONG_LONG */ PY_LONG_LONG *p = va_arg( *p_va, PY_LONG_LONG * ); PY_LONG_LONG ival; @@ -793,7 +792,6 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, *p = ival; break; } -#endif case 'f': {/* float */ float *p = va_arg(*p_va, float *); @@ -2088,10 +2086,8 @@ skipitem(const char **p_format, va_list *p_va, int flags) case 'I': /* int sized bitfield */ case 'l': /* long int */ case 'k': /* long int sized bitfield */ -#ifdef HAVE_LONG_LONG case 'L': /* PY_LONG_LONG */ case 'K': /* PY_LONG_LONG sized bitfield */ -#endif case 'n': /* Py_ssize_t */ case 'f': /* float */ case 'd': /* double */ -- cgit v1.2.1 From 914b59533a094e815809f62441d02b560e23057b Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Tue, 6 Sep 2016 10:46:49 -0700 Subject: replace PY_LONG_LONG with long long --- Python/getargs.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) (limited to 'Python/getargs.c') diff --git a/Python/getargs.c b/Python/getargs.c index 008a4346fb..0854cc45e6 100644 --- a/Python/getargs.c +++ b/Python/getargs.c @@ -769,13 +769,13 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, break; } - case 'L': {/* PY_LONG_LONG */ - PY_LONG_LONG *p = va_arg( *p_va, PY_LONG_LONG * ); - PY_LONG_LONG ival; + case 'L': {/* long long */ + long long *p = va_arg( *p_va, long long * ); + long long ival; if (float_argument_error(arg)) RETURN_ERR_OCCURRED; ival = PyLong_AsLongLong(arg); - if (ival == (PY_LONG_LONG)-1 && PyErr_Occurred()) + if (ival == (long long)-1 && PyErr_Occurred()) RETURN_ERR_OCCURRED; else *p = ival; @@ -783,8 +783,8 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, } case 'K': { /* long long sized bitfield */ - unsigned PY_LONG_LONG *p = va_arg(*p_va, unsigned PY_LONG_LONG *); - unsigned PY_LONG_LONG ival; + unsigned long long *p = va_arg(*p_va, unsigned long long *); + unsigned long long ival; if (PyLong_Check(arg)) ival = PyLong_AsUnsignedLongLongMask(arg); else @@ -2086,8 +2086,8 @@ skipitem(const char **p_format, va_list *p_va, int flags) case 'I': /* int sized bitfield */ case 'l': /* long int */ case 'k': /* long int sized bitfield */ - case 'L': /* PY_LONG_LONG */ - case 'K': /* PY_LONG_LONG sized bitfield */ + case 'L': /* long long */ + case 'K': /* long long sized bitfield */ case 'n': /* Py_ssize_t */ case 'f': /* float */ case 'd': /* double */ -- cgit v1.2.1 From 14e79902895c8f92b5cf80483619fac1e3832425 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 9 Sep 2016 17:40:22 -0700 Subject: Add METH_FASTCALL calling convention Issue #27810: Add a new calling convention for C functions: PyObject* func(PyObject *self, PyObject **args, Py_ssize_t nargs, PyObject *kwnames); Where args is a C array of positional arguments followed by values of keyword arguments. nargs is the number of positional arguments, kwnames are keys of keyword arguments. kwnames can be NULL. --- Python/getargs.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'Python/getargs.c') diff --git a/Python/getargs.c b/Python/getargs.c index 0854cc45e6..5e85ea4fc1 100644 --- a/Python/getargs.c +++ b/Python/getargs.c @@ -1992,8 +1992,9 @@ vgetargskeywordsfast(PyObject *args, PyObject *keywords, return cleanreturn(0, &freelist); } } - else if (i < nargs) + else if (i < nargs) { current_arg = PyTuple_GET_ITEM(args, i); + } if (current_arg) { msg = convertitem(current_arg, &format, p_va, flags, -- cgit v1.2.1 From e44f8ec767caa8cedbbf3ce18b0c573df619e694 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 9 Sep 2016 17:40:38 -0700 Subject: Emit METH_FASTCALL code in Argument Clinic Issue #27810: * Modify vgetargskeywordsfast() to work on a C array of PyObject* rather than working on a tuple directly. * Add _PyArg_ParseStack() * Argument Clinic now emits code using the new METH_FASTCALL calling convention --- Python/getargs.c | 184 +++++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 157 insertions(+), 27 deletions(-) (limited to 'Python/getargs.c') diff --git a/Python/getargs.c b/Python/getargs.c index 5e85ea4fc1..017098e226 100644 --- a/Python/getargs.c +++ b/Python/getargs.c @@ -79,6 +79,10 @@ static int vgetargskeywords(PyObject *, PyObject *, const char *, char **, va_list *, int); static int vgetargskeywordsfast(PyObject *, PyObject *, struct _PyArg_Parser *, va_list *, int); +static int vgetargskeywordsfast_impl(PyObject **args, Py_ssize_t nargs, + PyObject *keywords, PyObject *kwnames, + struct _PyArg_Parser *parser, + va_list *p_va, int flags); static const char *skipitem(const char **, va_list *, int); int @@ -1469,6 +1473,46 @@ _PyArg_ParseTupleAndKeywordsFast_SizeT(PyObject *args, PyObject *keywords, return retval; } +int +_PyArg_ParseStack(PyObject **args, Py_ssize_t nargs, PyObject *kwnames, + struct _PyArg_Parser *parser, ...) +{ + int retval; + va_list va; + + if ((kwnames != NULL && !PyTuple_Check(kwnames)) || + parser == NULL) + { + PyErr_BadInternalCall(); + return 0; + } + + va_start(va, parser); + retval = vgetargskeywordsfast_impl(args, nargs, NULL, kwnames, parser, &va, 0); + va_end(va); + return retval; +} + +int +_PyArg_ParseStack_SizeT(PyObject **args, Py_ssize_t nargs, PyObject *kwnames, + struct _PyArg_Parser *parser, ...) +{ + int retval; + va_list va; + + if ((kwnames != NULL && !PyTuple_Check(kwnames)) || + parser == NULL) + { + PyErr_BadInternalCall(); + return 0; + } + + va_start(va, parser); + retval = vgetargskeywordsfast_impl(args, nargs, NULL, kwnames, parser, &va, FLAG_SIZE_T); + va_end(va); + return retval; +} + int _PyArg_VaParseTupleAndKeywordsFast(PyObject *args, PyObject *keywords, @@ -1899,10 +1943,37 @@ parser_clear(struct _PyArg_Parser *parser) Py_CLEAR(parser->kwtuple); } +static PyObject* +find_keyword(PyObject *kwnames, PyObject **kwstack, PyObject *key) +{ + Py_ssize_t i, nkwargs; + + nkwargs = PyTuple_GET_SIZE(kwnames); + for (i=0; i < nkwargs; i++) { + PyObject *kwname = PyTuple_GET_ITEM(kwnames, i); + + /* ptr==ptr should match in most cases since keyword keys + should be interned strings */ + if (kwname == key) { + return kwstack[i]; + } + if (!PyUnicode_Check(kwname)) { + /* ignore non-string keyword keys: + an error will be raised above */ + continue; + } + if (_PyUnicode_EQ(kwname, key)) { + return kwstack[i]; + } + } + return NULL; +} + static int -vgetargskeywordsfast(PyObject *args, PyObject *keywords, - struct _PyArg_Parser *parser, - va_list *p_va, int flags) +vgetargskeywordsfast_impl(PyObject **args, Py_ssize_t nargs, + PyObject *keywords, PyObject *kwnames, + struct _PyArg_Parser *parser, + va_list *p_va, int flags) { PyObject *kwtuple; char msgbuf[512]; @@ -1911,17 +1982,20 @@ vgetargskeywordsfast(PyObject *args, PyObject *keywords, const char *msg; PyObject *keyword; int i, pos, len; - Py_ssize_t nargs, nkeywords; + Py_ssize_t nkeywords; PyObject *current_arg; freelistentry_t static_entries[STATIC_FREELIST_ENTRIES]; freelist_t freelist; + PyObject **kwstack = NULL; freelist.entries = static_entries; freelist.first_available = 0; freelist.entries_malloced = 0; - assert(args != NULL && PyTuple_Check(args)); assert(keywords == NULL || PyDict_Check(keywords)); + assert(kwnames == NULL || PyTuple_Check(kwnames)); + assert((keywords != NULL || kwnames != NULL) + || (keywords == NULL && kwnames == NULL)); assert(parser != NULL); assert(p_va != NULL); @@ -1942,8 +2016,16 @@ vgetargskeywordsfast(PyObject *args, PyObject *keywords, freelist.entries_malloced = 1; } - nargs = PyTuple_GET_SIZE(args); - nkeywords = (keywords == NULL) ? 0 : PyDict_Size(keywords); + if (keywords != NULL) { + nkeywords = PyDict_Size(keywords); + } + else if (kwnames != NULL) { + nkeywords = PyTuple_GET_SIZE(kwnames); + kwstack = args + nargs; + } + else { + nkeywords = 0; + } if (nargs + nkeywords > len) { PyErr_Format(PyExc_TypeError, "%s%s takes at most %d argument%s (%zd given)", @@ -1976,9 +2058,14 @@ vgetargskeywordsfast(PyObject *args, PyObject *keywords, current_arg = NULL; if (nkeywords && i >= pos) { - current_arg = PyDict_GetItem(keywords, keyword); - if (!current_arg && PyErr_Occurred()) { - return cleanreturn(0, &freelist); + if (keywords != NULL) { + current_arg = PyDict_GetItem(keywords, keyword); + if (!current_arg && PyErr_Occurred()) { + return cleanreturn(0, &freelist); + } + } + else { + current_arg = find_keyword(kwnames, kwstack, keyword); } } if (current_arg) { @@ -1993,7 +2080,7 @@ vgetargskeywordsfast(PyObject *args, PyObject *keywords, } } else if (i < nargs) { - current_arg = PyTuple_GET_ITEM(args, i); + current_arg = args[i]; } if (current_arg) { @@ -2040,24 +2127,52 @@ vgetargskeywordsfast(PyObject *args, PyObject *keywords, /* make sure there are no extraneous keyword arguments */ if (nkeywords > 0) { - PyObject *key, *value; - Py_ssize_t pos = 0; - while (PyDict_Next(keywords, &pos, &key, &value)) { - int match; - if (!PyUnicode_Check(key)) { - PyErr_SetString(PyExc_TypeError, - "keywords must be strings"); - return cleanreturn(0, &freelist); + if (keywords != NULL) { + PyObject *key, *value; + Py_ssize_t pos = 0; + while (PyDict_Next(keywords, &pos, &key, &value)) { + int match; + if (!PyUnicode_Check(key)) { + PyErr_SetString(PyExc_TypeError, + "keywords must be strings"); + return cleanreturn(0, &freelist); + } + match = PySequence_Contains(kwtuple, key); + if (match <= 0) { + if (!match) { + PyErr_Format(PyExc_TypeError, + "'%U' is an invalid keyword " + "argument for this function", + key); + } + return cleanreturn(0, &freelist); + } } - match = PySequence_Contains(kwtuple, key); - if (match <= 0) { - if (!match) { - PyErr_Format(PyExc_TypeError, - "'%U' is an invalid keyword " - "argument for this function", - key); + } + else { + Py_ssize_t j, nkwargs; + + nkwargs = PyTuple_GET_SIZE(kwnames); + for (j=0; j < nkwargs; j++) { + PyObject *key = PyTuple_GET_ITEM(kwnames, j); + int match; + + if (!PyUnicode_Check(key)) { + PyErr_SetString(PyExc_TypeError, + "keywords must be strings"); + return cleanreturn(0, &freelist); + } + + match = PySequence_Contains(kwtuple, key); + if (match <= 0) { + if (!match) { + PyErr_Format(PyExc_TypeError, + "'%U' is an invalid keyword " + "argument for this function", + key); + } + return cleanreturn(0, &freelist); } - return cleanreturn(0, &freelist); } } } @@ -2065,6 +2180,21 @@ vgetargskeywordsfast(PyObject *args, PyObject *keywords, return cleanreturn(1, &freelist); } +static int +vgetargskeywordsfast(PyObject *args, PyObject *keywords, + struct _PyArg_Parser *parser, va_list *p_va, int flags) +{ + PyObject **stack; + Py_ssize_t nargs; + + assert(args != NULL && PyTuple_Check(args)); + + stack = &PyTuple_GET_ITEM(args, 0); + nargs = PyTuple_GET_SIZE(args); + return vgetargskeywordsfast_impl(stack, nargs, keywords, NULL, + parser, p_va, flags); +} + static const char * skipitem(const char **p_format, va_list *p_va, int flags) -- cgit v1.2.1 From ad7dfd4fdad5752e927c009df2f36e681b299164 Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Fri, 9 Sep 2016 20:45:06 -0700 Subject: fix export of size_t parse stack function --- Python/getargs.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'Python/getargs.c') diff --git a/Python/getargs.c b/Python/getargs.c index 017098e226..87a5d26a88 100644 --- a/Python/getargs.c +++ b/Python/getargs.c @@ -26,6 +26,8 @@ int _PyArg_VaParseTupleAndKeywordsFast(PyObject *, PyObject *, #ifdef HAVE_DECLSPEC_DLL /* Export functions */ PyAPI_FUNC(int) _PyArg_Parse_SizeT(PyObject *, const char *, ...); +PyAPI_FUNC(int) _PyArg_ParseStack_SizeT(PyObject **args, Py_ssize_t nargs, PyObject *kwnames, + struct _PyArg_Parser *parser, ...); PyAPI_FUNC(int) _PyArg_ParseTuple_SizeT(PyObject *, const char *, ...); PyAPI_FUNC(int) _PyArg_ParseTupleAndKeywords_SizeT(PyObject *, PyObject *, const char *, char **, ...); -- cgit v1.2.1 From 4e5e1c4d80d5f21111dea3096cbde4292c4e1fd7 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 9 Sep 2016 20:56:52 -0700 Subject: Issue #27810: Fix getargs.c compilation on Windows --- Python/getargs.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'Python/getargs.c') diff --git a/Python/getargs.c b/Python/getargs.c index 87a5d26a88..7339191cf7 100644 --- a/Python/getargs.c +++ b/Python/getargs.c @@ -35,11 +35,12 @@ PyAPI_FUNC(PyObject *) _Py_BuildValue_SizeT(const char *, ...); PyAPI_FUNC(int) _PyArg_VaParse_SizeT(PyObject *, const char *, va_list); PyAPI_FUNC(int) _PyArg_VaParseTupleAndKeywords_SizeT(PyObject *, PyObject *, const char *, char **, va_list); - PyAPI_FUNC(int) _PyArg_ParseTupleAndKeywordsFast_SizeT(PyObject *, PyObject *, struct _PyArg_Parser *, ...); PyAPI_FUNC(int) _PyArg_VaParseTupleAndKeywordsFast_SizeT(PyObject *, PyObject *, struct _PyArg_Parser *, va_list); +PyAPI_FUNC(int) _PyArg_ParseStack_SizeT(PyObject **args, Py_ssize_t nargs, + PyObject *kwnames, struct _PyArg_Parser *parser, ...); #endif #define FLAG_COMPAT 1 -- cgit v1.2.1 From a772d529a832fd9b1523a1f54eb7bd4c69410bdb Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Sat, 10 Sep 2016 17:04:36 -0700 Subject: Backed out changeset 3934e070c9db --- Python/getargs.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'Python/getargs.c') diff --git a/Python/getargs.c b/Python/getargs.c index 7339191cf7..87a5d26a88 100644 --- a/Python/getargs.c +++ b/Python/getargs.c @@ -35,12 +35,11 @@ PyAPI_FUNC(PyObject *) _Py_BuildValue_SizeT(const char *, ...); PyAPI_FUNC(int) _PyArg_VaParse_SizeT(PyObject *, const char *, va_list); PyAPI_FUNC(int) _PyArg_VaParseTupleAndKeywords_SizeT(PyObject *, PyObject *, const char *, char **, va_list); + PyAPI_FUNC(int) _PyArg_ParseTupleAndKeywordsFast_SizeT(PyObject *, PyObject *, struct _PyArg_Parser *, ...); PyAPI_FUNC(int) _PyArg_VaParseTupleAndKeywordsFast_SizeT(PyObject *, PyObject *, struct _PyArg_Parser *, va_list); -PyAPI_FUNC(int) _PyArg_ParseStack_SizeT(PyObject **args, Py_ssize_t nargs, - PyObject *kwnames, struct _PyArg_Parser *parser, ...); #endif #define FLAG_COMPAT 1 -- cgit v1.2.1 From 31cb9f369965c4e70f7db1542686a0e845bb06db Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Tue, 20 Sep 2016 20:39:33 -0700 Subject: replace usage of Py_VA_COPY with the (C99) standard va_copy --- Python/getargs.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'Python/getargs.c') diff --git a/Python/getargs.c b/Python/getargs.c index 87a5d26a88..cd80eda0e2 100644 --- a/Python/getargs.c +++ b/Python/getargs.c @@ -142,7 +142,7 @@ PyArg_VaParse(PyObject *args, const char *format, va_list va) { va_list lva; - Py_VA_COPY(lva, va); + va_copy(lva, va); return vgetargs1(args, format, &lva, 0); } @@ -152,7 +152,7 @@ _PyArg_VaParse_SizeT(PyObject *args, const char *format, va_list va) { va_list lva; - Py_VA_COPY(lva, va); + va_copy(lva, va); return vgetargs1(args, format, &lva, FLAG_SIZE_T); } @@ -1402,7 +1402,7 @@ PyArg_VaParseTupleAndKeywords(PyObject *args, return 0; } - Py_VA_COPY(lva, va); + va_copy(lva, va); retval = vgetargskeywords(args, keywords, format, kwlist, &lva, 0); return retval; @@ -1426,7 +1426,7 @@ _PyArg_VaParseTupleAndKeywords_SizeT(PyObject *args, return 0; } - Py_VA_COPY(lva, va); + va_copy(lva, va); retval = vgetargskeywords(args, keywords, format, kwlist, &lva, FLAG_SIZE_T); @@ -1531,7 +1531,7 @@ _PyArg_VaParseTupleAndKeywordsFast(PyObject *args, PyObject *keywords, return 0; } - Py_VA_COPY(lva, va); + va_copy(lva, va); retval = vgetargskeywordsfast(args, keywords, parser, &lva, 0); return retval; @@ -1552,7 +1552,7 @@ _PyArg_VaParseTupleAndKeywordsFast_SizeT(PyObject *args, PyObject *keywords, return 0; } - Py_VA_COPY(lva, va); + va_copy(lva, va); retval = vgetargskeywordsfast(args, keywords, parser, &lva, FLAG_SIZE_T); return retval; -- cgit v1.2.1 From dcf5141d78a919ed53f1cd8acd2542c95be2e5a6 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Wed, 21 Sep 2016 11:37:27 +0200 Subject: va_end() all va_copy()ed va_lists. --- Python/getargs.c | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) (limited to 'Python/getargs.c') diff --git a/Python/getargs.c b/Python/getargs.c index cd80eda0e2..43656eb2aa 100644 --- a/Python/getargs.c +++ b/Python/getargs.c @@ -141,20 +141,26 @@ int PyArg_VaParse(PyObject *args, const char *format, va_list va) { va_list lva; + int retval; va_copy(lva, va); - return vgetargs1(args, format, &lva, 0); + retval = vgetargs1(args, format, &lva, 0); + va_end(lva); + return retval; } int _PyArg_VaParse_SizeT(PyObject *args, const char *format, va_list va) { va_list lva; + int retval; va_copy(lva, va); - return vgetargs1(args, format, &lva, FLAG_SIZE_T); + retval = vgetargs1(args, format, &lva, FLAG_SIZE_T); + va_end(lva); + return retval; } @@ -1405,6 +1411,7 @@ PyArg_VaParseTupleAndKeywords(PyObject *args, va_copy(lva, va); retval = vgetargskeywords(args, keywords, format, kwlist, &lva, 0); + va_end(lva); return retval; } @@ -1430,6 +1437,7 @@ _PyArg_VaParseTupleAndKeywords_SizeT(PyObject *args, retval = vgetargskeywords(args, keywords, format, kwlist, &lva, FLAG_SIZE_T); + va_end(lva); return retval; } @@ -1534,6 +1542,7 @@ _PyArg_VaParseTupleAndKeywordsFast(PyObject *args, PyObject *keywords, va_copy(lva, va); retval = vgetargskeywordsfast(args, keywords, parser, &lva, 0); + va_end(lva); return retval; } @@ -1555,6 +1564,7 @@ _PyArg_VaParseTupleAndKeywordsFast_SizeT(PyObject *args, PyObject *keywords, va_copy(lva, va); retval = vgetargskeywordsfast(args, keywords, parser, &lva, FLAG_SIZE_T); + va_end(lva); return retval; } -- cgit v1.2.1