Sprinkle lots of branch prediction hints in places where exceptions are being handled or otherwise exceptional cases occur.

5 files changed, 40 insertions, 39 deletions

diff --git a/Cython/Utility/AsyncGen.c b/Cython/Utility/AsyncGen.c
index 11b12e65d..f045f04ff 100644
--- a/Cython/Utility/AsyncGen.c
+++ b/Cython/Utility/AsyncGen.c
@@ -929,7 +929,7 @@ __Pyx_async_gen_athrow_send(__pyx_PyAsyncGenAThrow *o, PyObject *arg)
     } else {
         /* aclose() mode */
         if (retval) {
-            if (__pyx__PyAsyncGenWrappedValue_CheckExact(retval)) {
+            if (unlikely(__pyx__PyAsyncGenWrappedValue_CheckExact(retval))) {
                 Py_DECREF(retval);
                 goto yield_close;
             }
@@ -983,7 +983,7 @@ __Pyx_async_gen_athrow_throw(__pyx_PyAsyncGenAThrow *o, PyObject *args)
     } else {
         // aclose() mode
         PyObject *exc_type;
-        if (retval && __pyx__PyAsyncGenWrappedValue_CheckExact(retval)) {
+        if (unlikely(retval && __pyx__PyAsyncGenWrappedValue_CheckExact(retval))) {
             o->agt_gen->ag_running_async = 0;
             o->agt_state = __PYX_AWAITABLE_STATE_CLOSED;
             Py_DECREF(retval);
diff --git a/Cython/Utility/Builtins.c b/Cython/Utility/Builtins.c
index df5b0d86f..a8e8f0a08 100644
--- a/Cython/Utility/Builtins.c
+++ b/Cython/Utility/Builtins.c
@@ -68,7 +68,7 @@ static PyObject* __Pyx_PyExec3(PyObject* o, PyObject* globals, PyObject* locals)
 
     if (!globals || globals == Py_None) {
         globals = $moddict_cname;
-    } else if (!PyDict_Check(globals)) {
+    } else if (unlikely(!PyDict_Check(globals))) {
         PyErr_Format(PyExc_TypeError, "exec() arg 2 must be a dict, not %.200s",
                      Py_TYPE(globals)->tp_name);
         goto bad;
@@ -78,12 +78,12 @@ static PyObject* __Pyx_PyExec3(PyObject* o, PyObject* globals, PyObject* locals)
     }
 
     if (__Pyx_PyDict_GetItemStr(globals, PYIDENT("__builtins__")) == NULL) {
-        if (PyDict_SetItem(globals, PYIDENT("__builtins__"), PyEval_GetBuiltins()) < 0)
+        if (unlikely(PyDict_SetItem(globals, PYIDENT("__builtins__"), PyEval_GetBuiltins()) < 0))
             goto bad;
     }
 
     if (PyCode_Check(o)) {
-        if (__Pyx_PyCode_HasFreeVars((PyCodeObject *)o)) {
+        if (unlikely(__Pyx_PyCode_HasFreeVars((PyCodeObject *)o))) {
             PyErr_SetString(PyExc_TypeError,
                 "code object passed to exec() may not contain free variables");
             goto bad;
@@ -99,12 +99,12 @@ static PyObject* __Pyx_PyExec3(PyObject* o, PyObject* globals, PyObject* locals)
         if (PyUnicode_Check(o)) {
             cf.cf_flags = PyCF_SOURCE_IS_UTF8;
             s = PyUnicode_AsUTF8String(o);
-            if (!s) goto bad;
+            if (unlikely(!s)) goto bad;
             o = s;
         #if PY_MAJOR_VERSION >= 3
-        } else if (!PyBytes_Check(o)) {
+        } else if (unlikely(!PyBytes_Check(o))) {
         #else
-        } else if (!PyString_Check(o)) {
+        } else if (unlikely(!PyString_Check(o))) {
         #endif
             PyErr_Format(PyExc_TypeError,
                 "exec: arg 1 must be string, bytes or code object, got %.200s",
@@ -180,7 +180,7 @@ static CYTHON_INLINE int __Pyx_HasAttr(PyObject *o, PyObject *n) {
         return -1;
     }
     r = __Pyx_GetAttr(o, n);
-    if (unlikely(!r)) {
+    if (!r) {
         PyErr_Clear();
         return 0;
     } else {
@@ -196,7 +196,7 @@ static PyObject* __Pyx_Intern(PyObject* s); /* proto */
 //////////////////// Intern ////////////////////
 
 static PyObject* __Pyx_Intern(PyObject* s) {
-    if (!(likely(PyString_CheckExact(s)))) {
+    if (unlikely(!PyString_CheckExact(s))) {
         PyErr_Format(PyExc_TypeError, "Expected %.16s, got %.200s", "str", Py_TYPE(s)->tp_name);
         return 0;
     }
diff --git a/Cython/Utility/Coroutine.c b/Cython/Utility/Coroutine.c
index add9455f6..f3283497d 100644
--- a/Cython/Utility/Coroutine.c
+++ b/Cython/Utility/Coroutine.c
@@ -522,7 +522,7 @@ static int __Pyx_PyGen__FetchStopIterationValue(CYTHON_UNUSED PyThreadState *$lo
             value = Py_None;
         }
 #if PY_VERSION_HEX >= 0x030300A0
-        else if (__Pyx_IS_TYPE(ev, (PyTypeObject*)PyExc_StopIteration)) {
+        else if (likely(__Pyx_IS_TYPE(ev, (PyTypeObject*)PyExc_StopIteration))) {
             value = ((PyStopIterationObject *)ev)->value;
             Py_INCREF(value);
             Py_DECREF(ev);
@@ -905,7 +905,7 @@ static int __Pyx_Coroutine_CloseIter(__pyx_CoroutineObject *gen, PyObject *yf) {
         } else {
             retval = PyObject_CallFunction(meth, NULL);
             Py_DECREF(meth);
-            if (!retval)
+            if (unlikely(!retval))
                 err = -1;
         }
         gen->is_running = 0;
@@ -1079,7 +1079,7 @@ static PyObject *__Pyx_Coroutine_Throw(PyObject *self, PyObject *args) {
     PyObject *val = NULL;
     PyObject *tb = NULL;
 
-    if (!PyArg_UnpackTuple(args, (char *)"throw", 1, 3, &typ, &val, &tb))
+    if (unlikely(!PyArg_UnpackTuple(args, (char *)"throw", 1, 3, &typ, &val, &tb)))
         return NULL;
 
     return __Pyx__Coroutine_Throw(self, typ, val, tb, args, 1);
@@ -1130,10 +1130,10 @@ static void __Pyx_Coroutine_dealloc(PyObject *self) {
         // Generator is paused or unstarted, so we need to close
         PyObject_GC_Track(self);
 #if PY_VERSION_HEX >= 0x030400a1 && CYTHON_USE_TP_FINALIZE
-        if (PyObject_CallFinalizerFromDealloc(self))
+        if (unlikely(PyObject_CallFinalizerFromDealloc(self)))
 #else
         Py_TYPE(gen)->tp_del(self);
-        if (self->ob_refcnt > 0)
+        if (unlikely(self->ob_refcnt > 0))
 #endif
         {
             // resurrected.  :(
@@ -1249,7 +1249,7 @@ static void __Pyx_Coroutine_del(PyObject *self) {
     // Undo the temporary resurrection; can't use DECREF here, it would
     // cause a recursive call.
     assert(self->ob_refcnt > 0);
-    if (--self->ob_refcnt == 0) {
+    if (likely(--self->ob_refcnt == 0)) {
         // this is the normal path out
         return;
     }
@@ -2061,7 +2061,7 @@ static int __Pyx_patch_abc(void) {
     if (CYTHON_REGISTER_ABCS && !abc_patched) {
         PyObject *module;
         module = PyImport_ImportModule((PY_MAJOR_VERSION >= 3) ? "collections.abc" : "collections");
-        if (!module) {
+        if (unlikely(!module)) {
             PyErr_WriteUnraisable(NULL);
             if (unlikely(PyErr_WarnEx(PyExc_RuntimeWarning,
                     ((PY_MAJOR_VERSION >= 3) ?
@@ -2319,7 +2319,7 @@ static int __pyx_StopAsyncIteration_init(void) {
     __Pyx_PyExc_StopAsyncIteration = (PyObject*) __Pyx_FetchCommonType(&__Pyx__PyExc_StopAsyncIteration_type);
     if (unlikely(!__Pyx_PyExc_StopAsyncIteration))
         return -1;
-    if (builtins && unlikely(PyMapping_SetItemString(builtins, (char*) "StopAsyncIteration", __Pyx_PyExc_StopAsyncIteration) < 0))
+    if (likely(builtins) && unlikely(PyMapping_SetItemString(builtins, (char*) "StopAsyncIteration", __Pyx_PyExc_StopAsyncIteration) < 0))
         return -1;
 #endif
     return 0;
diff --git a/Cython/Utility/CythonFunction.c b/Cython/Utility/CythonFunction.c
index 0a6bfd9cf..76cd682f6 100644
--- a/Cython/Utility/CythonFunction.c
+++ b/Cython/Utility/CythonFunction.c
@@ -279,7 +279,7 @@ __Pyx_CyFunction_set_defaults(__pyx_CyFunctionObject *op, PyObject* value, CYTHO
     if (!value) {
         // del => explicit None to prevent rebuilding
         value = Py_None;
-    } else if (value != Py_None && !PyTuple_Check(value)) {
+    } else if (unlikely(value != Py_None && !PyTuple_Check(value))) {
         PyErr_SetString(PyExc_TypeError,
                         "__defaults__ must be set to a tuple object");
         return -1;
@@ -294,7 +294,7 @@ __Pyx_CyFunction_get_defaults(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *co
     PyObject* result = op->defaults_tuple;
     if (unlikely(!result)) {
         if (op->defaults_getter) {
-            if (__Pyx_CyFunction_init_defaults(op) < 0) return NULL;
+            if (unlikely(__Pyx_CyFunction_init_defaults(op) < 0)) return NULL;
             result = op->defaults_tuple;
         } else {
             result = Py_None;
@@ -309,7 +309,7 @@ __Pyx_CyFunction_set_kwdefaults(__pyx_CyFunctionObject *op, PyObject* value, CYT
     if (!value) {
         // del => explicit None to prevent rebuilding
         value = Py_None;
-    } else if (value != Py_None && !PyDict_Check(value)) {
+    } else if (unlikely(value != Py_None && !PyDict_Check(value))) {
         PyErr_SetString(PyExc_TypeError,
                         "__kwdefaults__ must be set to a dict object");
         return -1;
@@ -324,7 +324,7 @@ __Pyx_CyFunction_get_kwdefaults(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *
     PyObject* result = op->defaults_kwdict;
     if (unlikely(!result)) {
         if (op->defaults_getter) {
-            if (__Pyx_CyFunction_init_defaults(op) < 0) return NULL;
+            if (unlikely(__Pyx_CyFunction_init_defaults(op) < 0)) return NULL;
             result = op->defaults_kwdict;
         } else {
             result = Py_None;
@@ -338,7 +338,7 @@ static int
 __Pyx_CyFunction_set_annotations(__pyx_CyFunctionObject *op, PyObject* value, CYTHON_UNUSED void *context) {
     if (!value || value == Py_None) {
         value = NULL;
-    } else if (!PyDict_Check(value)) {
+    } else if (unlikely(!PyDict_Check(value))) {
         PyErr_SetString(PyExc_TypeError,
                         "__annotations__ must be set to a dict object");
         return -1;
@@ -696,7 +696,7 @@ static CYTHON_INLINE int __Pyx_CyFunction_Vectorcall_CheckArgs(__pyx_CyFunctionO
         }
         ret = 1;
     }
-    if (unlikely(kwnames) && PyTuple_GET_SIZE(kwnames)) {
+    if (unlikely(kwnames) && unlikely(PyTuple_GET_SIZE(kwnames))) {
         PyErr_Format(PyExc_TypeError,
                      "%.200s() takes no keyword arguments", cyfunc->func.m_ml->ml_name);
         return -1;
@@ -1090,7 +1090,7 @@ __pyx_FusedFunction_descr_get(PyObject *self, PyObject *obj, PyObject *type)
                     ((PyCFunctionObject *) func)->m_module,
                     ((__pyx_CyFunctionObject *) func)->func_globals,
                     ((__pyx_CyFunctionObject *) func)->func_code);
-    if (!meth)
+    if (unlikely(!meth))
         return NULL;
 
     // defaults needs copying fully rather than just copying the pointer
@@ -1100,9 +1100,10 @@ __pyx_FusedFunction_descr_get(PyObject *self, PyObject *obj, PyObject *type)
         PyObject **pydefaults;
         int i;
 
-        if (!__Pyx_CyFunction_InitDefaults((PyObject*)meth,
-                                      func->func.defaults_size,
-                                      func->func.defaults_pyobjects)) {
+        if (unlikely(!__Pyx_CyFunction_InitDefaults(
+                (PyObject*)meth,
+                func->func.defaults_size,
+                func->func.defaults_pyobjects))) {
             Py_XDECREF((PyObject*)meth);
             return NULL;
         }
@@ -1147,7 +1148,7 @@ __pyx_FusedFunction_getitem(__pyx_FusedFunctionObject *self, PyObject *idx)
     PyObject *unbound_result_func;
     PyObject *result_func = NULL;
 
-    if (self->__signatures__ == NULL) {
+    if (unlikely(self->__signatures__ == NULL)) {
         PyErr_SetString(PyExc_TypeError, "Function is not fused");
         return NULL;
     }
@@ -1159,7 +1160,7 @@ __pyx_FusedFunction_getitem(__pyx_FusedFunctionObject *self, PyObject *idx)
         PyObject *sep = NULL;
         int i;
 
-        if (!list)
+        if (unlikely(!list))
             return NULL;
 
         for (i = 0; i < n; i++) {
@@ -1172,14 +1173,14 @@ __pyx_FusedFunction_getitem(__pyx_FusedFunctionObject *self, PyObject *idx)
 #if !(CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS)
             Py_DECREF(item);
 #endif
-            if (!string || PyList_Append(list, string) < 0)
+            if (unlikely(!string) || unlikely(PyList_Append(list, string) < 0))
                 goto __pyx_err;
 
             Py_DECREF(string);
         }
 
         sep = PyUnicode_FromString("|");
-        if (sep)
+        if (likely(sep))
             signature = PyUnicode_Join(sep, list);
 __pyx_err:
 ;
@@ -1189,12 +1190,12 @@ __pyx_err:
         signature = _obj_to_str(idx);
     }
 
-    if (!signature)
+    if (unlikely(!signature))
         return NULL;
 
     unbound_result_func = PyObject_GetItem(self->__signatures__, signature);
 
-    if (unbound_result_func) {
+    if (likely(unbound_result_func)) {
         if (self->self) {
             __pyx_FusedFunctionObject *unbound = (__pyx_FusedFunctionObject *) unbound_result_func;
 
@@ -1251,7 +1252,7 @@ __pyx_FusedFunction_call(PyObject *func, PyObject *args, PyObject *kw)
         PyObject *self;
         Py_ssize_t i;
         new_args = PyTuple_New(argc + 1);
-        if (!new_args)
+        if (unlikely(!new_args))
             return NULL;
 
         self = binding_func->self;
@@ -1426,7 +1427,7 @@ static int __pyx_FusedFunction_init(void) {
     __pyx_FusedFunctionType_type.tp_base = __pyx_CyFunctionType;
     __pyx_FusedFunctionType = __Pyx_FetchCommonType(&__pyx_FusedFunctionType_type);
 #endif
-    if (__pyx_FusedFunctionType == NULL) {
+    if (unlikely(__pyx_FusedFunctionType == NULL)) {
         return -1;
     }
     return 0;
@@ -1452,9 +1453,9 @@ static PyObject* __Pyx_Method_ClassMethod(PyObject *method) {
 #else
     // It appears that PyMethodDescr_Type is not exposed anywhere in the CPython C-API
     static PyTypeObject *methoddescr_type = NULL;
-    if (methoddescr_type == NULL) {
+    if (unlikely(methoddescr_type == NULL)) {
        PyObject *meth = PyObject_GetAttrString((PyObject*)&PyList_Type, "append");
-       if (!meth) return NULL;
+       if (unlikely(!meth)) return NULL;
        methoddescr_type = Py_TYPE(meth);
        Py_DECREF(meth);
     }
diff --git a/Cython/Utility/FunctionArguments.c b/Cython/Utility/FunctionArguments.c
index 112ab89b6..77a1756e9 100644
--- a/Cython/Utility/FunctionArguments.c
+++ b/Cython/Utility/FunctionArguments.c
@@ -355,7 +355,7 @@ static int __Pyx_MergeKeywords(PyObject *kwdict, PyObject *source_mapping) {
     if (unlikely(!iter)) {
         // slow fallback: try converting to dict, then iterate
         PyObject *args;
-        if (!PyErr_ExceptionMatches(PyExc_AttributeError)) goto bad;
+        if (unlikely(!PyErr_ExceptionMatches(PyExc_AttributeError))) goto bad;
         PyErr_Clear();
         args = PyTuple_Pack(1, source_mapping);
         if (likely(args)) {