diff options
Diffstat (limited to 'Modules/gcmodule.c')
| -rw-r--r-- | Modules/gcmodule.c | 289 |
1 files changed, 228 insertions, 61 deletions
diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c index c66dd38126..6281a7c343 100644 --- a/Modules/gcmodule.c +++ b/Modules/gcmodule.c @@ -168,6 +168,18 @@ static Py_ssize_t long_lived_pending = 0; static int debug; static PyObject *tmod = NULL; +/* Running stats per generation */ +struct gc_generation_stats { + /* total number of collections */ + Py_ssize_t collections; + /* total number of collected objects */ + Py_ssize_t collected; + /* total number of uncollectable objects (put into gc.garbage) */ + Py_ssize_t uncollectable; +}; + +static struct gc_generation_stats generation_stats[NUM_GENERATIONS]; + /*-------------------------------------------------------------------------- gc_refs values. @@ -211,10 +223,10 @@ GC_TENTATIVELY_UNREACHABLE #define GC_REACHABLE _PyGC_REFS_REACHABLE #define GC_TENTATIVELY_UNREACHABLE _PyGC_REFS_TENTATIVELY_UNREACHABLE -#define IS_TRACKED(o) ((AS_GC(o))->gc.gc_refs != GC_UNTRACKED) -#define IS_REACHABLE(o) ((AS_GC(o))->gc.gc_refs == GC_REACHABLE) +#define IS_TRACKED(o) (_PyGC_REFS(o) != GC_UNTRACKED) +#define IS_REACHABLE(o) (_PyGC_REFS(o) == GC_REACHABLE) #define IS_TENTATIVELY_UNREACHABLE(o) ( \ - (AS_GC(o))->gc.gc_refs == GC_TENTATIVELY_UNREACHABLE) + _PyGC_REFS(o) == GC_TENTATIVELY_UNREACHABLE) /*** list functions ***/ @@ -329,8 +341,8 @@ update_refs(PyGC_Head *containers) { PyGC_Head *gc = containers->gc.gc_next; for (; gc != containers; gc = gc->gc.gc_next) { - assert(gc->gc.gc_refs == GC_REACHABLE); - gc->gc.gc_refs = Py_REFCNT(FROM_GC(gc)); + assert(_PyGCHead_REFS(gc) == GC_REACHABLE); + _PyGCHead_SET_REFS(gc, Py_REFCNT(FROM_GC(gc))); /* Python's cyclic gc should never see an incoming refcount * of 0: if something decref'ed to 0, it should have been * deallocated immediately at that time. @@ -349,7 +361,7 @@ update_refs(PyGC_Head *containers) * so serious that maybe this should be a release-build * check instead of an assert? */ - assert(gc->gc.gc_refs != 0); + assert(_PyGCHead_REFS(gc) != 0); } } @@ -364,9 +376,9 @@ visit_decref(PyObject *op, void *data) * generation being collected, which can be recognized * because only they have positive gc_refs. */ - assert(gc->gc.gc_refs != 0); /* else refcount was too small */ - if (gc->gc.gc_refs > 0) - gc->gc.gc_refs--; + assert(_PyGCHead_REFS(gc) != 0); /* else refcount was too small */ + if (_PyGCHead_REFS(gc) > 0) + _PyGCHead_DECREF(gc); } return 0; } @@ -395,7 +407,7 @@ visit_reachable(PyObject *op, PyGC_Head *reachable) { if (PyObject_IS_GC(op)) { PyGC_Head *gc = AS_GC(op); - const Py_ssize_t gc_refs = gc->gc.gc_refs; + const Py_ssize_t gc_refs = _PyGCHead_REFS(gc); if (gc_refs == 0) { /* This is in move_unreachable's 'young' list, but @@ -403,7 +415,7 @@ visit_reachable(PyObject *op, PyGC_Head *reachable) * we need to do is tell move_unreachable that it's * reachable. */ - gc->gc.gc_refs = 1; + _PyGCHead_SET_REFS(gc, 1); } else if (gc_refs == GC_TENTATIVELY_UNREACHABLE) { /* This had gc_refs = 0 when move_unreachable got @@ -413,7 +425,7 @@ visit_reachable(PyObject *op, PyGC_Head *reachable) * again. */ gc_list_move(gc, reachable); - gc->gc.gc_refs = 1; + _PyGCHead_SET_REFS(gc, 1); } /* Else there's nothing to do. * If gc_refs > 0, it must be in move_unreachable's 'young' @@ -457,7 +469,7 @@ move_unreachable(PyGC_Head *young, PyGC_Head *unreachable) while (gc != young) { PyGC_Head *next; - if (gc->gc.gc_refs) { + if (_PyGCHead_REFS(gc)) { /* gc is definitely reachable from outside the * original 'young'. Mark it as such, and traverse * its pointers to find any other objects that may @@ -468,8 +480,8 @@ move_unreachable(PyGC_Head *young, PyGC_Head *unreachable) */ PyObject *op = FROM_GC(gc); traverseproc traverse = Py_TYPE(op)->tp_traverse; - assert(gc->gc.gc_refs > 0); - gc->gc.gc_refs = GC_REACHABLE; + assert(_PyGCHead_REFS(gc) > 0); + _PyGCHead_SET_REFS(gc, GC_REACHABLE); (void) traverse(op, (visitproc)visit_reachable, (void *)young); @@ -488,7 +500,7 @@ move_unreachable(PyGC_Head *young, PyGC_Head *unreachable) */ next = gc->gc.gc_next; gc_list_move(gc, unreachable); - gc->gc.gc_refs = GC_TENTATIVELY_UNREACHABLE; + _PyGCHead_SET_REFS(gc, GC_TENTATIVELY_UNREACHABLE); } gc = next; } @@ -508,22 +520,19 @@ untrack_dicts(PyGC_Head *head) } } -/* Return true if object has a finalization method. */ +/* Return true if object has a pre-PEP 442 finalization method. */ static int -has_finalizer(PyObject *op) +has_legacy_finalizer(PyObject *op) { - if (PyGen_CheckExact(op)) - return PyGen_NeedsFinalizing((PyGenObject *)op); - else - return op->ob_type->tp_del != NULL; + return op->ob_type->tp_del != NULL; } -/* Move the objects in unreachable with __del__ methods into `finalizers`. +/* Move the objects in unreachable with tp_del slots into `finalizers`. * Objects moved into `finalizers` have gc_refs set to GC_REACHABLE; the * objects remaining in unreachable are left at GC_TENTATIVELY_UNREACHABLE. */ static void -move_finalizers(PyGC_Head *unreachable, PyGC_Head *finalizers) +move_legacy_finalizers(PyGC_Head *unreachable, PyGC_Head *finalizers) { PyGC_Head *gc; PyGC_Head *next; @@ -537,14 +546,14 @@ move_finalizers(PyGC_Head *unreachable, PyGC_Head *finalizers) assert(IS_TENTATIVELY_UNREACHABLE(op)); next = gc->gc.gc_next; - if (has_finalizer(op)) { + if (has_legacy_finalizer(op)) { gc_list_move(gc, finalizers); - gc->gc.gc_refs = GC_REACHABLE; + _PyGCHead_SET_REFS(gc, GC_REACHABLE); } } } -/* A traversal callback for move_finalizer_reachable. */ +/* A traversal callback for move_legacy_finalizer_reachable. */ static int visit_move(PyObject *op, PyGC_Head *tolist) { @@ -552,7 +561,7 @@ visit_move(PyObject *op, PyGC_Head *tolist) if (IS_TENTATIVELY_UNREACHABLE(op)) { PyGC_Head *gc = AS_GC(op); gc_list_move(gc, tolist); - gc->gc.gc_refs = GC_REACHABLE; + _PyGCHead_SET_REFS(gc, GC_REACHABLE); } } return 0; @@ -562,7 +571,7 @@ visit_move(PyObject *op, PyGC_Head *tolist) * into finalizers set. */ static void -move_finalizer_reachable(PyGC_Head *finalizers) +move_legacy_finalizer_reachable(PyGC_Head *finalizers) { traverseproc traverse; PyGC_Head *gc = finalizers->gc.gc_next; @@ -735,7 +744,7 @@ debug_cycle(char *msg, PyObject *op) msg, Py_TYPE(op)->tp_name, op); } -/* Handle uncollectable garbage (cycles with finalizers, and stuff reachable +/* Handle uncollectable garbage (cycles with tp_del slots, and stuff reachable * only from such cycles). * If DEBUG_SAVEALL, all objects in finalizers are appended to the module * garbage list (a Python list), else only the objects in finalizers with @@ -745,7 +754,7 @@ debug_cycle(char *msg, PyObject *op) * The finalizers list is made empty on a successful return. */ static int -handle_finalizers(PyGC_Head *finalizers, PyGC_Head *old) +handle_legacy_finalizers(PyGC_Head *finalizers, PyGC_Head *old) { PyGC_Head *gc = finalizers->gc.gc_next; @@ -757,7 +766,7 @@ handle_finalizers(PyGC_Head *finalizers, PyGC_Head *old) for (; gc != finalizers; gc = gc->gc.gc_next) { PyObject *op = FROM_GC(gc); - if ((debug & DEBUG_SAVEALL) || has_finalizer(op)) { + if ((debug & DEBUG_SAVEALL) || has_legacy_finalizer(op)) { if (PyList_Append(garbage, op) < 0) return -1; } @@ -767,6 +776,62 @@ handle_finalizers(PyGC_Head *finalizers, PyGC_Head *old) return 0; } +static void +finalize_garbage(PyGC_Head *collectable, PyGC_Head *old) +{ + destructor finalize; + PyGC_Head *gc = collectable->gc.gc_next; + + for (; gc != collectable; gc = gc->gc.gc_next) { + PyObject *op = FROM_GC(gc); + + if (!_PyGCHead_FINALIZED(gc) && + PyType_HasFeature(Py_TYPE(op), Py_TPFLAGS_HAVE_FINALIZE) && + (finalize = Py_TYPE(op)->tp_finalize) != NULL) { + _PyGCHead_SET_FINALIZED(gc, 1); + Py_INCREF(op); + finalize(op); + if (Py_REFCNT(op) == 1) { + /* op will be destroyed */ + gc = gc->gc.gc_prev; + } + Py_DECREF(op); + } + } +} + +/* Walk the collectable list and check that they are really unreachable + from the outside (some objects could have been resurrected by a + finalizer). */ +static int +check_garbage(PyGC_Head *collectable) +{ + PyGC_Head *gc; + for (gc = collectable->gc.gc_next; gc != collectable; + gc = gc->gc.gc_next) { + _PyGCHead_SET_REFS(gc, Py_REFCNT(FROM_GC(gc))); + assert(_PyGCHead_REFS(gc) != 0); + } + subtract_refs(collectable); + for (gc = collectable->gc.gc_next; gc != collectable; + gc = gc->gc.gc_next) { + assert(_PyGCHead_REFS(gc) >= 0); + if (_PyGCHead_REFS(gc) != 0) + return -1; + } + return 0; +} + +static void +revive_garbage(PyGC_Head *collectable) +{ + PyGC_Head *gc; + for (gc = collectable->gc.gc_next; gc != collectable; + gc = gc->gc.gc_next) { + _PyGCHead_SET_REFS(gc, GC_REACHABLE); + } +} + /* Break reference cycles by clearing the containers involved. This is * tricky business as the lists can be changing and we don't know which * objects may be freed. It is possible I screwed something up here. @@ -780,7 +845,6 @@ delete_garbage(PyGC_Head *collectable, PyGC_Head *old) PyGC_Head *gc = collectable->gc.gc_next; PyObject *op = FROM_GC(gc); - assert(IS_TENTATIVELY_UNREACHABLE(op)); if (debug & DEBUG_SAVEALL) { PyList_Append(garbage, op); } @@ -794,7 +858,7 @@ delete_garbage(PyGC_Head *collectable, PyGC_Head *old) if (collectable->gc.gc_next == gc) { /* object is still alive, move it, it may die later */ gc_list_move(gc, old); - gc->gc.gc_refs = GC_REACHABLE; + _PyGCHead_SET_REFS(gc, GC_REACHABLE); } } } @@ -841,7 +905,8 @@ get_time(void) /* This is the main function. Read this to understand how the * collection process works. */ static Py_ssize_t -collect(int generation, Py_ssize_t *n_collected, Py_ssize_t *n_uncollectable) +collect(int generation, Py_ssize_t *n_collected, Py_ssize_t *n_uncollectable, + int nofail) { int i; Py_ssize_t m = 0; /* # objects collected */ @@ -852,6 +917,7 @@ collect(int generation, Py_ssize_t *n_collected, Py_ssize_t *n_uncollectable) PyGC_Head finalizers; /* objects with, & reachable from, __del__ */ PyGC_Head *gc; double t1 = 0.0; + struct gc_generation_stats *stats = &generation_stats[generation]; if (debug & DEBUG_STATS) { PySys_WriteStderr("gc: collecting generation %d...\n", @@ -915,19 +981,15 @@ collect(int generation, Py_ssize_t *n_collected, Py_ssize_t *n_uncollectable) } /* All objects in unreachable are trash, but objects reachable from - * finalizers can't safely be deleted. Python programmers should take - * care not to create such things. For Python, finalizers means - * instance objects with __del__ methods. Weakrefs with callbacks - * can also call arbitrary Python code but they will be dealt with by - * handle_weakrefs(). + * legacy finalizers (e.g. tp_del) can't safely be deleted. */ gc_list_init(&finalizers); - move_finalizers(&unreachable, &finalizers); - /* finalizers contains the unreachable objects with a finalizer; + move_legacy_finalizers(&unreachable, &finalizers); + /* finalizers contains the unreachable objects with a legacy finalizer; * unreachable objects reachable *from* those are also uncollectable, * and we move those into the finalizers list too. */ - move_finalizer_reachable(&finalizers); + move_legacy_finalizer_reachable(&finalizers); /* Collect statistics on collectable objects found and print * debugging information. @@ -943,11 +1005,20 @@ collect(int generation, Py_ssize_t *n_collected, Py_ssize_t *n_uncollectable) /* Clear weakrefs and invoke callbacks as necessary. */ m += handle_weakrefs(&unreachable, old); - /* Call tp_clear on objects in the unreachable set. This will cause - * the reference cycles to be broken. It may also cause some objects - * in finalizers to be freed. - */ - delete_garbage(&unreachable, old); + /* Call tp_finalize on objects which have one. */ + finalize_garbage(&unreachable, old); + + if (check_garbage(&unreachable)) { + revive_garbage(&unreachable); + gc_list_merge(&unreachable, old); + } + else { + /* Call tp_clear on objects in the unreachable set. This will cause + * the reference cycles to be broken. It may also cause some objects + * in finalizers to be freed. + */ + delete_garbage(&unreachable, old); + } /* Collect statistics on uncollectable objects found and print * debugging information. */ @@ -978,7 +1049,7 @@ collect(int generation, Py_ssize_t *n_collected, Py_ssize_t *n_uncollectable) * reachable list of garbage. The programmer has to deal with * this if they insist on creating this type of structure. */ - (void)handle_finalizers(&finalizers, old); + (void)handle_legacy_finalizers(&finalizers, old); /* Clear free list only during the collection of the highest * generation */ @@ -987,16 +1058,25 @@ collect(int generation, Py_ssize_t *n_collected, Py_ssize_t *n_uncollectable) } if (PyErr_Occurred()) { - if (gc_str == NULL) - gc_str = PyUnicode_FromString("garbage collection"); - PyErr_WriteUnraisable(gc_str); - Py_FatalError("unexpected exception during garbage collection"); + if (nofail) { + PyErr_Clear(); + } + else { + if (gc_str == NULL) + gc_str = PyUnicode_FromString("garbage collection"); + PyErr_WriteUnraisable(gc_str); + Py_FatalError("unexpected exception during garbage collection"); + } } + /* Update stats */ if (n_collected) *n_collected = m; if (n_uncollectable) *n_uncollectable = n; + stats->collections++; + stats->collected += m; + stats->uncollectable += n; return n+m; } @@ -1045,7 +1125,7 @@ collect_with_callback(int generation) { Py_ssize_t result, collected, uncollectable; invoke_gc_callback("start", generation, 0, 0); - result = collect(generation, &collected, &uncollectable); + result = collect(generation, &collected, &uncollectable, 0); invoke_gc_callback("stop", generation, collected, uncollectable); return result; } @@ -1343,6 +1423,52 @@ gc_get_objects(PyObject *self, PyObject *noargs) return result; } +PyDoc_STRVAR(gc_get_stats__doc__, +"get_stats() -> [...]\n" +"\n" +"Return a list of dictionaries containing per-generation statistics.\n"); + +static PyObject * +gc_get_stats(PyObject *self, PyObject *noargs) +{ + int i; + PyObject *result; + struct gc_generation_stats stats[NUM_GENERATIONS], *st; + + /* To get consistent values despite allocations while constructing + the result list, we use a snapshot of the running stats. */ + for (i = 0; i < NUM_GENERATIONS; i++) { + stats[i] = generation_stats[i]; + } + + result = PyList_New(0); + if (result == NULL) + return NULL; + + for (i = 0; i < NUM_GENERATIONS; i++) { + PyObject *dict; + st = &stats[i]; + dict = Py_BuildValue("{snsnsn}", + "collections", st->collections, + "collected", st->collected, + "uncollectable", st->uncollectable + ); + if (dict == NULL) + goto error; + if (PyList_Append(result, dict)) { + Py_DECREF(dict); + goto error; + } + Py_DECREF(dict); + } + return result; + +error: + Py_XDECREF(result); + return NULL; +} + + PyDoc_STRVAR(gc_is_tracked__doc__, "is_tracked(obj) -> bool\n" "\n" @@ -1372,6 +1498,7 @@ PyDoc_STRVAR(gc__doc__, "isenabled() -- Returns true if automatic collection is enabled.\n" "collect() -- Do a full collection right now.\n" "get_count() -- Return the current collection counts.\n" +"get_stats() -- Return list of dictionaries containing per-generation stats.\n" "set_debug() -- Set debugging flags.\n" "get_debug() -- Get debugging flags.\n" "set_threshold() -- Set the collection thresholds.\n" @@ -1393,6 +1520,7 @@ static PyMethodDef GcMethods[] = { {"collect", (PyCFunction)gc_collect, METH_VARARGS | METH_KEYWORDS, gc_collect__doc__}, {"get_objects", gc_get_objects,METH_NOARGS, gc_get_objects__doc__}, + {"get_stats", gc_get_stats, METH_NOARGS, gc_get_stats__doc__}, {"is_tracked", gc_is_tracked, METH_O, gc_is_tracked__doc__}, {"get_referrers", gc_get_referrers, METH_VARARGS, gc_get_referrers__doc__}, @@ -1480,8 +1608,29 @@ PyGC_Collect(void) return n; } +Py_ssize_t +_PyGC_CollectNoFail(void) +{ + Py_ssize_t n; + + /* Ideally, this function is only called on interpreter shutdown, + and therefore not recursively. Unfortunately, when there are daemon + threads, a daemon thread can start a cyclic garbage collection + during interpreter shutdown (and then never finish it). + See http://bugs.python.org/issue8713#msg195178 for an example. + */ + if (collecting) + n = 0; + else { + collecting = 1; + n = collect(NUM_GENERATIONS - 1, NULL, NULL, 1); + collecting = 0; + } + return n; +} + void -_PyGC_Fini(void) +_PyGC_DumpShutdownStats(void) { if (!(debug & DEBUG_SAVEALL) && garbage != NULL && PyList_GET_SIZE(garbage) > 0) { @@ -1492,8 +1641,12 @@ _PyGC_Fini(void) else message = "gc: %zd uncollectable objects at " \ "shutdown; use gc.set_debug(gc.DEBUG_UNCOLLECTABLE) to list them"; - if (PyErr_WarnFormat(PyExc_ResourceWarning, 0, message, - PyList_GET_SIZE(garbage)) < 0) + /* PyErr_WarnFormat does too many things and we are at shutdown, + the warnings module's dependencies (e.g. linecache) may be gone + already. */ + if (PyErr_WarnExplicitFormat(PyExc_ResourceWarning, "gc", 0, + "gc", NULL, message, + PyList_GET_SIZE(garbage))) PyErr_WriteUnraisable(NULL); if (debug & DEBUG_UNCOLLECTABLE) { PyObject *repr = NULL, *bytes = NULL; @@ -1502,7 +1655,7 @@ _PyGC_Fini(void) PyErr_WriteUnraisable(garbage); else { PySys_WriteStderr( - " %s\n", + " %s\n", PyBytes_AS_STRING(bytes) ); } @@ -1510,7 +1663,13 @@ _PyGC_Fini(void) Py_XDECREF(bytes); } } +} + +void +_PyGC_Fini(void) +{ Py_CLEAR(callbacks); + Py_CLEAR(tmod); } /* for debugging */ @@ -1555,7 +1714,8 @@ _PyObject_GC_Malloc(size_t basicsize) sizeof(PyGC_Head) + basicsize); if (g == NULL) return PyErr_NoMemory(); - g->gc.gc_refs = GC_UNTRACKED; + g->gc.gc_refs = 0; + _PyGCHead_SET_REFS(g, GC_UNTRACKED); generations[0].count++; /* number of allocated GC objects */ if (generations[0].count > generations[0].threshold && enabled && @@ -1582,8 +1742,15 @@ _PyObject_GC_New(PyTypeObject *tp) PyVarObject * _PyObject_GC_NewVar(PyTypeObject *tp, Py_ssize_t nitems) { - const size_t size = _PyObject_VAR_SIZE(tp, nitems); - PyVarObject *op = (PyVarObject *) _PyObject_GC_Malloc(size); + size_t size; + PyVarObject *op; + + if (nitems < 0) { + PyErr_BadInternalCall(); + return NULL; + } + size = _PyObject_VAR_SIZE(tp, nitems); + op = (PyVarObject *) _PyObject_GC_Malloc(size); if (op != NULL) op = PyObject_INIT_VAR(op, tp, nitems); return op; |