Mark generators left on stack when clearing frame as zombies.

Author: markshannon

Include/internal/pycore_frame.h

@@ -46,11 +46,12 @@ typedef enum _framestate {
     FRAME_SUSPENDED_YIELD_FROM = -1,
     FRAME_EXECUTING = 0,
     FRAME_COMPLETED = 1,
-    FRAME_CLEARED = 4
+    FRAME_CLEARED = 4,
+    FRAME_ZOMBIE = 5, /* For generators left on the stack of cleared threads */
 } PyFrameState;
 
 #define FRAME_STATE_SUSPENDED(S) ((S) == FRAME_SUSPENDED || (S) == FRAME_SUSPENDED_YIELD_FROM)
-#define FRAME_STATE_FINISHED(S) ((S) >= FRAME_COMPLETED)
+#define FRAME_STATE_FINISHED(S) ((S) > FRAME_EXECUTING)
 
 enum _frameowner {
     FRAME_OWNED_BY_THREAD = 0,

Objects/genobject.c

@@ -58,7 +58,7 @@ gen_traverse(PyObject *self, visitproc visit, void *arg)
     PyGenObject *gen = _PyGen_CAST(self);
     Py_VISIT(gen->gi_name);
     Py_VISIT(gen->gi_qualname);
-    if (gen->gi_frame_state < FRAME_EXECUTING) {
+    if (gen->gi_frame_state < FRAME_EXECUTING || gen->gi_frame_state == FRAME_ZOMBIE) {
         _PyInterpreterFrame *frame = &gen->gi_iframe;
         assert(frame->frame_obj == NULL ||
                frame->frame_obj->f_frame->owner == FRAME_OWNED_BY_GENERATOR);

Python/pystate.c

@@ -1636,18 +1636,28 @@ PyThreadState_Clear(PyThreadState *tstate)
 
     int verbose = _PyInterpreterState_GetConfig(tstate->interp)->verbose;
 
-    if (verbose && tstate->current_frame != NULL) {
-        /* bpo-20526: After the main thread calls
-           _PyInterpreterState_SetFinalizing() in Py_FinalizeEx()
-           (or in Py_EndInterpreter() for subinterpreters),
-           threads must exit when trying to take the GIL.
-           If a thread exit in the middle of _PyEval_EvalFrameDefault(),
-           tstate->frame is not reset to its previous value.
-           It is more likely with daemon threads, but it can happen
-           with regular threads if threading._shutdown() fails
-           (ex: interrupted by CTRL+C). */
-        fprintf(stderr,
-          "PyThreadState_Clear: warning: thread still has a frame\n");
+    if (tstate->current_frame != NULL) {
+        _PyInterpreterFrame *frame = tstate->current_frame;
+        if (verbose) {
+            /* bpo-20526: After the main thread calls
+            _PyInterpreterState_SetFinalizing() in Py_FinalizeEx()
+            (or in Py_EndInterpreter() for subinterpreters),
+            threads must exit when trying to take the GIL.
+            If a thread exit in the middle of _PyEval_EvalFrameDefault(),
+            tstate->frame is not reset to its previous value.
+            It is more likely with daemon threads, but it can happen
+            with regular threads if threading._shutdown() fails
+            (ex: interrupted by CTRL+C). */
+            fprintf(stderr,
+            "PyThreadState_Clear: warning: thread still has a frame\n");
+        }
+        do {
+            if (frame->owner == FRAME_OWNED_BY_GENERATOR) {
+                PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame);
+                gen->gi_frame_state = FRAME_ZOMBIE;
+            }
+            frame = frame->previous;
+        } while (frame != NULL);
     }
 
     /* At this point tstate shouldn't be used any more,