Reduce overhead of maintaining 'warm' byte in executors.

Author: markshannon

Include/internal/pycore_ceval.h

@@ -332,7 +332,6 @@ _PyEval_SpecialMethodCanSuggest(PyObject *self, int oparg);
 #define _PY_GC_SCHEDULED_BIT (1U << 4)
 #define _PY_EVAL_PLEASE_STOP_BIT (1U << 5)
 #define _PY_EVAL_EXPLICIT_MERGE_BIT (1U << 6)
-#define _PY_EVAL_JIT_INVALIDATE_COLD_BIT (1U << 7)
 
 /* Reserve a few bits for future use */
 #define _PY_EVAL_EVENTS_BITS 8

Include/internal/pycore_interp_structs.h

@@ -953,7 +953,7 @@ struct _is {
     struct _PyExecutorObject *executor_deletion_list_head;
     struct _PyExecutorObject *cold_executor;
     int executor_deletion_list_remaining_capacity;
-    size_t trace_run_counter;
+    int jited_ops_until_cleanup;
     _rare_events rare_events;
     PyDict_WatchCallback builtins_dict_watcher;
 

Include/internal/pycore_optimizer.h

@@ -115,8 +115,8 @@ PyAPI_FUNC(void) _Py_Executors_InvalidateCold(PyInterpreterState *interp);
 #endif
 
 // Used as the threshold to trigger executor invalidation when
-// trace_run_counter is greater than this value.
-#define JIT_CLEANUP_THRESHOLD 100000
+// the number of uops jitted reaches this threshold
+#define JIT_CLEANUP_THRESHOLD 10000
 
 // This is the length of the trace we project initially.
 #define UOP_MAX_TRACE_LENGTH 800

Misc/NEWS.d/next/Core_and_Builtins/2025-08-22-10-51-50.gh-issue-138050.ICaGJJ.rst

@@ -0,0 +1,2 @@
+Move cleanup of cold pieces of jitted code from the eval breaker to the jit
+compiler. This reduces the overhead of maintaining the "warm" bit.

Python/bytecodes.c

@@ -5367,10 +5367,6 @@ dummy_func(
 
         tier2 op(_MAKE_WARM, (--)) {
             current_executor->vm_data.warm = true;
-            // It's okay if this ends up going negative.
-            if (--tstate->interp->trace_run_counter == 0) {
-                _Py_set_eval_breaker_bit(tstate, _PY_EVAL_JIT_INVALIDATE_COLD_BIT);
-            }
         }
 
         tier2 op(_FATAL_ERROR, (--)) {

Python/ceval_gil.c

@@ -1395,12 +1395,6 @@ _Py_HandlePending(PyThreadState *tstate)
         _Py_RunGC(tstate);
     }
 
-    if ((breaker & _PY_EVAL_JIT_INVALIDATE_COLD_BIT) != 0) {
-        _Py_unset_eval_breaker_bit(tstate, _PY_EVAL_JIT_INVALIDATE_COLD_BIT);
-        _Py_Executors_InvalidateCold(tstate->interp);
-        tstate->interp->trace_run_counter = JIT_CLEANUP_THRESHOLD;
-    }
-
     /* GIL drop request */
     if ((breaker & _PY_GIL_DROP_REQUEST_BIT) != 0) {
         /* Give another thread a chance */

Python/executor_cases.c.h

@@ -152,6 +152,13 @@ _PyOptimizer_Optimize(
     }
     (*executor_ptr)->vm_data.chain_depth = chain_depth;
     assert((*executor_ptr)->vm_data.valid);
+    assert((*executor_ptr)->vm_data.warm);
+    PyInterpreterState *interp = PyInterpreterState_Get();
+    interp->jited_ops_until_cleanup -= (*executor_ptr)->code_size;
+    if (interp->jited_ops_until_cleanup <= 0) {
+        _Py_Executors_InvalidateCold(interp);
+        interp->jited_ops_until_cleanup = JIT_CLEANUP_THRESHOLD;
+    }
     return 1;
 }
 

Python/optimizer.c

@@ -574,7 +574,7 @@ init_interpreter(PyInterpreterState *interp,
     interp->executor_list_head = NULL;
     interp->executor_deletion_list_head = NULL;
     interp->executor_deletion_list_remaining_capacity = 0;
-    interp->trace_run_counter = JIT_CLEANUP_THRESHOLD;
+    interp->jited_ops_until_cleanup = JIT_CLEANUP_THRESHOLD;
     if (interp != &runtime->_main_interpreter) {
         /* Fix the self-referential, statically initialized fields. */
         interp->dtoa = (struct _dtoa_state)_dtoa_state_INIT(interp);