TOS caching for the interpreter. Work in progress.

Author: markshannon

Include/internal/pycore_interpframe.h

@@ -43,7 +43,7 @@ static inline PyFunctionObject *_PyFrame_GetFunction(_PyInterpreterFrame *f) {
 }
 
 static inline _PyStackRef *_PyFrame_Stackbase(_PyInterpreterFrame *f) {
-    return (f->localsplus + _PyFrame_GetCode(f)->co_nlocalsplus);
+    return (f->localsplus + _PyFrame_GetCode(f)->co_nlocalsplus + 1);
 }
 
 static inline _PyStackRef _PyFrame_StackPeek(_PyInterpreterFrame *f) {
@@ -134,7 +134,7 @@ _PyFrame_Initialize(
     frame->f_builtins = func_obj->func_builtins;
     frame->f_globals = func_obj->func_globals;
     frame->f_locals = locals;
-    frame->stackpointer = frame->localsplus + code->co_nlocalsplus;
+    frame->stackpointer = frame->localsplus + code->co_nlocalsplus + 1;
     frame->frame_obj = NULL;
 #ifdef Py_GIL_DISABLED
     _PyFrame_InitializeTLBC(tstate, frame, code);
@@ -146,6 +146,7 @@ _PyFrame_Initialize(
     frame->owner = FRAME_OWNED_BY_THREAD;
     frame->visited = 0;
 #ifdef Py_DEBUG
+    frame->localsplus[code->co_nlocalsplus] = PyStackRef_NULL;
     frame->lltrace = 0;
 #endif
 
@@ -317,7 +318,7 @@ _PyFrame_PushTrampolineUnchecked(PyThreadState *tstate, PyCodeObject *code, int
 #endif
     frame->f_locals = NULL;
     assert(stackdepth <= code->co_stacksize);
-    frame->stackpointer = frame->localsplus + code->co_nlocalsplus + stackdepth;
+    frame->stackpointer = frame->localsplus + code->co_nlocalsplus + stackdepth + 1;
     frame->frame_obj = NULL;
 #ifdef Py_GIL_DISABLED
     _PyFrame_InitializeTLBC(tstate, frame, code);

Objects/codeobject.c

@@ -546,7 +546,7 @@ init_code(PyCodeObject *co, struct _PyCodeConstructor *con)
     /* derived values */
     co->co_nlocalsplus = nlocalsplus;
     co->co_nlocals = nlocals;
-    co->co_framesize = nlocalsplus + con->stacksize + FRAME_SPECIALS_SIZE;
+    co->co_framesize = nlocalsplus + con->stacksize + FRAME_SPECIALS_SIZE + 1;
     co->co_ncellvars = ncellvars;
     co->co_nfreevars = nfreevars;
 #ifdef Py_GIL_DISABLED

Python/bytecodes.c

@@ -204,7 +204,7 @@ dummy_func(
                 ptrdiff_t off = this_instr - _PyFrame_GetBytecode(frame);
                 frame->tlbc_index = ((_PyThreadStateImpl *)tstate)->tlbc_index;
                 frame->instr_ptr = bytecode + off;
-                // Make sure this_instr gets reset correctley for any uops that
+                // Make sure this_instr gets reset correctly for any uops that
                 // follow
                 next_instr = frame->instr_ptr;
                 DISPATCH();
@@ -874,18 +874,19 @@ dummy_func(
             // Deopt unless 0 <= sub < PyList_Size(list)
             DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub));
             Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0];
-#ifdef Py_GIL_DISABLED
-            PyObject *res_o = _PyList_GetItemRef((PyListObject*)list, index);
-            DEOPT_IF(res_o == NULL);
-            STAT_INC(BINARY_OP, hit);
-            res = PyStackRef_FromPyObjectSteal(res_o);
-#else
-            DEOPT_IF(index >= PyList_GET_SIZE(list));
-            STAT_INC(BINARY_OP, hit);
-            PyObject *res_o = PyList_GET_ITEM(list, index);
-            assert(res_o != NULL);
-            res = PyStackRef_FromPyObjectNew(res_o);
-#endif
+            if (Py_FREE_THREADING) {
+                PyObject *res_o = _PyList_GetItemRef((PyListObject*)list, index);
+                DEOPT_IF(res_o == NULL);
+                STAT_INC(BINARY_OP, hit);
+                res = PyStackRef_FromPyObjectSteal(res_o);
+            }
+            else {
+                DEOPT_IF(index >= PyList_GET_SIZE(list));
+                STAT_INC(BINARY_OP, hit);
+                PyObject *res_o = PyList_GET_ITEM(list, index);
+                assert(res_o != NULL);
+                res = PyStackRef_FromPyObjectNew(res_o);
+            }
             STAT_INC(BINARY_OP, hit);
             DECREF_INPUTS();
         }
@@ -1111,7 +1112,6 @@ dummy_func(
             tstate->current_frame = frame->previous;
             assert(!_PyErr_Occurred(tstate));
             PyObject *result = PyStackRef_AsPyObjectSteal(retval);
-            SYNC_SP(); /* Not strictly necessary, but prevents warnings */
             return result;
         }
 
@@ -3168,26 +3168,31 @@ dummy_func(
             assert(Py_TYPE(iter_o) == &PyListIter_Type);
             PyListObject *seq = it->it_seq;
             assert(seq);
+            // The code generator doesn't understand #ifdef Py_GIL_DISABLED
+            // so put in some control flow
+            if (Py_FREE_THREADING) {
 #ifdef Py_GIL_DISABLED
-            assert(_PyObject_IsUniquelyReferenced(iter_o));
-            assert(_Py_IsOwnedByCurrentThread((PyObject *)seq) ||
-                   _PyObject_GC_IS_SHARED(seq));
-            STAT_INC(FOR_ITER, hit);
-            int result = _PyList_GetItemRefNoLock(seq, it->it_index, &next);
-            // A negative result means we lost a race with another thread
-            // and we need to take the slow path.
-            DEOPT_IF(result < 0);
-            if (result == 0) {
-                it->it_index = -1;
-                /* Jump forward oparg, then skip following END_FOR instruction */
-                JUMPBY(oparg + 1);
-                DISPATCH();
-            }
-            it->it_index++;
-#else
-            assert(it->it_index < PyList_GET_SIZE(seq));
-            next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(seq, it->it_index++));
+                assert(_PyObject_IsUniquelyReferenced(iter_o));
+                assert(_Py_IsOwnedByCurrentThread((PyObject *)seq) ||
+                    _PyObject_GC_IS_SHARED(seq));
+                STAT_INC(FOR_ITER, hit);
+                int result = _PyList_GetItemRefNoLock(seq, it->it_index, &next);
+                // A negative result means we lost a race with another thread
+                // and we need to take the slow path.
+                DEOPT_IF(result < 0);
+                if (result == 0) {
+                    it->it_index = -1;
+                    /* Jump forward oparg, then skip following END_FOR instruction */
+                    JUMPBY(oparg + 1);
+                    DISPATCH();
+                }
+                it->it_index++;
 #endif
+            }
+            else {
+                assert(it->it_index < PyList_GET_SIZE(seq));
+                next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(seq, it->it_index++));
+            }
         }
 
         // Only used by Tier 2
@@ -3994,8 +3999,9 @@ dummy_func(
                 tstate, (PyCodeObject *)&_Py_InitCleanup, 1, frame);
             assert(_PyFrame_GetBytecode(shim)[0].op.code == EXIT_INIT_CHECK);
             assert(_PyFrame_GetBytecode(shim)[1].op.code == RETURN_VALUE);
+            shim->localsplus[0] = PyStackRef_NULL;
             /* Push self onto stack of shim */
-            shim->localsplus[0] = PyStackRef_DUP(self[0]);
+            shim->localsplus[1] = PyStackRef_DUP(self[0]);
             _PyInterpreterFrame *temp = _PyEvalFramePushAndInit(
                 tstate, init[0], NULL, args-1, oparg+1, NULL, shim);
             DEAD(init);
@@ -4022,7 +4028,6 @@ dummy_func(
             _PUSH_FRAME;
 
         inst(EXIT_INIT_CHECK, (should_be_none -- )) {
-            assert(STACK_LEVEL() == 2);
             if (!PyStackRef_IsNone(should_be_none)) {
                 PyErr_Format(PyExc_TypeError,
                     "__init__() should return None, not '%.200s'",

Python/ceval.c

@@ -151,18 +151,6 @@ dump_item(_PyStackRef item)
         printf("<nil>");
         return;
     }
-    if (
-        obj == Py_None
-        || PyBool_Check(obj)
-        || PyLong_CheckExact(obj)
-        || PyFloat_CheckExact(obj)
-        || PyUnicode_CheckExact(obj)
-    ) {
-        if (PyObject_Print(obj, stdout, 0) == 0) {
-            return;
-        }
-        PyErr_Clear();
-    }
     // Don't call __repr__(), it might recurse into the interpreter.
     printf("<%s at %p>", Py_TYPE(obj)->tp_name, (void *)obj);
 }
@@ -197,6 +185,7 @@ dump_stack(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer)
 
 static void
 lltrace_instruction(_PyInterpreterFrame *frame,
+                    _PyStackRef _tos,
                     _PyStackRef *stack_pointer,
                     _Py_CODEUNIT *next_instr,
                     int opcode,
@@ -205,6 +194,9 @@ lltrace_instruction(_PyInterpreterFrame *frame,
     if (frame->owner >= FRAME_OWNED_BY_INTERPRETER) {
         return;
     }
+    printf("_tos = ");
+    dump_item(_tos);
+    printf("; ");
     dump_stack(frame, stack_pointer);
     const char *opname = _PyOpcode_OpName[opcode];
     assert(opname != NULL);
@@ -274,6 +266,7 @@ maybe_lltrace_resume_frame(_PyInterpreterFrame *frame, PyObject *globals)
             lltrace = *python_lltrace - '0';  // TODO: Parse an int and all that
         }
     }
+    lltrace = 5;
     if (lltrace >= 5) {
         lltrace_resume_frame(frame);
     }
@@ -950,6 +943,13 @@ _PyObjectArray_Free(PyObject **array, PyObject **scratch)
 /* This setting is reversed below following _PyEval_EvalFrameDefault */
 #endif
 
+#ifdef Py_GIL_DISABLED
+#define Py_FREE_THREADING 1
+#else
+#define Py_FREE_THREADING 0
+#endif
+
+
 #if Py_TAIL_CALL_INTERP
 #include "opcode_targets.h"
 #include "generated_cases.c.h"
@@ -986,7 +986,9 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int
      * These are cached values from the frame and code object.  */
     _Py_CODEUNIT *next_instr;
     _PyStackRef *stack_pointer;
-
+#ifdef Py_DEBUG
+    entry_frame.localsplus[0] = PyStackRef_NULL;
+#endif
 #if defined(Py_DEBUG) && !defined(Py_STACKREF_DEBUG)
     /* Set these to invalid but identifiable values for debugging. */
     entry_frame.f_funcobj = (_PyStackRef){.bits = 0xaaa0};
@@ -997,7 +999,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int
 #endif
     entry_frame.f_executable = PyStackRef_None;
     entry_frame.instr_ptr = (_Py_CODEUNIT *)_Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS + 1;
-    entry_frame.stackpointer = entry_frame.localsplus;
+    entry_frame.stackpointer = entry_frame.localsplus + 1;
     entry_frame.owner = FRAME_OWNED_BY_INTERPRETER;
     entry_frame.visited = 0;
     entry_frame.return_offset = 0;
@@ -1044,7 +1046,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int
     _PyExecutorObject *current_executor = NULL;
     const _PyUOpInstruction *next_uop = NULL;
 #endif
-
+    _PyStackRef _tos;
 #if Py_TAIL_CALL_INTERP
     return _TAIL_CALL_start_frame(frame, NULL, tstate, NULL, 0);
 #else

Python/ceval_macros.h

@@ -110,7 +110,7 @@
 /* PRE_DISPATCH_GOTO() does lltrace if enabled. Normally a no-op */
 #ifdef Py_DEBUG
 #define PRE_DISPATCH_GOTO() if (frame->lltrace >= 5) { \
-    lltrace_instruction(frame, stack_pointer, next_instr, opcode, oparg); }
+    lltrace_instruction(frame, _tos, stack_pointer, next_instr, opcode, oparg); }
 #else
 #define PRE_DISPATCH_GOTO() ((void)0)
 #endif
@@ -238,7 +238,7 @@ GETITEM(PyObject *v, Py_ssize_t i) {
 #endif
 
 #define WITHIN_STACK_BOUNDS() \
-   (frame->owner == FRAME_OWNED_BY_INTERPRETER || (STACK_LEVEL() >= 0 && STACK_LEVEL() <= STACK_SIZE()))
+   (frame->owner == FRAME_OWNED_BY_INTERPRETER || (STACK_LEVEL() >= -1 && STACK_LEVEL() <= STACK_SIZE()))
 
 /* Data access macros */
 #define FRAME_CO_CONSTS (_PyFrame_GetCode(frame)->co_consts)