Avoid layers of indirection allocating and freeing strs and ints

Author: markshannon

Include/cpython/object.h

@@ -506,3 +506,7 @@ PyAPI_FUNC(int) PyType_Unwatch(int watcher_id, PyObject *type);
  * assigned, or 0 if a new tag could not be assigned.
  */
 PyAPI_FUNC(int) PyUnstable_Type_AssignVersionTag(PyTypeObject *type);
+
+/* Streamlined allocate and free variants */
+void *_PyObject_MallocFast(size_t size);
+void _PyObject_FreeFast(void *ptr);

Objects/longobject.c

@@ -154,7 +154,7 @@ _PyLong_New(Py_ssize_t size)
        sizeof() instead of the offsetof, but this risks being
        incorrect in the presence of padding between the header
        and the digits. */
-    result = PyObject_Malloc(offsetof(PyLongObject, long_value.ob_digit) +
+    result = _PyObject_MallocFast(offsetof(PyLongObject, long_value.ob_digit) +
                              ndigits*sizeof(digit));
     if (!result) {
         PyErr_NoMemory();
@@ -206,7 +206,7 @@ _PyLong_FromMedium(sdigit x)
     assert(!IS_SMALL_INT(x));
     assert(is_medium_int(x));
     /* We could use a freelist here */
-    PyLongObject *v = PyObject_Malloc(sizeof(PyLongObject));
+    PyLongObject *v = _PyObject_MallocFast(sizeof(PyLongObject));
     if (v == NULL) {
         PyErr_NoMemory();
         return NULL;
@@ -3553,7 +3553,13 @@ long_dealloc(PyObject *self)
             }
         }
     }
-    Py_TYPE(self)->tp_free(self);
+    if (PyLong_CheckExact(self)) {
+        assert(Py_TYPE(self)->tp_free == PyObject_Free);
+        _PyObject_FreeFast(self);
+    }
+    else {
+        Py_TYPE(self)->tp_free(self);
+    }
 }
 
 static Py_hash_t

Objects/obmalloc.c

@@ -3446,4 +3446,47 @@ _PyObject_DebugMallocStats(FILE *out)
     }
 }
 
+void *
+_PyObject_MallocFast(size_t size)
+{
+    assert(size != 0);
+    OBJECT_STAT_INC_COND(allocations512, size < 512);
+    OBJECT_STAT_INC_COND(allocations4k, size >= 512 && size < 4094);
+    OBJECT_STAT_INC_COND(allocations_big, size >= 4094);
+    OBJECT_STAT_INC(allocations);
+    if (_PyObject.malloc == _PyObject_Malloc) {
+        OMState *state = get_state();
+        void* ptr = pymalloc_alloc(state, NULL, size);
+        if (LIKELY(ptr != NULL)) {
+            return ptr;
+        }
+        ptr = PyMem_RawMalloc(size);
+        if (ptr != NULL) {
+            raw_allocated_blocks++;
+        }
+        return ptr;
+    }
+    return _PyObject.malloc(_PyObject.ctx, size);
+}
+
+void
+_PyObject_FreeFast(void *ptr)
+{
+    /* PyObject_Free(NULL) has no effect */
+    assert(ptr != NULL);
+    OBJECT_STAT_INC(frees);
+    if (_PyObject.free == _PyObject_Free) {
+        OMState *state = get_state();
+        if (UNLIKELY(!pymalloc_free(state, NULL, ptr))) {
+            /* pymalloc didn't allocate this address */
+            PyMem_RawFree(ptr);
+            raw_allocated_blocks--;
+        }
+    }
+    else {
+        _PyObject.free(_PyObject.ctx, ptr);
+    }
+}
+
+
 #endif /* #ifdef WITH_PYMALLOC */

Objects/unicodeobject.c

@@ -1235,7 +1235,7 @@ PyUnicode_New(Py_ssize_t size, Py_UCS4 maxchar)
      * PyObject_New() so we are able to allocate space for the object and
      * it's data buffer.
      */
-    obj = (PyObject *) PyObject_Malloc(struct_size + (size + 1) * char_size);
+    obj = (PyObject *) _PyObject_MallocFast(struct_size + (size + 1) * char_size);
     if (obj == NULL) {
         return PyErr_NoMemory();
     }
@@ -1596,7 +1596,13 @@ unicode_dealloc(PyObject *unicode)
         PyMem_Free(_PyUnicode_DATA_ANY(unicode));
     }
 
-    Py_TYPE(unicode)->tp_free(unicode);
+    if (PyUnicode_CheckExact(unicode)) {
+        assert(Py_TYPE(unicode)->tp_free == PyObject_Free);
+        _PyObject_FreeFast(unicode);
+    }
+    else {
+        Py_TYPE(unicode)->tp_free(unicode);
+    }
 }
 
 #ifdef Py_DEBUG