Add variants of object malloc and free that avoid indirection and use them for strings and ints.

Author: markshannon

Include/cpython/object.h

@@ -506,3 +506,7 @@ PyAPI_FUNC(int) PyType_Unwatch(int watcher_id, PyObject *type);
  * assigned, or 0 if a new tag could not be assigned.
  */
 PyAPI_FUNC(int) PyUnstable_Type_AssignVersionTag(PyTypeObject *type);
+
+/* Streamlined allocate and free variants */
+void *_PyObject_MallocFast(size_t size);
+void _PyObject_FreeFast(void *ptr);

Objects/longobject.c

@@ -154,7 +154,7 @@ _PyLong_New(Py_ssize_t size)
        sizeof() instead of the offsetof, but this risks being
        incorrect in the presence of padding between the header
        and the digits. */
-    result = PyObject_Malloc(offsetof(PyLongObject, long_value.ob_digit) +
+    result = _PyObject_MallocFast(offsetof(PyLongObject, long_value.ob_digit) +
                              ndigits*sizeof(digit));
     if (!result) {
         PyErr_NoMemory();
@@ -206,7 +206,7 @@ _PyLong_FromMedium(sdigit x)
     assert(!IS_SMALL_INT(x));
     assert(is_medium_int(x));
     /* We could use a freelist here */
-    PyLongObject *v = PyObject_Malloc(sizeof(PyLongObject));
+    PyLongObject *v = _PyObject_MallocFast(sizeof(PyLongObject));
     if (v == NULL) {
         PyErr_NoMemory();
         return NULL;
@@ -3553,7 +3553,11 @@ long_dealloc(PyObject *self)
             }
         }
     }
-    Py_TYPE(self)->tp_free(self);
+    if (PyLong_CheckExact(self)) {
+        _PyObject_FreeFast(self);
+    } else {
+        Py_TYPE(self)->tp_free(self);
+    }
 }
 
 static Py_hash_t

Objects/obmalloc.c

@@ -2073,7 +2073,7 @@ allocate_from_new_pool(OMState *state, uint size)
    or when the max memory limit has been reached.
 */
 static inline void*
-pymalloc_alloc(OMState *state, void *Py_UNUSED(ctx), size_t nbytes)
+pymalloc_alloc(OMState *state, size_t nbytes)
 {
 #ifdef WITH_VALGRIND
     if (UNLIKELY(running_on_valgrind == -1)) {
@@ -2084,9 +2084,7 @@ pymalloc_alloc(OMState *state, void *Py_UNUSED(ctx), size_t nbytes)
     }
 #endif
 
-    if (UNLIKELY(nbytes == 0)) {
-        return NULL;
-    }
+    assert(nbytes != 0);
     if (UNLIKELY(nbytes > SMALL_REQUEST_THRESHOLD)) {
         return NULL;
     }
@@ -2123,8 +2121,11 @@ pymalloc_alloc(OMState *state, void *Py_UNUSED(ctx), size_t nbytes)
 void *
 _PyObject_Malloc(void *ctx, size_t nbytes)
 {
+    if (UNLIKELY(nbytes == 0)) {
+        return NULL;
+    }
     OMState *state = get_state();
-    void* ptr = pymalloc_alloc(state, ctx, nbytes);
+    void* ptr = pymalloc_alloc(state, nbytes);
     if (LIKELY(ptr != NULL)) {
         return ptr;
     }
@@ -2142,9 +2143,12 @@ _PyObject_Calloc(void *ctx, size_t nelem, size_t elsize)
 {
     assert(elsize == 0 || nelem <= (size_t)PY_SSIZE_T_MAX / elsize);
     size_t nbytes = nelem * elsize;
+    if (UNLIKELY(nbytes == 0)) {
+        return NULL;
+    }
 
     OMState *state = get_state();
-    void* ptr = pymalloc_alloc(state, ctx, nbytes);
+    void* ptr = pymalloc_alloc(state, nbytes);
     if (LIKELY(ptr != NULL)) {
         memset(ptr, 0, nbytes);
         return ptr;
@@ -2345,7 +2349,7 @@ insert_to_freepool(OMState *state, poolp pool)
    Return 1 if it was freed.
    Return 0 if the block was not allocated by pymalloc_alloc(). */
 static inline int
-pymalloc_free(OMState *state, void *Py_UNUSED(ctx), void *p)
+pymalloc_free(OMState *state, void *p)
 {
     assert(p != NULL);
 
@@ -2411,7 +2415,7 @@ _PyObject_Free(void *ctx, void *p)
     }
 
     OMState *state = get_state();
-    if (UNLIKELY(!pymalloc_free(state, ctx, p))) {
+    if (UNLIKELY(!pymalloc_free(state, p))) {
         /* pymalloc didn't allocate this address */
         PyMem_RawFree(p);
         raw_allocated_blocks--;
@@ -3447,3 +3451,45 @@ _PyObject_DebugMallocStats(FILE *out)
 }
 
 #endif /* #ifdef WITH_PYMALLOC */
+
+void *
+_PyObject_MallocFast(size_t size)
+{
+    assert(size != 0);
+    OBJECT_STAT_INC_COND(allocations512, size < 512);
+    OBJECT_STAT_INC_COND(allocations4k, size >= 512 && size < 4094);
+    OBJECT_STAT_INC_COND(allocations_big, size >= 4094);
+    OBJECT_STAT_INC(allocations);
+    if (_PyObject.malloc == _PyObject_Malloc) {
+        OMState *state = get_state();
+        void* ptr = pymalloc_alloc(state, size);
+        if (LIKELY(ptr != NULL)) {
+            return ptr;
+        }
+        ptr = PyMem_RawMalloc(size);
+        if (ptr != NULL) {
+            raw_allocated_blocks++;
+        }
+        return ptr;
+    }
+    return _PyObject.malloc(_PyObject.ctx, size);
+}
+
+void
+_PyObject_FreeFast(void *ptr)
+{
+    /* PyObject_Free(NULL) has no effect */
+    assert(ptr != NULL);
+    OBJECT_STAT_INC(frees);
+    if (_PyObject.free == _PyObject_Free) {
+        OMState *state = get_state();
+        if (UNLIKELY(!pymalloc_free(state, ptr))) {
+            /* pymalloc didn't allocate this address */
+            PyMem_RawFree(ptr);
+            raw_allocated_blocks--;
+        }
+    }
+    else {
+        _PyObject.free(_PyObject.ctx, ptr);
+    }
+}

Objects/unicodeobject.c

@@ -1235,7 +1235,7 @@ PyUnicode_New(Py_ssize_t size, Py_UCS4 maxchar)
      * PyObject_New() so we are able to allocate space for the object and
      * it's data buffer.
      */
-    obj = (PyObject *) PyObject_Malloc(struct_size + (size + 1) * char_size);
+    obj = (PyObject *) _PyObject_MallocFast(struct_size + (size + 1) * char_size);
     if (obj == NULL) {
         return PyErr_NoMemory();
     }
@@ -1596,7 +1596,12 @@ unicode_dealloc(PyObject *unicode)
         PyMem_Free(_PyUnicode_DATA_ANY(unicode));
     }
 
-    Py_TYPE(unicode)->tp_free(unicode);
+    if (PyUnicode_CheckExact(unicode)) {
+        _PyObject_FreeFast(unicode);
+    }
+    else {
+        Py_TYPE(unicode)->tp_free(unicode);
+    }
 }
 
 #ifdef Py_DEBUG