Streamline PyObject_Malloc and PyObject_Free. Use the fast variants in more places

Author: markshannon

Objects/bytesobject.c

@@ -93,7 +93,7 @@ _PyBytes_FromSize(Py_ssize_t size, int use_calloc)
     if (use_calloc)
         op = (PyBytesObject *)PyObject_Calloc(1, PyBytesObject_SIZE + size);
     else
-        op = (PyBytesObject *)PyObject_Malloc(PyBytesObject_SIZE + size);
+        op = (PyBytesObject *)_PyObject_MallocFast(PyBytesObject_SIZE + size);
     if (op == NULL) {
         return PyErr_NoMemory();
     }
@@ -160,7 +160,7 @@ PyBytes_FromString(const char *str)
     }
 
     /* Inline PyObject_NewVar */
-    op = (PyBytesObject *)PyObject_Malloc(PyBytesObject_SIZE + size);
+    op = (PyBytesObject *)_PyObject_MallocFast(PyBytesObject_SIZE + size);
     if (op == NULL) {
         return PyErr_NoMemory();
     }
@@ -1470,7 +1470,7 @@ bytes_repeat(PyBytesObject *a, Py_ssize_t n)
             "repeated bytes are too long");
         return NULL;
     }
-    op = (PyBytesObject *)PyObject_Malloc(PyBytesObject_SIZE + nbytes);
+    op = (PyBytesObject *)_PyObject_MallocFast(PyBytesObject_SIZE + nbytes);
     if (op == NULL) {
         return PyErr_NoMemory();
     }

Objects/codeobject.c

@@ -1749,7 +1749,7 @@ code_dealloc(PyCodeObject *co)
         PyObject_ClearWeakRefs((PyObject*)co);
     }
     free_monitoring_data(co->_co_monitoring);
-    PyObject_Free(co);
+    _PyObject_FreeFast(co);
 }
 
 static PyObject *

Objects/floatobject.c

@@ -144,7 +144,7 @@ PyFloat_FromDouble(double fval)
     else
 #endif
     {
-        op = PyObject_Malloc(sizeof(PyFloatObject));
+        op = _PyObject_MallocFast(sizeof(PyFloatObject));
         if (!op) {
             return PyErr_NoMemory();
         }
@@ -252,15 +252,15 @@ _PyFloat_ExactDealloc(PyObject *obj)
 #ifdef WITH_FREELISTS
     struct _Py_float_freelist *float_freelist = get_float_freelist();
     if (float_freelist->numfree >= PyFloat_MAXFREELIST || float_freelist->numfree < 0) {
-        PyObject_Free(op);
+        _PyObject_FreeFast(op);
         return;
     }
     float_freelist->numfree++;
     Py_SET_TYPE(op, (PyTypeObject *)float_freelist->items);
     float_freelist->items = op;
     OBJECT_STAT_INC(to_freelist);
 #else
-    PyObject_Free(op);
+    _PyObject_FreeFast(op);
 #endif
 }
 

Objects/object.c

@@ -448,7 +448,7 @@ PyObject_InitVar(PyVarObject *op, PyTypeObject *tp, Py_ssize_t size)
 PyObject *
 _PyObject_New(PyTypeObject *tp)
 {
-    PyObject *op = (PyObject *) PyObject_Malloc(_PyObject_SIZE(tp));
+    PyObject *op = (PyObject *) _PyObject_MallocFast(_PyObject_SIZE(tp));
     if (op == NULL) {
         return PyErr_NoMemory();
     }
@@ -461,7 +461,7 @@ _PyObject_NewVar(PyTypeObject *tp, Py_ssize_t nitems)
 {
     PyVarObject *op;
     const size_t size = _PyObject_VAR_SIZE(tp, nitems);
-    op = (PyVarObject *) PyObject_Malloc(size);
+    op = (PyVarObject *) _PyObject_MallocFast(size);
     if (op == NULL) {
         return (PyVarObject *)PyErr_NoMemory();
     }

Objects/obmalloc.c

@@ -1281,19 +1281,6 @@ _PyMem_FiniDelayed(PyInterpreterState *interp)
 /* the "object" allocator */
 /**************************/
 
-void *
-PyObject_Malloc(size_t size)
-{
-    /* see PyMem_RawMalloc() */
-    if (size > (size_t)PY_SSIZE_T_MAX)
-        return NULL;
-    OBJECT_STAT_INC_COND(allocations512, size < 512);
-    OBJECT_STAT_INC_COND(allocations4k, size >= 512 && size < 4094);
-    OBJECT_STAT_INC_COND(allocations_big, size >= 4094);
-    OBJECT_STAT_INC(allocations);
-    return _PyObject.malloc(_PyObject.ctx, size);
-}
-
 void *
 PyObject_Calloc(size_t nelem, size_t elsize)
 {
@@ -1316,14 +1303,6 @@ PyObject_Realloc(void *ptr, size_t new_size)
     return _PyObject.realloc(_PyObject.ctx, ptr, new_size);
 }
 
-void
-PyObject_Free(void *ptr)
-{
-    OBJECT_STAT_INC(frees);
-    _PyObject.free(_PyObject.ctx, ptr);
-}
-
-
 /* If we're using GCC, use __builtin_expect() to reduce overhead of
    the valgrind checks */
 #if defined(__GNUC__) && (__GNUC__ > 2) && defined(__OPTIMIZE__)
@@ -2084,9 +2063,7 @@ pymalloc_alloc(OMState *state, void *Py_UNUSED(ctx), size_t nbytes)
     }
 #endif
 
-    if (UNLIKELY(nbytes == 0)) {
-        return NULL;
-    }
+    assert(nbytes != 0 && nbytes <= (size_t)PY_SSIZE_T_MAX/2);
     if (UNLIKELY(nbytes > SMALL_REQUEST_THRESHOLD)) {
         return NULL;
     }
@@ -2119,17 +2096,23 @@ pymalloc_alloc(OMState *state, void *Py_UNUSED(ctx), size_t nbytes)
     return (void *)bp;
 }
 
+static inline bool
+zero_or_too_large(size_t nbytes)
+{
+    return (nbytes-1) >= (size_t)PY_SSIZE_T_MAX;
+}
 
 void *
 _PyObject_Malloc(void *ctx, size_t nbytes)
 {
     OMState *state = get_state();
-    void* ptr = pymalloc_alloc(state, ctx, nbytes);
-    if (LIKELY(ptr != NULL)) {
-        return ptr;
+    if (LIKELY(!zero_or_too_large(nbytes))) {
+        void* ptr = pymalloc_alloc(state, ctx, nbytes);
+        if (LIKELY(ptr != NULL)) {
+            return ptr;
+        }
     }
-
-    ptr = PyMem_RawMalloc(nbytes);
+    void* ptr = PyMem_RawMalloc(nbytes);
     if (ptr != NULL) {
         raw_allocated_blocks++;
     }
@@ -2144,13 +2127,14 @@ _PyObject_Calloc(void *ctx, size_t nelem, size_t elsize)
     size_t nbytes = nelem * elsize;
 
     OMState *state = get_state();
-    void* ptr = pymalloc_alloc(state, ctx, nbytes);
-    if (LIKELY(ptr != NULL)) {
-        memset(ptr, 0, nbytes);
-        return ptr;
+    if (LIKELY(!zero_or_too_large(nbytes))) {
+        void *ptr = pymalloc_alloc(state, ctx, nbytes);
+        if (LIKELY(ptr != NULL)) {
+            memset(ptr, 0, nbytes);
+            return ptr;
+        }
     }
-
-    ptr = PyMem_RawCalloc(nelem, elsize);
+    void *ptr = PyMem_RawCalloc(nelem, elsize);
     if (ptr != NULL) {
         raw_allocated_blocks++;
     }
@@ -3449,14 +3433,14 @@ _PyObject_DebugMallocStats(FILE *out)
 void *
 _PyObject_MallocFast(size_t size)
 {
-    assert(size != 0);
+    assert(!zero_or_too_large(size));
     OBJECT_STAT_INC_COND(allocations512, size < 512);
     OBJECT_STAT_INC_COND(allocations4k, size >= 512 && size < 4094);
     OBJECT_STAT_INC_COND(allocations_big, size >= 4094);
     OBJECT_STAT_INC(allocations);
     if (_PyObject.malloc == _PyObject_Malloc) {
         OMState *state = get_state();
-        void* ptr = pymalloc_alloc(state, NULL, size);
+        void *ptr = pymalloc_alloc(state, NULL, size);
         if (LIKELY(ptr != NULL)) {
             return ptr;
         }
@@ -3469,10 +3453,22 @@ _PyObject_MallocFast(size_t size)
     return _PyObject.malloc(_PyObject.ctx, size);
 }
 
+void *
+PyObject_Malloc(size_t size)
+{
+    if (LIKELY(!zero_or_too_large(size))) {
+        return _PyObject_MallocFast(size);
+    }
+    if (size == 0) {
+        return _PyObject.malloc(_PyObject.ctx, 0);
+    }
+    return NULL;
+}
+
 void
 _PyObject_FreeFast(void *ptr)
 {
-    /* PyObject_Free(NULL) has no effect */
+    /* Don't check for NULL */
     assert(ptr != NULL);
     OBJECT_STAT_INC(frees);
     if (_PyObject.free == _PyObject_Free) {
@@ -3488,5 +3484,14 @@ _PyObject_FreeFast(void *ptr)
     }
 }
 
+void
+PyObject_Free(void *ptr)
+{
+    if (ptr == NULL) {
+        return;
+    }
+    _PyObject_FreeFast(ptr);
+}
+
 
 #endif /* #ifdef WITH_PYMALLOC */

Python/bytecodes.c

@@ -427,24 +427,24 @@ dummy_func(
         pure op(_BINARY_OP_MULTIPLY_INT, (left, right -- res)) {
             STAT_INC(BINARY_OP, hit);
             res = _PyLong_Multiply((PyLongObject *)left, (PyLongObject *)right);
-            _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free);
-            _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free);
+            _Py_DECREF_SPECIALIZED(right, (destructor)_PyObject_FreeFast);
+            _Py_DECREF_SPECIALIZED(left, (destructor)_PyObject_FreeFast);
             ERROR_IF(res == NULL, error);
         }
 
         pure op(_BINARY_OP_ADD_INT, (left, right -- res)) {
             STAT_INC(BINARY_OP, hit);
             res = _PyLong_Add((PyLongObject *)left, (PyLongObject *)right);
-            _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free);
-            _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free);
+            _Py_DECREF_SPECIALIZED(right, (destructor)_PyObject_FreeFast);
+            _Py_DECREF_SPECIALIZED(left, (destructor)_PyObject_FreeFast);
             ERROR_IF(res == NULL, error);
         }
 
         pure op(_BINARY_OP_SUBTRACT_INT, (left, right -- res)) {
             STAT_INC(BINARY_OP, hit);
             res = _PyLong_Subtract((PyLongObject *)left, (PyLongObject *)right);
-            _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free);
-            _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free);
+            _Py_DECREF_SPECIALIZED(right, (destructor)_PyObject_FreeFast);
+            _Py_DECREF_SPECIALIZED(left, (destructor)_PyObject_FreeFast);
             ERROR_IF(res == NULL, error);
         }
 
@@ -612,7 +612,7 @@ dummy_func(
             res = PyList_GET_ITEM(list, index);
             assert(res != NULL);
             Py_INCREF(res);
-            _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free);
+            _Py_DECREF_SPECIALIZED(sub, (destructor)_PyObject_FreeFast);
             Py_DECREF(list);
         }
 
@@ -627,7 +627,7 @@ dummy_func(
             DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c);
             STAT_INC(BINARY_SUBSCR, hit);
             res = (PyObject*)&_Py_SINGLETON(strings).ascii[c];
-            _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free);
+            _Py_DECREF_SPECIALIZED(sub, (destructor)_PyObject_FreeFast);
             Py_DECREF(str);
         }