Skip to content

Commit 89d2a51

Browse files
committed
Add variants of object malloc and free that avoid indirection and use them for strings and ints.
1 parent c520bf9 commit 89d2a51

File tree

4 files changed

+72
-13
lines changed

4 files changed

+72
-13
lines changed

Include/cpython/object.h

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -506,3 +506,7 @@ PyAPI_FUNC(int) PyType_Unwatch(int watcher_id, PyObject *type);
506506
* assigned, or 0 if a new tag could not be assigned.
507507
*/
508508
PyAPI_FUNC(int) PyUnstable_Type_AssignVersionTag(PyTypeObject *type);
509+
510+
/* Streamlined allocate and free variants */
511+
void *_PyObject_MallocFast(size_t size);
512+
void _PyObject_FreeFast(void *ptr);

Objects/longobject.c

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -154,7 +154,7 @@ _PyLong_New(Py_ssize_t size)
154154
sizeof() instead of the offsetof, but this risks being
155155
incorrect in the presence of padding between the header
156156
and the digits. */
157-
result = PyObject_Malloc(offsetof(PyLongObject, long_value.ob_digit) +
157+
result = _PyObject_MallocFast(offsetof(PyLongObject, long_value.ob_digit) +
158158
ndigits*sizeof(digit));
159159
if (!result) {
160160
PyErr_NoMemory();
@@ -206,7 +206,7 @@ _PyLong_FromMedium(sdigit x)
206206
assert(!IS_SMALL_INT(x));
207207
assert(is_medium_int(x));
208208
/* We could use a freelist here */
209-
PyLongObject *v = PyObject_Malloc(sizeof(PyLongObject));
209+
PyLongObject *v = _PyObject_MallocFast(sizeof(PyLongObject));
210210
if (v == NULL) {
211211
PyErr_NoMemory();
212212
return NULL;
@@ -3553,7 +3553,11 @@ long_dealloc(PyObject *self)
35533553
}
35543554
}
35553555
}
3556-
Py_TYPE(self)->tp_free(self);
3556+
if (PyLong_CheckExact(self)) {
3557+
_PyObject_FreeFast(self);
3558+
} else {
3559+
Py_TYPE(self)->tp_free(self);
3560+
}
35573561
}
35583562

35593563
static Py_hash_t

Objects/obmalloc.c

Lines changed: 54 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -2073,7 +2073,7 @@ allocate_from_new_pool(OMState *state, uint size)
20732073
or when the max memory limit has been reached.
20742074
*/
20752075
static inline void*
2076-
pymalloc_alloc(OMState *state, void *Py_UNUSED(ctx), size_t nbytes)
2076+
pymalloc_alloc(OMState *state, size_t nbytes)
20772077
{
20782078
#ifdef WITH_VALGRIND
20792079
if (UNLIKELY(running_on_valgrind == -1)) {
@@ -2084,9 +2084,7 @@ pymalloc_alloc(OMState *state, void *Py_UNUSED(ctx), size_t nbytes)
20842084
}
20852085
#endif
20862086

2087-
if (UNLIKELY(nbytes == 0)) {
2088-
return NULL;
2089-
}
2087+
assert(nbytes != 0);
20902088
if (UNLIKELY(nbytes > SMALL_REQUEST_THRESHOLD)) {
20912089
return NULL;
20922090
}
@@ -2123,8 +2121,11 @@ pymalloc_alloc(OMState *state, void *Py_UNUSED(ctx), size_t nbytes)
21232121
void *
21242122
_PyObject_Malloc(void *ctx, size_t nbytes)
21252123
{
2124+
if (UNLIKELY(nbytes == 0)) {
2125+
return NULL;
2126+
}
21262127
OMState *state = get_state();
2127-
void* ptr = pymalloc_alloc(state, ctx, nbytes);
2128+
void* ptr = pymalloc_alloc(state, nbytes);
21282129
if (LIKELY(ptr != NULL)) {
21292130
return ptr;
21302131
}
@@ -2142,9 +2143,12 @@ _PyObject_Calloc(void *ctx, size_t nelem, size_t elsize)
21422143
{
21432144
assert(elsize == 0 || nelem <= (size_t)PY_SSIZE_T_MAX / elsize);
21442145
size_t nbytes = nelem * elsize;
2146+
if (UNLIKELY(nbytes == 0)) {
2147+
return NULL;
2148+
}
21452149

21462150
OMState *state = get_state();
2147-
void* ptr = pymalloc_alloc(state, ctx, nbytes);
2151+
void* ptr = pymalloc_alloc(state, nbytes);
21482152
if (LIKELY(ptr != NULL)) {
21492153
memset(ptr, 0, nbytes);
21502154
return ptr;
@@ -2345,7 +2349,7 @@ insert_to_freepool(OMState *state, poolp pool)
23452349
Return 1 if it was freed.
23462350
Return 0 if the block was not allocated by pymalloc_alloc(). */
23472351
static inline int
2348-
pymalloc_free(OMState *state, void *Py_UNUSED(ctx), void *p)
2352+
pymalloc_free(OMState *state, void *p)
23492353
{
23502354
assert(p != NULL);
23512355

@@ -2411,7 +2415,7 @@ _PyObject_Free(void *ctx, void *p)
24112415
}
24122416

24132417
OMState *state = get_state();
2414-
if (UNLIKELY(!pymalloc_free(state, ctx, p))) {
2418+
if (UNLIKELY(!pymalloc_free(state, p))) {
24152419
/* pymalloc didn't allocate this address */
24162420
PyMem_RawFree(p);
24172421
raw_allocated_blocks--;
@@ -3447,3 +3451,45 @@ _PyObject_DebugMallocStats(FILE *out)
34473451
}
34483452

34493453
#endif /* #ifdef WITH_PYMALLOC */
3454+
3455+
void *
3456+
_PyObject_MallocFast(size_t size)
3457+
{
3458+
assert(size != 0);
3459+
OBJECT_STAT_INC_COND(allocations512, size < 512);
3460+
OBJECT_STAT_INC_COND(allocations4k, size >= 512 && size < 4094);
3461+
OBJECT_STAT_INC_COND(allocations_big, size >= 4094);
3462+
OBJECT_STAT_INC(allocations);
3463+
if (_PyObject.malloc == _PyObject_Malloc) {
3464+
OMState *state = get_state();
3465+
void* ptr = pymalloc_alloc(state, size);
3466+
if (LIKELY(ptr != NULL)) {
3467+
return ptr;
3468+
}
3469+
ptr = PyMem_RawMalloc(size);
3470+
if (ptr != NULL) {
3471+
raw_allocated_blocks++;
3472+
}
3473+
return ptr;
3474+
}
3475+
return _PyObject.malloc(_PyObject.ctx, size);
3476+
}
3477+
3478+
void
3479+
_PyObject_FreeFast(void *ptr)
3480+
{
3481+
/* PyObject_Free(NULL) has no effect */
3482+
assert(ptr != NULL);
3483+
OBJECT_STAT_INC(frees);
3484+
if (_PyObject.free == _PyObject_Free) {
3485+
OMState *state = get_state();
3486+
if (UNLIKELY(!pymalloc_free(state, ptr))) {
3487+
/* pymalloc didn't allocate this address */
3488+
PyMem_RawFree(ptr);
3489+
raw_allocated_blocks--;
3490+
}
3491+
}
3492+
else {
3493+
_PyObject.free(_PyObject.ctx, ptr);
3494+
}
3495+
}

Objects/unicodeobject.c

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1235,7 +1235,7 @@ PyUnicode_New(Py_ssize_t size, Py_UCS4 maxchar)
12351235
* PyObject_New() so we are able to allocate space for the object and
12361236
* it's data buffer.
12371237
*/
1238-
obj = (PyObject *) PyObject_Malloc(struct_size + (size + 1) * char_size);
1238+
obj = (PyObject *) _PyObject_MallocFast(struct_size + (size + 1) * char_size);
12391239
if (obj == NULL) {
12401240
return PyErr_NoMemory();
12411241
}
@@ -1596,7 +1596,12 @@ unicode_dealloc(PyObject *unicode)
15961596
PyMem_Free(_PyUnicode_DATA_ANY(unicode));
15971597
}
15981598

1599-
Py_TYPE(unicode)->tp_free(unicode);
1599+
if (PyUnicode_CheckExact(unicode)) {
1600+
_PyObject_FreeFast(unicode);
1601+
}
1602+
else {
1603+
Py_TYPE(unicode)->tp_free(unicode);
1604+
}
16001605
}
16011606

16021607
#ifdef Py_DEBUG

0 commit comments

Comments
 (0)