[POC] trim

include/umf/memory_pool.h

@@ -196,6 +196,15 @@ umf_result_t umfPoolSetTag(umf_memory_pool_handle_t hPool, void *tag,
 /// @return UMF_RESULT_SUCCESS on success.
 umf_result_t umfPoolGetTag(umf_memory_pool_handle_t hPool, void **tag);
 
+///
+/// @brief Trims memory pool to keep at least \p minBytesToKeep bytes of memory
+///        if possible.
+/// @param hPool specified memory pool
+/// @param minBytesToKeep minimum number of bytes to keep in the pool
+/// @return UMF_RESULT_SUCCESS on success or appropriate error code on failure.
+umf_result_t umfPoolTrimMemory(umf_memory_pool_handle_t hPool,
+                               size_t minBytesToKeep);
+
 #ifdef __cplusplus
 }
 #endif

include/umf/memory_pool_ops.h

@@ -168,6 +168,16 @@ typedef struct umf_memory_pool_ops_t {
                             const char *name, void *arg, size_t size,
                             umf_ctl_query_type_t queryType, va_list args);
 
+    ///
+    /// @brief Trims memory of the pool, removing resources that are not needed
+    ///        to keep the pool operational.
+    /// @param pool pointer to the memory pool
+    /// @param minBytesToKeep minimum number of bytes to keep in the pool if
+    ///        possible.
+    /// @return UMF_RESULT_SUCCESS on success or appropriate error code on
+    ///         failure.
+    ///
+    umf_result_t (*ext_trim_memory)(void *pool, size_t minBytesToKeep);
 } umf_memory_pool_ops_t;
 
 #ifdef __cplusplus

src/libumf.def

@@ -144,3 +144,5 @@ EXPORTS
     umfJemallocPoolParamsDestroy
     umfJemallocPoolParamsSetNumArenas
     umfPoolGetName
+; Added in UMF_1.1
+    umfPoolTrimMemory

src/libumf.map

@@ -141,3 +141,7 @@ UMF_1.0 {
     local:
         *;
 };
+
+UMF_1.1 {
+        umfPoolTrimMemory;
+} UMF_1.0;

src/memory_pool.c

@@ -167,6 +167,13 @@ umfDefaultCtlPoolHandle(void *hPool, umf_ctl_query_source_t operationType,
     return UMF_RESULT_ERROR_NOT_SUPPORTED;
 }
 
+static umf_result_t umfDefaultTrimMemory(void *provider,
+                                         size_t minBytesToKeep) {
+    (void)provider;
+    (void)minBytesToKeep;
+    return UMF_RESULT_ERROR_NOT_SUPPORTED;
+}
+
 // logical sum (OR) of all umf_pool_create_flags_t flags
 static const umf_pool_create_flags_t UMF_POOL_CREATE_FLAG_ALL =
     UMF_POOL_CREATE_FLAG_OWN_PROVIDER | UMF_POOL_CREATE_FLAG_DISABLE_TRACKING;
@@ -189,9 +196,9 @@ static umf_result_t umfPoolCreateInternal(const umf_memory_pool_ops_t *ops,
                                           const void *params,
                                           umf_pool_create_flags_t flags,
                                           umf_memory_pool_handle_t *hPool) {
-    if (!ops || !provider || !hPool) {
-        return UMF_RESULT_ERROR_INVALID_ARGUMENT;
-    }
+    UMF_CHECK((ops != NULL), UMF_RESULT_ERROR_INVALID_ARGUMENT);
+    UMF_CHECK((provider != NULL), UMF_RESULT_ERROR_INVALID_ARGUMENT);
+    UMF_CHECK((hPool != NULL), UMF_RESULT_ERROR_INVALID_ARGUMENT);
 
     // validate flags
     if (flags & ~UMF_POOL_CREATE_FLAG_ALL) {
@@ -234,6 +241,10 @@ static umf_result_t umfPoolCreateInternal(const umf_memory_pool_ops_t *ops,
         pool->ops.ext_ctl = umfDefaultCtlPoolHandle;
     }
 
+    if (NULL == pool->ops.ext_trim_memory) {
+        pool->ops.ext_trim_memory = umfDefaultTrimMemory;
+    }
+
     if (NULL == utils_mutex_init(&pool->lock)) {
         LOG_ERR("Failed to initialize mutex for pool");
         ret = UMF_RESULT_ERROR_UNKNOWN;
@@ -277,6 +288,8 @@ static umf_result_t umfPoolCreateInternal(const umf_memory_pool_ops_t *ops,
 }
 
 umf_result_t umfPoolDestroy(umf_memory_pool_handle_t hPool) {
+    UMF_CHECK((hPool != NULL), UMF_RESULT_ERROR_INVALID_ARGUMENT);
+
     if (umf_ba_global_is_destroyed()) {
         return UMF_RESULT_ERROR_UNKNOWN;
     }
@@ -454,3 +467,10 @@ umf_result_t umfPoolGetTag(umf_memory_pool_handle_t hPool, void **tag) {
     utils_mutex_unlock(&hPool->lock);
     return UMF_RESULT_SUCCESS;
 }
+
+umf_result_t umfPoolTrimMemory(umf_memory_pool_handle_t hPool,
+                               size_t minBytesToKeep) {
+    UMF_CHECK((hPool != NULL), UMF_RESULT_ERROR_INVALID_ARGUMENT);
+
+    return hPool->ops.ext_trim_memory(hPool->pool_priv, minBytesToKeep);
+}

src/pool/pool_disjoint.c

@@ -1133,6 +1133,47 @@ static umf_result_t disjoint_pool_get_name(void *pool, const char **name) {
     return UMF_RESULT_SUCCESS;
 }
 
+umf_result_t disjoint_pool_trim_memory(void *pool, size_t minBytesToKeep) {
+    disjoint_pool_t *hPool = (disjoint_pool_t *)pool;
+    if (hPool == NULL) {
+        return UMF_RESULT_ERROR_INVALID_ARGUMENT;
+    }
+
+    for (size_t i = 0; i < hPool->buckets_num; i++) {
+        bucket_t *bucket = hPool->buckets[i];
+        utils_mutex_lock(&bucket->bucket_lock);
+
+        int skip = (int)minBytesToKeep;
+
+        // remove empty slabs from the pool
+        slab_list_item_t *it = NULL, *tmp = NULL;
+        LL_FOREACH_SAFE(bucket->available_slabs, it, tmp) {
+            slab_t *slab = it->val;
+            if (slab->num_chunks_allocated == 0) {
+                // skip first minBytesToKeep bytes from each bucket
+                if (skip > 0) {
+                    skip -= (int)slab->slab_size;
+                    continue;
+                }
+
+                // remove slab
+                pool_unregister_slab(hPool, slab);
+                DL_DELETE(bucket->available_slabs, it);
+                assert(bucket->available_slabs_num > 0);
+                bucket->available_slabs_num--;
+                destroy_slab(slab);
+
+                // update stats
+                bucket_update_stats(bucket, 0, -1);
+            }
+        }
+
+        utils_mutex_unlock(&bucket->bucket_lock);
+    }
+
+    return UMF_RESULT_SUCCESS;
+}
+
 static umf_memory_pool_ops_t UMF_DISJOINT_POOL_OPS = {
     .version = UMF_POOL_OPS_VERSION_CURRENT,
     .initialize = disjoint_pool_initialize,
@@ -1146,6 +1187,7 @@ static umf_memory_pool_ops_t UMF_DISJOINT_POOL_OPS = {
     .get_last_allocation_error = disjoint_pool_get_last_allocation_error,
     .get_name = disjoint_pool_get_name,
     .ext_ctl = disjoint_pool_ctl,
+    .ext_trim_memory = disjoint_pool_trim_memory,
 };
 
 const umf_memory_pool_ops_t *umfDisjointPoolOps(void) {

src/pool/pool_jemalloc.c

@@ -563,6 +563,23 @@ static umf_result_t op_get_name(void *pool, const char **name) {
     return UMF_RESULT_SUCCESS;
 }
 
+static umf_result_t op_trim_memory(void *pool, size_t minBytesToKeep) {
+    (void)minBytesToKeep; // unused - TODO?
+
+    jemalloc_memory_pool_t *je_pool = (jemalloc_memory_pool_t *)pool;
+    for (size_t i = 0; i < je_pool->n_arenas; i++) {
+        char cmd[64];
+        unsigned arena = je_pool->arena_index[i];
+        snprintf(cmd, sizeof(cmd), "arena.%u.purge", arena);
+        if (je_mallctl(cmd, NULL, NULL, NULL, 0)) {
+            LOG_ERR("Could not purge jemalloc arena %u", arena);
+            return UMF_RESULT_ERROR_UNKNOWN;
+        }
+    }
+
+    return UMF_RESULT_SUCCESS;
+}
+
 static umf_memory_pool_ops_t UMF_JEMALLOC_POOL_OPS = {
     .version = UMF_POOL_OPS_VERSION_CURRENT,
     .initialize = op_initialize,
@@ -575,6 +592,7 @@ static umf_memory_pool_ops_t UMF_JEMALLOC_POOL_OPS = {
     .free = op_free,
     .get_last_allocation_error = op_get_last_allocation_error,
     .get_name = op_get_name,
+    .ext_trim_memory = op_trim_memory,
 };
 
 const umf_memory_pool_ops_t *umfJemallocPoolOps(void) {

src/pool/pool_proxy.c

@@ -136,6 +136,14 @@ static umf_result_t proxy_get_name(void *pool, const char **name) {
     return UMF_RESULT_SUCCESS;
 }
 
+// TODO remove if na
+static umf_result_t proxy_trim_memory(void *pool, size_t minBytesToKeep) {
+    (void)pool;
+    (void)minBytesToKeep;
+
+    return UMF_RESULT_SUCCESS;
+}
+
 static umf_memory_pool_ops_t UMF_PROXY_POOL_OPS = {
     .version = UMF_POOL_OPS_VERSION_CURRENT,
     .initialize = proxy_pool_initialize,
@@ -147,7 +155,9 @@ static umf_memory_pool_ops_t UMF_PROXY_POOL_OPS = {
     .malloc_usable_size = proxy_malloc_usable_size,
     .free = proxy_free,
     .get_last_allocation_error = proxy_get_last_allocation_error,
-    .get_name = proxy_get_name};
+    .get_name = proxy_get_name,
+    .ext_trim_memory = proxy_trim_memory,
+};
 
 const umf_memory_pool_ops_t *umfProxyPoolOps(void) {
     return &UMF_PROXY_POOL_OPS;

src/pool/pool_scalable.c

@@ -60,6 +60,7 @@ typedef struct tbb_callbacks_t {
     bool (*pool_destroy)(void *);
     void *(*pool_identify)(void *object);
     size_t (*pool_msize)(void *, void *);
+    int (*pool_allocation_command)(int, void *);
 #ifdef _WIN32
     HMODULE lib_handle;
 #else
@@ -422,12 +423,14 @@ static umf_result_t tbb_get_last_allocation_error(void *pool) {
     return TLS_last_allocation_error;
 }
 
+static void initialize_pool_ctl(void) {}
+
 static umf_result_t pool_ctl(void *hPool, umf_ctl_query_source_t operationType,
                              const char *name, void *arg, size_t size,
                              umf_ctl_query_type_t query_type, va_list args) {
     (void)operationType; // unused
     umf_memory_pool_handle_t pool_provider = (umf_memory_pool_handle_t)hPool;
-    utils_init_once(&ctl_initialized, NULL);
+    utils_init_once(&ctl_initialized, initialize_pool_ctl);
     return ctl_query(&pool_scallable_ctl_root, pool_provider->pool_priv,
                      CTL_QUERY_PROGRAMMATIC, name, query_type, arg, size, args);
 }
@@ -438,6 +441,15 @@ static umf_result_t scalable_get_name(void *pool, const char **name) {
     return UMF_RESULT_SUCCESS;
 }
 
+// TODO remove if na
+static umf_result_t scalable_trim_memory(void *pool, size_t minBytesToKeep) {
+    (void)pool;           // unused
+    (void)minBytesToKeep; // unused
+
+    //scalable_allocation_command?
+    return UMF_RESULT_SUCCESS;
+}
+
 static umf_memory_pool_ops_t UMF_SCALABLE_POOL_OPS = {
     .version = UMF_POOL_OPS_VERSION_CURRENT,
     .initialize = tbb_pool_initialize,
@@ -451,6 +463,7 @@ static umf_memory_pool_ops_t UMF_SCALABLE_POOL_OPS = {
     .get_last_allocation_error = tbb_get_last_allocation_error,
     .ext_ctl = pool_ctl,
     .get_name = scalable_get_name,
+    .ext_trim_memory = scalable_trim_memory,
 };
 
 const umf_memory_pool_ops_t *umfScalablePoolOps(void) {

src/provider/provider_tracking.c

@@ -472,6 +472,7 @@ umf_result_t umfMemoryTrackerGetAllocInfo(const void *ptr,
                                           umf_alloc_info_t *pAllocInfo) {
     assert(pAllocInfo);
 
+    // TODO unlikely
     if (ptr == NULL) {
         return UMF_RESULT_ERROR_INVALID_ARGUMENT;
     }

src/utils/utils_posix_concurrency.c

@@ -38,6 +38,11 @@ int utils_mutex_unlock(utils_mutex_t *m) {
 }
 
 void utils_init_once(UTIL_ONCE_FLAG *flag, void (*oneCb)(void)) {
+    if (oneCb == NULL) {
+        LOG_FATAL("utils_init_once: callback is NULL");
+        return;
+    }
+
     pthread_once(flag, oneCb);
 }
 

test/common/pool.hpp

@@ -108,25 +108,26 @@ typedef struct pool_base_t {
     umf_result_t initialize(umf_memory_provider_handle_t) noexcept {
         return UMF_RESULT_SUCCESS;
     };
-    void *malloc([[maybe_unused]] size_t size) noexcept { return nullptr; }
+    void *malloc(size_t) noexcept { return nullptr; }
     void *calloc(size_t, size_t) noexcept { return nullptr; }
     void *realloc(void *, size_t) noexcept { return nullptr; }
     void *aligned_malloc(size_t, size_t) noexcept { return nullptr; }
-    umf_result_t malloc_usable_size(const void *, size_t *size) noexcept {
-        if (size) {
-            *size = 0;
-        }
-        return UMF_RESULT_SUCCESS;
+    umf_result_t malloc_usable_size(const void *, size_t *) noexcept {
+        return UMF_RESULT_ERROR_UNKNOWN;
     }
-    umf_result_t free(void *) noexcept { return UMF_RESULT_SUCCESS; }
+    umf_result_t free(void *) noexcept { return UMF_RESULT_ERROR_UNKNOWN; }
     umf_result_t get_last_allocation_error() noexcept {
-        return UMF_RESULT_SUCCESS;
+        return UMF_RESULT_ERROR_UNKNOWN;
     }
-    umf_result_t get_name(const char **name) noexcept {
-        if (name) {
-            *name = "pool_base";
-        }
-        return UMF_RESULT_SUCCESS;
+    umf_result_t get_name(const char **) noexcept {
+        return UMF_RESULT_ERROR_UNKNOWN;
+    }
+    umf_result_t ext_ctl(umf_ctl_query_source_t, const char *, void *, size_t,
+                         umf_ctl_query_type_t, va_list) noexcept {
+        return UMF_RESULT_ERROR_UNKNOWN;
+    }
+    umf_result_t ext_trim_memory(size_t) noexcept {
+        return UMF_RESULT_ERROR_UNKNOWN;
     }
 } pool_base_t;
 
@@ -177,6 +178,11 @@ struct malloc_pool : public pool_base_t {
         }
         return UMF_RESULT_SUCCESS;
     }
+
+    umf_result_t ext_trim_memory(size_t) noexcept {
+        // malloc_pool frees all memory immediately, so we have nothing to trim
+        return UMF_RESULT_SUCCESS;
+    }
 };
 
 umf_memory_pool_ops_t MALLOC_POOL_OPS =

test/common/pool_trace.c

@@ -99,6 +99,14 @@ static umf_result_t traceGetName(void *pool, const char **name) {
     return UMF_RESULT_SUCCESS;
 }
 
+static umf_result_t traceTrimMemory(void *pool, size_t minBytesToKeep) {
+    trace_pool_t *trace_pool = (trace_pool_t *)pool;
+
+    trace_pool->params.trace_handler(trace_pool->params.trace_context,
+                                     "trim_memory");
+    return umfPoolTrimMemory(trace_pool->params.hUpstreamPool, minBytesToKeep);
+}
+
 umf_memory_pool_ops_t UMF_TRACE_POOL_OPS = {
     .version = UMF_POOL_OPS_VERSION_CURRENT,
     .initialize = traceInitialize,
@@ -111,4 +119,5 @@ umf_memory_pool_ops_t UMF_TRACE_POOL_OPS = {
     .free = traceFree,
     .get_last_allocation_error = traceGetLastStatus,
     .get_name = traceGetName,
+    .ext_trim_memory = traceTrimMemory,
 };