From 658769f8b40ebfe5f3b8193db040d888fff6dfe9 Mon Sep 17 00:00:00 2001 From: Danny Zaken Date: Tue, 11 Mar 2025 14:42:21 +0200 Subject: [PATCH 1/3] :x - agent_blocks_reclaimer looks for deleted unreclaimed blocks. - This query can take a long time for large tables. - This index is usually small, and is only updated for deleted blocks and not for new blocks inserts Signed-off-by: Danny Zaken (cherry picked from commit 983d5f23336bbf19ea905b87b6b64ae81582a150) (cherry picked from commit f6f58a8b4869a4caf19bb9e3b76b4d94313a3b3c) From 6bde3cb9330a9f3974979b3bb27341a2bdf49a92 Mon Sep 17 00:00:00 2001 From: Danny Zaken Date: Wed, 22 May 2024 13:30:43 +0300 Subject: [PATCH 2/3] fix for list_objects * for postgres queries, there is no need to make an extra DB query, after processing the results of the first query. * in Mongo map-reduce, we could get to a situation where the query returned less than the limit, even though there are still relevant entries. * in Postgres, this is not happening, since the map_common_prefixes, iterates until it finds enough records (or until it reaches the end). * Performing the extra DB query in postgres, can cause a very long query that eventually returns 0 entries. One scenario is where we have a directory that is the last one returned in the previous query, and it has a large number of objects under it (.e.g /folder/file0 .. /folder/file9999999). when returning for the extra query, the marker that is used is 'folder/', so map_common_prefixes starts iterating from 'folder/', looking for all objects that do not start with 'folder/', but ends with a '/' (the delimiters). for postgres there is no smart way to do it, so it just going over all objects under 'folder/' and filetering them out. this can take over several minutes to complete. Signed-off-by: Danny Zaken (cherry picked from commit 219699d7c1e71ae506433c2a1365adaf9c9774e1) (cherry picked from commit d0f3ee2047f69b258845abeb26bc106edcd765b2) From c7e3af2d78d55516d6ca54ad36727776c3626194 Mon Sep 17 00:00:00 2001 From: Danny Zaken Date: Mon, 28 Apr 2025 12:19:29 +0300 Subject: [PATCH 3/3] increased the delay values of the objects_reclaimer Signed-off-by: Danny Zaken (cherry picked from commit 249c3fe4884a1da03441fa4de76b5327ec2d2a27) (cherry picked from commit 5057cbd466c437b19d6e42b26cb1b80a21f3883d) --- config.js | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/config.js b/config.js index 6c5f9cbbf3..b3f0b7fc21 100644 --- a/config.js +++ b/config.js @@ -64,7 +64,7 @@ config.BUFFERS_MEM_LIMIT_MIN = 32 * 1024 * 1024; // just some workable minimum s config.BUFFERS_MEM_LIMIT_MAX = 4 * 1024 * 1024 * 1024; config.BUFFERS_MEM_LIMIT = Math.min( config.BUFFERS_MEM_LIMIT_MAX, - Math.max(Math.floor(config.CONTAINER_MEM_LIMIT / 4), config.BUFFERS_MEM_LIMIT_MIN,) + Math.max(Math.floor(config.CONTAINER_MEM_LIMIT / 4), config.BUFFERS_MEM_LIMIT_MIN, ) ); //////////////////////// @@ -386,10 +386,10 @@ config.BUCKET_RECLAIMER_BATCH_DELAY = 100; config.BUCKET_RECLAIMER_ERROR_DELAY = 3000; config.OBJECT_RECLAIMER_ENABLED = true; -config.OBJECT_RECLAIMER_EMPTY_DELAY = 30000; +config.OBJECT_RECLAIMER_EMPTY_DELAY = 60 * 60 * 1000; // 1 hour delay config.OBJECT_RECLAIMER_BATCH_SIZE = 100; -config.OBJECT_RECLAIMER_BATCH_DELAY = 100; -config.OBJECT_RECLAIMER_ERROR_DELAY = 3000; +config.OBJECT_RECLAIMER_BATCH_DELAY = 10 * 60 * 1000; // 10 minutes delay between batches +config.OBJECT_RECLAIMER_ERROR_DELAY = 10 * 60 * 1000; // 10 minutes delay between batches; ////////////////// @@ -752,10 +752,10 @@ config.NSFS_BUF_POOL_MEM_LIMIT_S = Math.min(Math.floor(config.NSFS_MAX_MEM_SIZE_ config.NSFS_WANTED_BUFFERS_NUMBER) * config.NSFS_BUF_SIZE_S; // Semaphore size will give 90% of remainning memory to large buffer size, 10% to medium config.NSFS_BUF_POOL_MEM_LIMIT_M = range_utils.align_down((config.BUFFERS_MEM_LIMIT - - config.NSFS_BUF_POOL_MEM_LIMIT_S - config.NSFS_BUF_POOL_MEM_LIMIT_XS) * 0.1, + config.NSFS_BUF_POOL_MEM_LIMIT_S - config.NSFS_BUF_POOL_MEM_LIMIT_XS) * 0.1, config.NSFS_BUF_SIZE_M); config.NSFS_BUF_POOL_MEM_LIMIT_L = range_utils.align_down((config.BUFFERS_MEM_LIMIT - - config.NSFS_BUF_POOL_MEM_LIMIT_S - config.NSFS_BUF_POOL_MEM_LIMIT_XS) * 0.9, + config.NSFS_BUF_POOL_MEM_LIMIT_S - config.NSFS_BUF_POOL_MEM_LIMIT_XS) * 0.9, config.NSFS_BUF_SIZE_L); config.NSFS_BUF_WARMUP_SPARSE_FILE_READS = true;