topo, funk: change Funk workspace use unlocked pages

.github/workflows/backtest.yml

@@ -43,3 +43,8 @@ jobs:
           sudo prlimit --pid=$$ --nofile=1048576
           sudo prlimit --pid=$$ --memlock=unlimited
           DUMP_DIR=../dump make run-runtime-backtest
+
+      - name: fini
+        if: always()
+        run: |
+          sudo $OBJDIR/bin/firedancer-dev configure fini all --config ../dump/mainnet-308392063-v2.3.0_backtest.toml

contrib/offline-replay/offline_replay.toml

@@ -40,3 +40,5 @@
 [paths]
     identity_key = "{identity_key_path}"
     vote_account = "{vote_account_path}"
+[hugetlbfs]
+    mount_path = "/data/firedancer/mnt"

src/app/firedancer-dev/main.c

@@ -54,11 +54,13 @@ extern configure_stage_t fd_cfg_stage_kill;
 extern configure_stage_t fd_cfg_stage_netns;
 extern configure_stage_t fd_cfg_stage_genesis;
 extern configure_stage_t fd_cfg_stage_keys;
+extern configure_stage_t fd_cfg_stage_normalpage;
 
 configure_stage_t * STAGES[] = {
   &fd_cfg_stage_kill,
   &fd_cfg_stage_netns,
   &fd_cfg_stage_hugetlbfs,
+  &fd_cfg_stage_normalpage,
   &fd_cfg_stage_sysctl,
   &fd_cfg_stage_hyperthreads,
   &fd_cfg_stage_ethtool_channels,

src/app/firedancer/topology.c

@@ -112,9 +112,10 @@ setup_topo_funk( fd_topo_t *  topo,
   ulong wksp_idx = fd_topo_find_wksp( topo, wksp_name );
   FD_TEST( wksp_idx!=ULONG_MAX );
   fd_topo_wksp_t * wksp = &topo->workspaces[ wksp_idx ];
-  ulong part_max = fd_wksp_part_max_est( funk_footprint, 1U<<14U );
+  ulong part_max = fd_wksp_part_max_est( funk_footprint+(heap_size_gib*(1UL<<30)), 1U<<14U );
   if( FD_UNLIKELY( !part_max ) ) FD_LOG_ERR(( "fd_wksp_part_max_est(%lu,16KiB) failed", funk_footprint ));
   wksp->part_max += part_max;
+  wksp->is_locked = 0;
 
   return obj;
 }

src/app/shared/commands/run/run.c

@@ -477,6 +477,9 @@ workspace_path( config_t const *       config,
     case FD_SHMEM_GIGANTIC_PAGE_SZ:
       mount_path = config->hugetlbfs.gigantic_page_mount_path;
       break;
+    case FD_SHMEM_NORMAL_PAGE_SZ:
+      mount_path = config->hugetlbfs.normal_page_mount_path;
+      break;
     default:
       FD_LOG_ERR(( "invalid page size %lu", wksp->page_sz ));
   }
@@ -516,6 +519,7 @@ warn_unknown_files( config_t const * config,
     int known_file = 0;
     for( ulong i=0UL; i<config->topo.wksp_cnt; i++ ) {
       fd_topo_wksp_t const * wksp = &config->topo.workspaces[ i ];
+      if( !wksp->is_locked ) continue;
 
       char expected_path[ PATH_MAX ];
       workspace_path( config, wksp, expected_path );

src/app/shared/fd_config.c

@@ -292,6 +292,11 @@ fd_config_fill( fd_config_t * config,
     NULL,
     "%s/.huge",
     config->hugetlbfs.mount_path ) );
+  FD_TEST( fd_cstr_printf_check( config->hugetlbfs.normal_page_mount_path,
+    sizeof(config->hugetlbfs.normal_page_mount_path),
+    NULL,
+    "%s/.normal",
+    config->hugetlbfs.mount_path ) );
 
   ulong max_page_sz = fd_cstr_to_shmem_page_sz( config->hugetlbfs.max_page_size );
   if( FD_UNLIKELY( max_page_sz!=FD_SHMEM_HUGE_PAGE_SZ && max_page_sz!=FD_SHMEM_GIGANTIC_PAGE_SZ ) ) FD_LOG_ERR(( "[hugetlbfs.max_page_size] must be \"huge\" or \"gigantic\"" ));

src/app/shared/fd_config.h

@@ -253,6 +253,7 @@ struct fd_config {
   struct {
     char  gigantic_page_mount_path[ PATH_MAX ];
     char  huge_page_mount_path[ PATH_MAX ];
+    char  normal_page_mount_path[ PATH_MAX ];
     char  mount_path[ PATH_MAX ];
     char  max_page_size[ 16 ];
     ulong gigantic_page_threshold_mib;

src/app/shared_dev/Local.mk

@@ -30,6 +30,7 @@ $(call add-objs,commands/configure/kill,fddev_shared)
 ifdef FD_HAS_INT128
 $(call add-objs,commands/configure/genesis,fddev_shared)
 endif
+$(call add-objs,commands/configure/normalpage,fddev_shared)
 
 endif
 endif

src/app/shared_dev/commands/configure/normalpage.c

@@ -0,0 +1,103 @@
+/* This stage configures the normal pages directory, which is where the
+   files backing memory-mapped unlocked workspaces should be stored.
+
+   The files backing these workspaces are stored in the normal pages
+   directory configured by this command, and follow the normal workspace
+   shmem file naming convention. */
+
+#include "../../../shared/commands/configure/configure.h"
+
+#include "../../../platform/fd_file_util.h"
+
+#include <unistd.h>
+#include <errno.h>
+#include <stdio.h>
+#include <stdlib.h> /* strtoul */
+#include <dirent.h>
+#include <sys/stat.h>
+#include <sys/mount.h>
+#include <linux/capability.h>
+
+static void
+init( config_t const * config ) {
+  char const * path = config->hugetlbfs.normal_page_mount_path;
+
+  FD_LOG_NOTICE(( "RUN: `mkdir -p %s`", path ));
+  if( FD_UNLIKELY( -1==fd_file_util_mkdir_all( path, config->uid, config->gid ) ) ) {
+    FD_LOG_ERR(( "could not create normal page directory `%s` (%i-%s)", path, errno, fd_io_strerror( errno ) ));
+  }
+  if( FD_UNLIKELY( chown( path, config->uid, config->gid ) ) )
+    FD_LOG_ERR(( "chown of normal page directory `%s` failed (%i-%s)", path, errno, fd_io_strerror( errno ) ));
+  if( FD_UNLIKELY( chmod( config->hugetlbfs.normal_page_mount_path, S_IRUSR | S_IWUSR | S_IXUSR ) ) )
+    FD_LOG_ERR(( "chmod of normal page directory `%s` failed (%i-%s)", config->hugetlbfs.normal_page_mount_path, errno, fd_io_strerror( errno ) ));
+}
+
+static int
+is_mountpoint( char const * directory ) {
+  struct stat st;
+  int result = stat( directory, &st );
+  if( FD_UNLIKELY( -1==result && errno==ENOENT ) ) return 0;
+  if( FD_UNLIKELY( -1==result ) ) FD_LOG_ERR(( "failed to stat `%s` (%i-%s)", directory, errno, fd_io_strerror( errno ) ));
+
+  char parent_path[ PATH_MAX+4UL ];
+  FD_TEST( fd_cstr_printf_check( parent_path, sizeof(parent_path), NULL, "%s/..", directory ) );
+
+  struct stat st_parent;
+  result = stat( parent_path, &st_parent );
+  if( FD_UNLIKELY( -1==result && errno==ENOENT ) ) return 0;
+  if( FD_UNLIKELY( -1==result ) ) FD_LOG_ERR(( "failed to stat `%s` (%i-%s)", parent_path, errno, fd_io_strerror( errno ) ));
+
+  return st_parent.st_dev!=st.st_dev;
+}
+
+static void
+fini( config_t const * config,
+      int              pre_init ) {
+  (void)pre_init;
+  char const * path = config->hugetlbfs.normal_page_mount_path;
+
+  /* fd_shmem_cfg mounts a tmpfs filesystem onto the .normal directory
+     sometimes, which is the expected way to manage normal pages, but
+     not what is done by firedancer-dev to support a special temporary
+     funk use case where normal pages are backed by disk.  To prevent
+     fighting with the other script, we unmount the normal pages if they
+     have been mounted. */
+
+  if( FD_UNLIKELY( is_mountpoint( path ) ) ) {
+    FD_LOG_NOTICE(( "RUN: `umount %s`", path ));
+    if( FD_UNLIKELY( -1==umount( path ) && errno!=EINVAL && errno!=ENOENT ) )
+      FD_LOG_ERR(( "error unmounting normal pages directory at `%s` (%i-%s)", path, errno, fd_io_strerror( errno ) ));
+  }
+
+  FD_LOG_NOTICE(( "RUN: `rm -rf %s`", path ));
+  if( FD_UNLIKELY( -1==fd_file_util_rmtree( path, 1 ) ) ) FD_LOG_ERR(( "error removing normal pages directory at `%s` (%i-%s)", path, errno, fd_io_strerror( errno ) ));
+}
+
+
+static configure_result_t
+check( config_t const * config ) {
+  char const * path = config->hugetlbfs.normal_page_mount_path;
+
+  struct stat st;
+  int result = stat( path, &st );
+  if( FD_UNLIKELY( result && errno!=ENOENT ) )
+    PARTIALLY_CONFIGURED( "failed to stat `%s` (%i-%s)", path, errno, fd_io_strerror( errno ) );
+
+  if( FD_UNLIKELY( is_mountpoint( path ) ) )
+    PARTIALLY_CONFIGURED( "normal pages directory `%s` is a mountpoint", path );
+
+  if( FD_UNLIKELY( result ) )
+    NOT_CONFIGURED( "normal pages directory `%s` does not exist", path );
+
+  CHECK( check_dir( path, config->uid, config->gid, S_IFDIR | S_IRUSR | S_IWUSR | S_IXUSR ) );
+
+  CONFIGURE_OK();
+}
+
+configure_stage_t fd_cfg_stage_normalpage = {
+  .name            = "normalpage",
+  .always_recreate = 0,
+  .init            = init,
+  .fini            = fini,
+  .check           = check,
+};

src/disco/topo/fd_topo.c

@@ -17,7 +17,7 @@ fd_topo_join_workspace( fd_topo_t *      topo,
   char name[ PATH_MAX ];
   FD_TEST( fd_cstr_printf_check( name, PATH_MAX, NULL, "%s_%s.wksp", topo->app_name, wksp->name ) );
 
-  wksp->wksp = fd_wksp_join( fd_shmem_join( name, mode, NULL, NULL, NULL ) );
+  wksp->wksp = fd_wksp_join( fd_shmem_join( name, mode, NULL, NULL, NULL, wksp->is_locked ) );
   if( FD_UNLIKELY( !wksp->wksp ) ) FD_LOG_ERR(( "fd_wksp_join failed" ));
 }
 
@@ -82,15 +82,17 @@ fd_topo_create_workspace( fd_topo_t *      topo,
   ulong sub_cpu_idx [ 1 ] = { fd_shmem_cpu_idx( wksp->numa_idx ) };
 
   int err;
-  if( FD_UNLIKELY( update_existing ) ) {
+  if( FD_UNLIKELY( !wksp->is_locked ) ) {
+    err = fd_shmem_create_multi_unlocked( name, wksp->page_sz, wksp->page_cnt, S_IRUSR | S_IWUSR ); /* logs details */
+  } else if( FD_UNLIKELY( update_existing ) ) {
     err = fd_shmem_update_multi( name, wksp->page_sz, 1, sub_page_cnt, sub_cpu_idx, S_IRUSR | S_IWUSR ); /* logs details */
   } else {
     err = fd_shmem_create_multi( name, wksp->page_sz, 1, sub_page_cnt, sub_cpu_idx, S_IRUSR | S_IWUSR ); /* logs details */
   }
   if( FD_UNLIKELY( err && errno==ENOMEM ) ) return -1;
   else if( FD_UNLIKELY( err ) ) FD_LOG_ERR(( "fd_shmem_create_multi failed" ));
 
-  void * shmem = fd_shmem_join( name, FD_SHMEM_JOIN_MODE_READ_WRITE, NULL, NULL, NULL ); /* logs details */
+  void * shmem = fd_shmem_join( name, FD_SHMEM_JOIN_MODE_READ_WRITE, NULL, NULL, NULL, wksp->is_locked ); /* logs details */
 
   void * wkspmem = fd_wksp_new( shmem, name, 0U, wksp->part_max, wksp->total_footprint ); /* logs details */
   if( FD_UNLIKELY( !wkspmem ) ) FD_LOG_ERR(( "fd_wksp_new failed" ));
@@ -223,6 +225,8 @@ fd_topo_mlock_max_tile1( fd_topo_t const *      topo,
   ulong tile_mem = 0UL;
 
   for( ulong i=0UL; i<topo->wksp_cnt; i++ ) {
+    if( FD_UNLIKELY( !topo->workspaces[ i ].is_locked ) ) continue;
+
     if( FD_UNLIKELY( -1!=tile_needs_wksp( topo, tile, i ) ) )
       tile_mem += topo->workspaces[ i ].page_cnt * topo->workspaces[ i ].page_sz;
   }
@@ -296,6 +300,7 @@ FD_FN_PURE ulong
 fd_topo_mlock( fd_topo_t const * topo ) {
   ulong result = 0UL;
   for( ulong i=0UL; i<topo->wksp_cnt; i++ ) {
+    if( FD_UNLIKELY( !topo->workspaces[ i ].is_locked ) ) continue;
     result += topo->workspaces[ i ].page_cnt * topo->workspaces[ i ].page_sz;
   }
   return result;
@@ -385,7 +390,7 @@ fd_topo_print_log( int         stdout,
 
     char size[ 24 ];
     fd_topo_mem_sz_string( wksp->page_sz * wksp->page_cnt, size );
-    PRINT( "  %2lu (%7s): %12s  page_cnt=%3lu  page_sz=%-8s  numa_idx=%-2lu  footprint=%10lu  loose=%lu\n", i, size, wksp->name, wksp->page_cnt, fd_shmem_page_sz_to_cstr( wksp->page_sz ), wksp->numa_idx, wksp->known_footprint, wksp->total_footprint - wksp->known_footprint );
+    PRINT( "  %2lu (%7s): %12s  page_cnt=%3lu  page_sz=%-8s  numa_idx=%-2lu  footprint=%10lu  loose=%10lu  is_locked=%d\n", i, size, wksp->name, wksp->page_cnt, fd_shmem_page_sz_to_cstr( wksp->page_sz ), wksp->numa_idx, wksp->known_footprint, wksp->total_footprint - wksp->known_footprint, wksp->is_locked );
   }
 
   PRINT( "\nOBJECTS\n" );

src/disco/topo/fd_topo.h

@@ -35,6 +35,8 @@ typedef struct {
 
   ulong numa_idx;     /* The index of the NUMA node on the system that this workspace should be allocated from. */
 
+  int   is_locked;    /* If the workspace should use pages locked and pinned to a specific numa node. */
+
   /* Computed fields.  These are not supplied as configuration but calculated as needed. */
   struct {
     ulong page_sz;  /* The size of the pages that this workspace is backed by.  One of FD_PAGE_SIZE_*. */

src/disco/topo/fd_topo_run.c

@@ -220,7 +220,7 @@ fd_topo_tile_stack_join( char const * app_name,
   char name[ PATH_MAX ];
   FD_TEST( fd_cstr_printf_check( name, PATH_MAX, NULL, "%s_stack_%s%lu", app_name, tile_name, tile_kind_id ) );
 
-  uchar * stack = fd_shmem_join( name, FD_SHMEM_JOIN_MODE_READ_WRITE, NULL, NULL, NULL );
+  uchar * stack = fd_shmem_join( name, FD_SHMEM_JOIN_MODE_READ_WRITE, NULL, NULL, NULL, 1 );
   if( FD_UNLIKELY( !stack ) ) FD_LOG_ERR(( "fd_shmem_join failed" ));
 
   /* Make space for guard lo and guard hi */

src/disco/topo/fd_topob.c

@@ -41,6 +41,7 @@ fd_topob_wksp( fd_topo_t *  topo,
   fd_topo_wksp_t * wksp = &topo->workspaces[ topo->wksp_cnt ];
   strncpy( wksp->name, name, sizeof(wksp->name) );
   wksp->id = topo->wksp_cnt;
+  wksp->is_locked = 1;
   topo->wksp_cnt++;
 }
 
@@ -647,6 +648,11 @@ fd_topob_finish( fd_topo_t *                topo,
     if( total_wksp_footprint < topo->gigantic_page_threshold ) page_sz = FD_SHMEM_HUGE_PAGE_SZ;
     if( FD_UNLIKELY( page_sz!=FD_SHMEM_HUGE_PAGE_SZ && page_sz!=FD_SHMEM_GIGANTIC_PAGE_SZ ) ) FD_LOG_ERR(( "invalid page_sz" ));
 
+    /* If the workspace is not locked, we can't use huge pages. */
+    if( FD_UNLIKELY( !wksp->is_locked ) ) {
+      page_sz = FD_SHMEM_NORMAL_PAGE_SZ;
+    }
+
     ulong wksp_aligned_footprint = fd_ulong_align_up( total_wksp_footprint, page_sz );
 
     /* Give any leftover space in the underlying shared memory to the

src/discof/restore/fd_snapin_tile.c

@@ -228,6 +228,7 @@ scratch_footprint( fd_topo_tile_t const * tile ) {
   l = FD_LAYOUT_APPEND( l, alignof(fd_snapin_tile_t),  sizeof(fd_snapin_tile_t)       );
   l = FD_LAYOUT_APPEND( l, fd_snapshot_parser_align(), fd_snapshot_parser_footprint() );
   l = FD_LAYOUT_APPEND( l, fd_scratch_smem_align(),    fd_scratch_smem_footprint( FD_SNAPIN_SCRATCH_MAX ) );
+  l = FD_LAYOUT_APPEND( l, fd_scratch_fmem_align(),    fd_scratch_fmem_footprint( FD_SNAPIN_SCRATCH_DEPTH ) );
   return FD_LAYOUT_FINI( l, alignof(fd_snapin_tile_t) );
 }
 

src/flamenco/runtime/tests/run_backtest_ci.sh

@@ -1,20 +1,21 @@
 #!/bin/bash
 set -e
+HUGE_TLBFS_MOUNT_PATH="/data/svc_firedancer/mnt/.fd"
 
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-308392063-v2.3.0 -s snapshot-308392062-FDuB6CFKod14xGRGmdiRpQx2uaKyp3GDkyai2Ba7eH8d.tar.zst -y 5 -m 2000000 -e 308392090 -c 2.3.0
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l devnet-350814254-v2.3.0 -s snapshot-350814253-G5P3eNtkWUGkZ8b871wvf6d78wYxBJp637PCWJuQByZa.tar.zst -y 3 -m 2000000 -e 350814284 -c 2.3.0
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l testnet-281546597-v2.3.0 -s snapshot-281546592-EbVCTHZzWyya8tJ4CaSvgMgZVpzDVzmifaWxp4Aiet5A.tar.zst -y 3 -m 2000000 -e 281546597 -c 2.3.0
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-324823213-v2.3.0 -s snapshot-324823212-sk3zF16dk7gEsgLf1mGDhj6qRp87kFjJdEWZmhr4Kju.tar.zst -y 4 -m 2000000 -e 324823214 -c 2.3.0
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-325467935-v2.3.0 -s snapshot-325467934-7VZstGKYwU8Y7A952RQjfCKD5Qdbu9t8D84e5h1mHXA6.tar.zst -y 4 -m 2000000 -e 325467936 -c 2.3.0
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l testnet-283927487-v2.3.0 -s snapshot-283927486-7gCbg5g4BnD9SkQUjpHvhepWsQTpo2WaZaA5bhcNBMhG.tar.zst -y 3 -m 2000000 -e 283927497 -c 2.3.0
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l testnet-321168308-v2.3.0 -s snapshot-321168307-DecxjCHDsiQgbqZPHptgz1tystKi3QmV3Rd3QEgcxs2W.tar.zst -y 3 -m 2000000 -e 321168308 -c 2.3.0
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-327324660-v2.3.0 -s snapshot-327324659-RFg6ep2QLa8uzAkZku6xMnZQYVaUN9jwkVEfsnWYB25.tar.zst -y 4 -m 2000000 -e 327324660 -c 2.3.0
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l devnet-370199634-v2.3.0 -s snapshot-370199633-D8mrtzcNV8iNVarHs4mi55QHrCfmzDScYL8BBYXUHAwW.tar.zst -y 3 -m 200000 -e 370199634 -c 2.3.0
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l devnet-378683870-v2.3.0 -s snapshot-378683869-8tgAVwwBiDFETa2pWJPgzJZg5csvpwj51t1EFfrnVKCh.tar.zst -y 3 -m 2000000 -e 378683872 -c 2.3.0
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-330219081-v2.3.0 -s snapshot-330219080-6fafBFhTgbHxNiE7wtJTF1ERUyVyTCJz2d52tMtrfG2Z.tar.zst -y 4 -m 2000000 -e 330219082 -c 2.3.0
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l devnet-372721907-v2.3.0 -s snapshot-372721906-FtUjok2JfLPwJCRVcioV12M8FWbbJaC91XEJzm4eZy53.tar.zst -y 3 -m 2000000 -e 372721910 -c 2.3.0
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-331691646-v2.3.0 -s snapshot-331691639-3NmZ4rd7nHfn6tuS4E5gUfAwWMoBAQ6K1yntNkyhPbrb.tar.zst -y 4 -m 2000000 -e 331691647 -c 2.3.0
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l testnet-336218682-v2.3.0 -s snapshot-336218681-BDsErdHkqa5iQGCNkSvQpeon8GLFsgeNEkckrMKboJ4N.tar.zst -y 5 -m 2000000 -e 336218683 -c 2.3.0
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l testnet-340269866-v2.3.0 -s snapshot-340269865-7kiU4fCQMuf97vs3827n1vn1oScnkYYvvQ1sGGa1QKVE.tar.zst -y 5 -m 2000000 -e 340269872 -c 2.3.0
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l devnet-390056400-v2.3.0 -s snapshot-390056399-6JQvYEDzQqcrjF4EoxkWUMuskm9aGoGRG94twEqXuuri.tar.zst -y 10 -m 2000000 -e 390056406 -c 2.3.0
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-254462437-v2.3.0 -s snapshot-254462437-9HqBi19BJJRZfHeBS3ZpkeP9B5SAxBxz6Kwug29yLHac.tar.zst -y 16 -m 10000000 -e 254462598 -c 2.3.0
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-308392063-v2.3.0 -s snapshot-308392062-FDuB6CFKod14xGRGmdiRpQx2uaKyp3GDkyai2Ba7eH8d.tar.zst -y 5 -m 2000000 -e 308392090 -c 2.3.0 -h $HUGE_TLBFS_MOUNT_PATH
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l devnet-350814254-v2.3.0 -s snapshot-350814253-G5P3eNtkWUGkZ8b871wvf6d78wYxBJp637PCWJuQByZa.tar.zst -y 3 -m 2000000 -e 350814284 -c 2.3.0 -h $HUGE_TLBFS_MOUNT_PATH
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l testnet-281546597-v2.3.0 -s snapshot-281546592-EbVCTHZzWyya8tJ4CaSvgMgZVpzDVzmifaWxp4Aiet5A.tar.zst -y 3 -m 2000000 -e 281546597 -c 2.3.0 -h $HUGE_TLBFS_MOUNT_PATH
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-324823213-v2.3.0 -s snapshot-324823212-sk3zF16dk7gEsgLf1mGDhj6qRp87kFjJdEWZmhr4Kju.tar.zst -y 4 -m 2000000 -e 324823214 -c 2.3.0 -h $HUGE_TLBFS_MOUNT_PATH
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-325467935-v2.3.0 -s snapshot-325467934-7VZstGKYwU8Y7A952RQjfCKD5Qdbu9t8D84e5h1mHXA6.tar.zst -y 4 -m 2000000 -e 325467936 -c 2.3.0 -h $HUGE_TLBFS_MOUNT_PATH
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l testnet-283927487-v2.3.0 -s snapshot-283927486-7gCbg5g4BnD9SkQUjpHvhepWsQTpo2WaZaA5bhcNBMhG.tar.zst -y 3 -m 2000000 -e 283927497 -c 2.3.0 -h $HUGE_TLBFS_MOUNT_PATH
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l testnet-321168308-v2.3.0 -s snapshot-321168307-DecxjCHDsiQgbqZPHptgz1tystKi3QmV3Rd3QEgcxs2W.tar.zst -y 3 -m 2000000 -e 321168308 -c 2.3.0 -h $HUGE_TLBFS_MOUNT_PATH
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-327324660-v2.3.0 -s snapshot-327324659-RFg6ep2QLa8uzAkZku6xMnZQYVaUN9jwkVEfsnWYB25.tar.zst -y 4 -m 2000000 -e 327324660 -c 2.3.0 -h $HUGE_TLBFS_MOUNT_PATH
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l devnet-370199634-v2.3.0 -s snapshot-370199633-D8mrtzcNV8iNVarHs4mi55QHrCfmzDScYL8BBYXUHAwW.tar.zst -y 3 -m 200000 -e 370199634 -c 2.3.0 -h $HUGE_TLBFS_MOUNT_PATH
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l devnet-378683870-v2.3.0 -s snapshot-378683869-8tgAVwwBiDFETa2pWJPgzJZg5csvpwj51t1EFfrnVKCh.tar.zst -y 3 -m 2000000 -e 378683872 -c 2.3.0 -h $HUGE_TLBFS_MOUNT_PATH
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-330219081-v2.3.0 -s snapshot-330219080-6fafBFhTgbHxNiE7wtJTF1ERUyVyTCJz2d52tMtrfG2Z.tar.zst -y 4 -m 2000000 -e 330219082 -c 2.3.0 -h $HUGE_TLBFS_MOUNT_PATH
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l devnet-372721907-v2.3.0 -s snapshot-372721906-FtUjok2JfLPwJCRVcioV12M8FWbbJaC91XEJzm4eZy53.tar.zst -y 3 -m 2000000 -e 372721910 -c 2.3.0 -h $HUGE_TLBFS_MOUNT_PATH
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-331691646-v2.3.0 -s snapshot-331691639-3NmZ4rd7nHfn6tuS4E5gUfAwWMoBAQ6K1yntNkyhPbrb.tar.zst -y 4 -m 2000000 -e 331691647 -c 2.3.0 -h $HUGE_TLBFS_MOUNT_PATH
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l testnet-336218682-v2.3.0 -s snapshot-336218681-BDsErdHkqa5iQGCNkSvQpeon8GLFsgeNEkckrMKboJ4N.tar.zst -y 5 -m 2000000 -e 336218683 -c 2.3.0 -h $HUGE_TLBFS_MOUNT_PATH
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l testnet-340269866-v2.3.0 -s snapshot-340269865-7kiU4fCQMuf97vs3827n1vn1oScnkYYvvQ1sGGa1QKVE.tar.zst -y 5 -m 2000000 -e 340269872 -c 2.3.0 -h $HUGE_TLBFS_MOUNT_PATH
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l devnet-390056400-v2.3.0 -s snapshot-390056399-6JQvYEDzQqcrjF4EoxkWUMuskm9aGoGRG94twEqXuuri.tar.zst -y 10 -m 2000000 -e 390056406 -c 2.3.0 -h $HUGE_TLBFS_MOUNT_PATH
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-254462437-v2.3.0 -s snapshot-254462437-9HqBi19BJJRZfHeBS3ZpkeP9B5SAxBxz6Kwug29yLHac.tar.zst -y 16 -m 10000000 -e 254462598 -c 2.3.0 -h $HUGE_TLBFS_MOUNT_PATH