funk, topo, util: make Funk workspace use unpinned pages

Author: topointon-jump

.github/workflows/backtest.yml

@@ -43,3 +43,8 @@ jobs:
           sudo prlimit --pid=$$ --nofile=1048576
           sudo prlimit --pid=$$ --memlock=unlimited
           DUMP_DIR=../dump make run-runtime-backtest
+
+      - name: fini
+        if: always()
+        run: |
+          sudo $OBJDIR/bin/firedancer-dev configure fini all

contrib/offline-replay/offline_replay.toml

@@ -40,3 +40,5 @@
 [paths]
     identity_key = "{identity_key_path}"
     vote_account = "{vote_account_path}"
+[hugetlbfs]
+    mount_path = "/data/firedancer/mnt"

src/app/firedancer-dev/main.c

@@ -54,11 +54,13 @@ extern configure_stage_t fd_cfg_stage_kill;
 extern configure_stage_t fd_cfg_stage_netns;
 extern configure_stage_t fd_cfg_stage_genesis;
 extern configure_stage_t fd_cfg_stage_keys;
+extern configure_stage_t fd_cfg_stage_normalpage;
 
 configure_stage_t * STAGES[] = {
   &fd_cfg_stage_kill,
   &fd_cfg_stage_netns,
   &fd_cfg_stage_hugetlbfs,
+  &fd_cfg_stage_normalpage,
   &fd_cfg_stage_sysctl,
   &fd_cfg_stage_hyperthreads,
   &fd_cfg_stage_ethtool_channels,

src/app/firedancer/topology.c

@@ -112,9 +112,10 @@ setup_topo_funk( fd_topo_t *  topo,
   ulong wksp_idx = fd_topo_find_wksp( topo, wksp_name );
   FD_TEST( wksp_idx!=ULONG_MAX );
   fd_topo_wksp_t * wksp = &topo->workspaces[ wksp_idx ];
-  ulong part_max = fd_wksp_part_max_est( funk_footprint, 1U<<14U );
+  ulong part_max = fd_wksp_part_max_est( funk_footprint+(heap_size_gib*(1UL<<30)), 1U<<14U );
   if( FD_UNLIKELY( !part_max ) ) FD_LOG_ERR(( "fd_wksp_part_max_est(%lu,16KiB) failed", funk_footprint ));
   wksp->part_max += part_max;
+  wksp->is_locked = 0;
 
   return obj;
 }

src/app/shared/commands/run/run.c

@@ -477,6 +477,9 @@ workspace_path( config_t const *       config,
     case FD_SHMEM_GIGANTIC_PAGE_SZ:
       mount_path = config->hugetlbfs.gigantic_page_mount_path;
       break;
+    case FD_SHMEM_NORMAL_PAGE_SZ:
+      mount_path = config->hugetlbfs.normal_page_mount_path;
+      break;
     default:
       FD_LOG_ERR(( "invalid page size %lu", wksp->page_sz ));
   }
@@ -516,6 +519,7 @@ warn_unknown_files( config_t const * config,
     int known_file = 0;
     for( ulong i=0UL; i<config->topo.wksp_cnt; i++ ) {
       fd_topo_wksp_t const * wksp = &config->topo.workspaces[ i ];
+      if( !wksp->is_locked ) continue;
 
       char expected_path[ PATH_MAX ];
       workspace_path( config, wksp, expected_path );

src/app/shared/fd_config.c

@@ -292,6 +292,11 @@ fd_config_fill( fd_config_t * config,
     NULL,
     "%s/.huge",
     config->hugetlbfs.mount_path ) );
+  FD_TEST( fd_cstr_printf_check( config->hugetlbfs.normal_page_mount_path,
+    sizeof(config->hugetlbfs.normal_page_mount_path),
+    NULL,
+    "%s/.normal",
+    config->hugetlbfs.mount_path ) );
 
   ulong max_page_sz = fd_cstr_to_shmem_page_sz( config->hugetlbfs.max_page_size );
   if( FD_UNLIKELY( max_page_sz!=FD_SHMEM_HUGE_PAGE_SZ && max_page_sz!=FD_SHMEM_GIGANTIC_PAGE_SZ ) ) FD_LOG_ERR(( "[hugetlbfs.max_page_size] must be \"huge\" or \"gigantic\"" ));

src/app/shared/fd_config.h

@@ -253,6 +253,7 @@ struct fd_config {
   struct {
     char  gigantic_page_mount_path[ PATH_MAX ];
     char  huge_page_mount_path[ PATH_MAX ];
+    char  normal_page_mount_path[ PATH_MAX ];
     char  mount_path[ PATH_MAX ];
     char  max_page_size[ 16 ];
     ulong gigantic_page_threshold_mib;

src/app/shared_dev/Local.mk

@@ -30,6 +30,7 @@ $(call add-objs,commands/configure/kill,fddev_shared)
 ifdef FD_HAS_INT128
 $(call add-objs,commands/configure/genesis,fddev_shared)
 endif
+$(call add-objs,commands/configure/normalpage,fddev_shared)
 
 endif
 endif

src/app/shared_dev/commands/configure/normalpage.c

@@ -0,0 +1,103 @@
+/* This stage configures the normal pages directory, which is where the
+   files backing memory-mapped unlocked workspaces should be stored.
+
+   The files backing these workspaces are stored in the normal pages
+   directory configured by this command, and follow the normal workspace
+   shmem file naming convention. */
+
+#include "../../../shared/commands/configure/configure.h"
+
+#include "../../../platform/fd_file_util.h"
+
+#include <unistd.h>
+#include <errno.h>
+#include <stdio.h>
+#include <stdlib.h> /* strtoul */
+#include <dirent.h>
+#include <sys/stat.h>
+#include <sys/mount.h>
+#include <linux/capability.h>
+
+static void
+init( config_t const * config ) {
+  char const * path = config->hugetlbfs.normal_page_mount_path;
+
+  FD_LOG_NOTICE(( "RUN: `mkdir -p %s`", path ));
+  if( FD_UNLIKELY( -1==fd_file_util_mkdir_all( path, config->uid, config->gid ) ) ) {
+    FD_LOG_ERR(( "could not create normal page directory `%s` (%i-%s)", path, errno, fd_io_strerror( errno ) ));
+  }
+  if( FD_UNLIKELY( chown( path, config->uid, config->gid ) ) )
+    FD_LOG_ERR(( "chown of normal page directory `%s` failed (%i-%s)", path, errno, fd_io_strerror( errno ) ));
+  if( FD_UNLIKELY( chmod( config->hugetlbfs.normal_page_mount_path, S_IRUSR | S_IWUSR | S_IXUSR ) ) )
+    FD_LOG_ERR(( "chmod of normal page directory `%s` failed (%i-%s)", config->hugetlbfs.normal_page_mount_path, errno, fd_io_strerror( errno ) ));
+}
+
+static int
+is_mountpoint( char const * directory ) {
+  struct stat st;
+  int result = stat( directory, &st );
+  if( FD_UNLIKELY( -1==result && errno==ENOENT ) ) return 0;
+  if( FD_UNLIKELY( -1==result ) ) FD_LOG_ERR(( "failed to stat `%s` (%i-%s)", directory, errno, fd_io_strerror( errno ) ));
+
+  char parent_path[ PATH_MAX+4UL ];
+  FD_TEST( fd_cstr_printf_check( parent_path, sizeof(parent_path), NULL, "%s/..", directory ) );
+
+  struct stat st_parent;
+  result = stat( parent_path, &st_parent );
+  if( FD_UNLIKELY( -1==result && errno==ENOENT ) ) return 0;
+  if( FD_UNLIKELY( -1==result ) ) FD_LOG_ERR(( "failed to stat `%s` (%i-%s)", parent_path, errno, fd_io_strerror( errno ) ));
+
+  return st_parent.st_dev!=st.st_dev;
+}
+
+static void
+fini( config_t const * config,
+      int              pre_init ) {
+  (void)pre_init;
+  char const * path = config->hugetlbfs.normal_page_mount_path;
+
+  /* fd_shmem_cfg mounts a tmpfs filesystem onto the .normal directory
+     sometimes, which is the expected way to manage normal pages, but
+     not what is done by firedancer-dev to support a special temporary
+     funk use case where normal pages are backed by disk.  To prevent
+     fighting with the other script, we unmount the normal pages if they
+     have been mounted. */
+
+  if( FD_UNLIKELY( is_mountpoint( path ) ) ) {
+    FD_LOG_NOTICE(( "RUN: `umount %s`", path ));
+    if( FD_UNLIKELY( -1==umount( path ) && errno!=EINVAL && errno!=ENOENT ) )
+      FD_LOG_ERR(( "error unmounting normal pages directory at `%s` (%i-%s)", path, errno, fd_io_strerror( errno ) ));
+  }
+
+  FD_LOG_NOTICE(( "RUN: `rm -rf %s`", path ));
+  if( FD_UNLIKELY( -1==fd_file_util_rmtree( path, 1 ) ) ) FD_LOG_ERR(( "error removing normal pages directory at `%s` (%i-%s)", path, errno, fd_io_strerror( errno ) ));
+}
+
+
+static configure_result_t
+check( config_t const * config ) {
+  char const * path = config->hugetlbfs.normal_page_mount_path;
+
+  struct stat st;
+  int result = stat( path, &st );
+  if( FD_UNLIKELY( result && errno!=ENOENT ) )
+    PARTIALLY_CONFIGURED( "failed to stat `%s` (%i-%s)", path, errno, fd_io_strerror( errno ) );
+
+  if( FD_UNLIKELY( is_mountpoint( path ) ) )
+    PARTIALLY_CONFIGURED( "normal pages directory `%s` is a mountpoint", path );
+
+  if( FD_UNLIKELY( result ) )
+    NOT_CONFIGURED( "normal pages directory `%s` does not exist", path );
+
+  CHECK( check_dir( path, config->uid, config->gid, S_IFDIR | S_IRUSR | S_IWUSR | S_IXUSR ) );
+
+  CONFIGURE_OK();
+}
+
+configure_stage_t fd_cfg_stage_normalpage = {
+  .name            = "normalpage",
+  .always_recreate = 0,
+  .init            = init,
+  .fini            = fini,
+  .check           = check,
+};

src/disco/topo/fd_topo.c

@@ -17,7 +17,7 @@ fd_topo_join_workspace( fd_topo_t *      topo,
   char name[ PATH_MAX ];
   FD_TEST( fd_cstr_printf_check( name, PATH_MAX, NULL, "%s_%s.wksp", topo->app_name, wksp->name ) );
 
-  wksp->wksp = fd_wksp_join( fd_shmem_join( name, mode, NULL, NULL, NULL ) );
+  wksp->wksp = fd_wksp_join( fd_shmem_join( name, mode, NULL, NULL, NULL, wksp->is_locked ) );
   if( FD_UNLIKELY( !wksp->wksp ) ) FD_LOG_ERR(( "fd_wksp_join failed" ));
 }
 
@@ -82,15 +82,17 @@ fd_topo_create_workspace( fd_topo_t *      topo,
   ulong sub_cpu_idx [ 1 ] = { fd_shmem_cpu_idx( wksp->numa_idx ) };
 
   int err;
-  if( FD_UNLIKELY( update_existing ) ) {
+  if( FD_UNLIKELY( !wksp->is_locked ) ) {
+    err = fd_shmem_create_multi_unlocked( name, wksp->page_sz, wksp->page_cnt, S_IRUSR | S_IWUSR ); /* logs details */
+  } else if( FD_UNLIKELY( update_existing ) ) {
     err = fd_shmem_update_multi( name, wksp->page_sz, 1, sub_page_cnt, sub_cpu_idx, S_IRUSR | S_IWUSR ); /* logs details */
   } else {
     err = fd_shmem_create_multi( name, wksp->page_sz, 1, sub_page_cnt, sub_cpu_idx, S_IRUSR | S_IWUSR ); /* logs details */
   }
   if( FD_UNLIKELY( err && errno==ENOMEM ) ) return -1;
   else if( FD_UNLIKELY( err ) ) FD_LOG_ERR(( "fd_shmem_create_multi failed" ));
 
-  void * shmem = fd_shmem_join( name, FD_SHMEM_JOIN_MODE_READ_WRITE, NULL, NULL, NULL ); /* logs details */
+  void * shmem = fd_shmem_join( name, FD_SHMEM_JOIN_MODE_READ_WRITE, NULL, NULL, NULL, wksp->is_locked ); /* logs details */
 
   void * wkspmem = fd_wksp_new( shmem, name, 0U, wksp->part_max, wksp->total_footprint ); /* logs details */
   if( FD_UNLIKELY( !wkspmem ) ) FD_LOG_ERR(( "fd_wksp_new failed" ));
@@ -223,6 +225,8 @@ fd_topo_mlock_max_tile1( fd_topo_t const *      topo,
   ulong tile_mem = 0UL;
 
   for( ulong i=0UL; i<topo->wksp_cnt; i++ ) {
+    if( FD_UNLIKELY( !topo->workspaces[ i ].is_locked ) ) continue;
+
     if( FD_UNLIKELY( -1!=tile_needs_wksp( topo, tile, i ) ) )
       tile_mem += topo->workspaces[ i ].page_cnt * topo->workspaces[ i ].page_sz;
   }
@@ -296,6 +300,7 @@ FD_FN_PURE ulong
 fd_topo_mlock( fd_topo_t const * topo ) {
   ulong result = 0UL;
   for( ulong i=0UL; i<topo->wksp_cnt; i++ ) {
+    if( FD_UNLIKELY( !topo->workspaces[ i ].is_locked ) ) continue;
     result += topo->workspaces[ i ].page_cnt * topo->workspaces[ i ].page_sz;
   }
   return result;
@@ -385,7 +390,7 @@ fd_topo_print_log( int         stdout,
 
     char size[ 24 ];
     fd_topo_mem_sz_string( wksp->page_sz * wksp->page_cnt, size );
-    PRINT( "  %2lu (%7s): %12s  page_cnt=%3lu  page_sz=%-8s  numa_idx=%-2lu  footprint=%10lu  loose=%lu\n", i, size, wksp->name, wksp->page_cnt, fd_shmem_page_sz_to_cstr( wksp->page_sz ), wksp->numa_idx, wksp->known_footprint, wksp->total_footprint - wksp->known_footprint );
+    PRINT( "  %2lu (%7s): %12s  page_cnt=%3lu  page_sz=%-8s  numa_idx=%-2lu  footprint=%10lu  loose=%10lu  is_locked=%d\n", i, size, wksp->name, wksp->page_cnt, fd_shmem_page_sz_to_cstr( wksp->page_sz ), wksp->numa_idx, wksp->known_footprint, wksp->total_footprint - wksp->known_footprint, wksp->is_locked );
   }
 
   PRINT( "\nOBJECTS\n" );