WIP: make Funk workspace use unpinned pages

Author: topointon-jump

src/app/firedancer/topology.c

@@ -116,6 +116,7 @@ setup_topo_funk( fd_topo_t *  topo,
   ulong part_max = fd_wksp_part_max_est( funk_footprint, 1U<<14U );
   if( FD_UNLIKELY( !part_max ) ) FD_LOG_ERR(( "fd_wksp_part_max_est(%lu,16KiB) failed", funk_footprint ));
   wksp->part_max += part_max;
+  wksp->is_locked = 0;
 
   return obj;
 }

src/app/shared/commands/configure/hugetlbfs.c

@@ -173,6 +173,16 @@ init( config_t const * config ) {
     if( FD_UNLIKELY( chmod( mount_path[ i ], S_IRUSR | S_IWUSR | S_IXUSR ) ) )
       FD_LOG_ERR(( "chmod of hugetlbfs at `%s` failed (%i-%s)", mount_path[ i ], errno, fd_io_strerror( errno ) ));
   }
+
+  /* Create the directory for the normal pages */
+  char unpinned_pages_mnt_path[ PATH_MAX ];
+  FD_TEST( fd_cstr_printf_check(
+    unpinned_pages_mnt_path,
+    sizeof(unpinned_pages_mnt_path), NULL, "%s/.normal", config->hugetlbfs.mount_path ) );
+  FD_LOG_NOTICE(( "RUN: `mkdir -p %s`", unpinned_pages_mnt_path ));
+  if( FD_UNLIKELY( -1==fd_file_util_mkdir_all( unpinned_pages_mnt_path, config->uid, config->gid ) ) ) {
+    FD_LOG_ERR(( "could not create mount directory `%s` (%i-%s)", unpinned_pages_mnt_path, errno, fd_io_strerror( errno ) ));
+  }
 }
 
 static void
@@ -225,6 +235,32 @@ warn_mount_users( char const * mount_path ) {
   if( FD_UNLIKELY( -1==closedir( dir ) ) ) FD_LOG_ERR(( "closedir (%i-%s)", errno, fd_io_strerror( errno ) ));
 }
 
+static int
+empty_dir_top_level( const char *dir_path ) {
+  DIR *dir = opendir( dir_path );
+  if( FD_UNLIKELY( !dir ) ) return (errno == ENOENT) ? 0 : -1;
+
+  struct dirent *entry;
+  while( ( entry = readdir(dir) ) ) {
+    /* Skip . and .. */
+    if( FD_UNLIKELY( !strcmp(entry->d_name, ".") || !strcmp(entry->d_name, "..") ) ) continue;
+
+    char path[PATH_MAX];
+    FD_TEST( fd_cstr_printf_check( path, PATH_MAX, NULL, "%s/%s", dir_path, entry->d_name ) );
+
+    /* Only remove regular files, not subdirectories */
+    struct stat st;
+    if( stat(path, &st) == 0 && S_ISREG(st.st_mode)) {
+      if( FD_UNLIKELY( unlink(path) && errno != ENOENT ) ) {
+        FD_LOG_WARNING(( "failed to remove file `%s` (%i-%s)", path, errno, fd_io_strerror(errno) ));
+      }
+    }
+  }
+
+  closedir( dir );
+  return 0;
+}
+
 static void
 fini( config_t const * config,
       int              pre_init ) {
@@ -271,6 +307,13 @@ fini( config_t const * config,
     if( FD_LIKELY( fclose( fp ) ) )
       FD_LOG_ERR(( "error closing `/proc/self/mounts` (%i-%s)", errno, fd_io_strerror( errno ) ));
 
+    /* For normal pages, we need to empty the directory first */
+    if( i == 2 ) {
+      FD_LOG_NOTICE(( "RUN: `rmdir %s`", mount_path[ i ] ));
+      if( FD_UNLIKELY( empty_dir_top_level(mount_path[i]) && errno != ENOENT ) )
+        FD_LOG_ERR(( "error removing hugetlbfs mount at `%s` (%i-%s)", mount_path[i], errno, fd_io_strerror(errno) ));
+    }
+
     FD_LOG_NOTICE(( "RUN: `rmdir %s`", mount_path[ i ] ));
     if( FD_UNLIKELY( rmdir( mount_path[ i ] ) && errno!=ENOENT ) )
       FD_LOG_ERR(( "error removing hugetlbfs mount at `%s` (%i-%s)", mount_path[ i ], errno, fd_io_strerror( errno ) ));
@@ -307,13 +350,18 @@ check( config_t const * config ) {
   int result2 = stat( mount_path[ 1 ], &st );
   if( FD_UNLIKELY( result2 && errno!=ENOENT ) )
     PARTIALLY_CONFIGURED( "failed to stat `%s` (%i-%s)", mount_path[ 1 ], errno, fd_io_strerror( errno ) );
+  int result3 = stat( config->hugetlbfs.normal_page_mount_path, &st );
+  if( FD_UNLIKELY( result3 && errno!=ENOENT ) )
+    PARTIALLY_CONFIGURED( "failed to stat `%s` (%i-%s)", config->hugetlbfs.normal_page_mount_path, errno, fd_io_strerror( errno ) );
 
-  if( FD_UNLIKELY( result1 && result2 ) )
-    NOT_CONFIGURED( "mounts `%s` and `%s` do not exist", mount_path[ 0 ], mount_path[ 1 ] );
+  if( FD_UNLIKELY( result1 && result2 && result3 ) )
+    NOT_CONFIGURED( "mounts `%s`, `%s` and `%s` do not exist", mount_path[ 0 ], mount_path[ 1 ], config->hugetlbfs.normal_page_mount_path );
   else if( FD_UNLIKELY( result1 ) )
     PARTIALLY_CONFIGURED( "mount `%s` does not exist", mount_path[ 0 ] );
   else if( FD_UNLIKELY( result2 ) )
     PARTIALLY_CONFIGURED( "mount `%s` does not exist", mount_path[ 1 ] );
+  else if( FD_UNLIKELY( result3 ) )
+    PARTIALLY_CONFIGURED( "mount `%s` does not exist", config->hugetlbfs.normal_page_mount_path );
 
   CHECK( check_dir( config->hugetlbfs.mount_path, config->uid, config->gid, S_IFDIR | S_IRUSR | S_IWUSR | S_IXUSR ) );
   for( ulong i=0UL; i<2UL; i++ ) {

src/app/shared/commands/run/run.c

@@ -472,6 +472,9 @@ workspace_path( config_t const *       config,
     case FD_SHMEM_GIGANTIC_PAGE_SZ:
       mount_path = config->hugetlbfs.gigantic_page_mount_path;
       break;
+    case FD_SHMEM_NORMAL_PAGE_SZ:
+      mount_path = config->hugetlbfs.normal_page_mount_path;
+      break;
     default:
       FD_LOG_ERR(( "invalid page size %lu", wksp->page_sz ));
   }
@@ -511,6 +514,7 @@ warn_unknown_files( config_t const * config,
     int known_file = 0;
     for( ulong i=0UL; i<config->topo.wksp_cnt; i++ ) {
       fd_topo_wksp_t const * wksp = &config->topo.workspaces[ i ];
+      if( !wksp->is_locked ) continue;
 
       char expected_path[ PATH_MAX ];
       workspace_path( config, wksp, expected_path );

src/app/shared/fd_config.c

@@ -292,6 +292,11 @@ fd_config_fill( fd_config_t * config,
     NULL,
     "%s/.huge",
     config->hugetlbfs.mount_path ) );
+  FD_TEST( fd_cstr_printf_check( config->hugetlbfs.normal_page_mount_path,
+    sizeof(config->hugetlbfs.normal_page_mount_path),
+    NULL,
+    "%s/.normal",
+    config->hugetlbfs.mount_path ) );
 
   ulong max_page_sz = fd_cstr_to_shmem_page_sz( config->hugetlbfs.max_page_size );
   if( FD_UNLIKELY( max_page_sz!=FD_SHMEM_HUGE_PAGE_SZ && max_page_sz!=FD_SHMEM_GIGANTIC_PAGE_SZ ) ) FD_LOG_ERR(( "[hugetlbfs.max_page_size] must be \"huge\" or \"gigantic\"" ));

src/app/shared/fd_config.h

@@ -242,6 +242,7 @@ struct fd_config {
   struct {
     char  gigantic_page_mount_path[ PATH_MAX ];
     char  huge_page_mount_path[ PATH_MAX ];
+    char  normal_page_mount_path[ PATH_MAX ];
     char  mount_path[ PATH_MAX ];
     char  max_page_size[ 16 ];
     ulong gigantic_page_threshold_mib;

src/disco/topo/fd_topo.c

@@ -82,7 +82,9 @@ fd_topo_create_workspace( fd_topo_t *      topo,
   ulong sub_cpu_idx [ 1 ] = { fd_shmem_cpu_idx( wksp->numa_idx ) };
 
   int err;
-  if( FD_UNLIKELY( update_existing ) ) {
+  if( FD_UNLIKELY( !wksp->is_locked ) ) {
+    err = fd_shmem_create_multi_unlocked( name, wksp->page_sz, wksp->page_cnt, S_IRUSR | S_IWUSR ); /* logs details */
+  } else if( FD_UNLIKELY( update_existing ) ) {
     err = fd_shmem_update_multi( name, wksp->page_sz, 1, sub_page_cnt, sub_cpu_idx, S_IRUSR | S_IWUSR ); /* logs details */
   } else {
     err = fd_shmem_create_multi( name, wksp->page_sz, 1, sub_page_cnt, sub_cpu_idx, S_IRUSR | S_IWUSR ); /* logs details */
@@ -223,6 +225,8 @@ fd_topo_mlock_max_tile1( fd_topo_t const *      topo,
   ulong tile_mem = 0UL;
 
   for( ulong i=0UL; i<topo->wksp_cnt; i++ ) {
+    if( FD_UNLIKELY( !topo->workspaces[ i ].is_locked ) ) continue;
+
     if( FD_UNLIKELY( -1!=tile_needs_wksp( topo, tile, i ) ) )
       tile_mem += topo->workspaces[ i ].page_cnt * topo->workspaces[ i ].page_sz;
   }
@@ -250,6 +254,7 @@ fd_topo_gigantic_page_cnt( fd_topo_t const * topo,
   for( ulong i=0UL; i<topo->wksp_cnt; i++ ) {
     fd_topo_wksp_t const * wksp = &topo->workspaces[ i ];
     if( FD_LIKELY( wksp->numa_idx!=numa_idx ) ) continue;
+    if( FD_UNLIKELY( !wksp->is_locked ) ) continue;
 
     if( FD_LIKELY( wksp->page_sz==FD_SHMEM_GIGANTIC_PAGE_SZ ) ) {
       result += wksp->page_cnt;
@@ -266,6 +271,7 @@ fd_topo_huge_page_cnt( fd_topo_t const * topo,
   for( ulong i=0UL; i<topo->wksp_cnt; i++ ) {
     fd_topo_wksp_t const * wksp = &topo->workspaces[ i ];
     if( FD_LIKELY( wksp->numa_idx!=numa_idx ) ) continue;
+    if( FD_UNLIKELY( !wksp->is_locked ) ) continue;
 
     if( FD_LIKELY( wksp->page_sz==FD_SHMEM_HUGE_PAGE_SZ ) ) {
       result += wksp->page_cnt;
@@ -296,6 +302,7 @@ FD_FN_PURE ulong
 fd_topo_mlock( fd_topo_t const * topo ) {
   ulong result = 0UL;
   for( ulong i=0UL; i<topo->wksp_cnt; i++ ) {
+    if( FD_UNLIKELY( !topo->workspaces[ i ].is_locked ) ) continue;
     result += topo->workspaces[ i ].page_cnt * topo->workspaces[ i ].page_sz;
   }
   return result;
@@ -385,7 +392,7 @@ fd_topo_print_log( int         stdout,
 
     char size[ 24 ];
     fd_topo_mem_sz_string( wksp->page_sz * wksp->page_cnt, size );
-    PRINT( "  %2lu (%7s): %12s  page_cnt=%3lu  page_sz=%-8s  numa_idx=%-2lu  footprint=%10lu  loose=%lu\n", i, size, wksp->name, wksp->page_cnt, fd_shmem_page_sz_to_cstr( wksp->page_sz ), wksp->numa_idx, wksp->known_footprint, wksp->total_footprint - wksp->known_footprint );
+    PRINT( "  %2lu (%7s): %12s  page_cnt=%3lu  page_sz=%-8s  numa_idx=%-2lu  footprint=%10lu  loose=%10lu  is_locked=%d\n", i, size, wksp->name, wksp->page_cnt, fd_shmem_page_sz_to_cstr( wksp->page_sz ), wksp->numa_idx, wksp->known_footprint, wksp->total_footprint - wksp->known_footprint, wksp->is_locked );
   }
 
   PRINT( "\nOBJECTS\n" );

src/disco/topo/fd_topo.h

@@ -30,16 +30,17 @@
    sits on top of 1 or more memory mapped gigantic or huge pages mounted
    to the hugetlbfs. */
 typedef struct {
-  ulong id;           /* The ID of this workspace.  Indexed from [0, wksp_cnt).  When placed in a topology, the ID must be the index of the workspace in the workspaces list. */
-  char  name[ 13UL ]; /* The name of this workspace, like "pack".  There can be at most one of each workspace name in a topology. */
+  ulong id;                  /* The ID of this workspace.  Indexed from [0, wksp_cnt).  When placed in a topology, the ID must be the index of the workspace in the workspaces list. */
+  char  name[ 13UL ];        /* The name of this workspace, like "pack".  There can be at most one of each workspace name in a topology. */
 
-  ulong numa_idx;     /* The index of the NUMA node on the system that this workspace should be allocated from. */
+  ulong numa_idx;            /* The index of the NUMA node on the system that this workspace should be allocated from. */
 
   /* Computed fields.  These are not supplied as configuration but calculated as needed. */
   struct {
-    ulong page_sz;  /* The size of the pages that this workspace is backed by.  One of FD_PAGE_SIZE_*. */
-    ulong page_cnt; /* The number of pages that must be mapped to this workspace to store all the data needed by consumers. */
-    ulong part_max; /* The maximum number of partitions in the underlying workspace.  There can only be this many allocations made at any one time. */
+    ulong page_sz;   /* The size of the pages that this workspace is backed by.  One of FD_PAGE_SIZE_*. */
+    ulong page_cnt;  /* The number of pages that must be mapped to this workspace to store all the data needed by consumers. */
+    int   is_locked; /* If the workspace should use pages locked and pinned to a specific numa node. */
+    ulong part_max;  /* The maximum number of partitions in the underlying workspace.  There can only be this many allocations made at any one time. */
 
     fd_wksp_t * wksp;            /* The workspace memory in the local process. */
     ulong       known_footprint; /* Total size in bytes of all data in Firedancer that will be stored in this workspace at startup. */

src/disco/topo/fd_topob.c

@@ -41,6 +41,7 @@ fd_topob_wksp( fd_topo_t *  topo,
   fd_topo_wksp_t * wksp = &topo->workspaces[ topo->wksp_cnt ];
   strncpy( wksp->name, name, sizeof(wksp->name) );
   wksp->id = topo->wksp_cnt;
+  wksp->is_locked = 1;
   topo->wksp_cnt++;
 }
 
@@ -644,6 +645,11 @@ fd_topob_finish( fd_topo_t *                topo,
     if( total_wksp_footprint < topo->gigantic_page_threshold ) page_sz = FD_SHMEM_HUGE_PAGE_SZ;
     if( FD_UNLIKELY( page_sz!=FD_SHMEM_HUGE_PAGE_SZ && page_sz!=FD_SHMEM_GIGANTIC_PAGE_SZ ) ) FD_LOG_ERR(( "invalid page_sz" ));
 
+    /* If the workspace is not locked, we can't use huge pages. */
+    if( FD_UNLIKELY( !wksp->is_locked ) ) {
+      page_sz = FD_SHMEM_NORMAL_PAGE_SZ;
+    }
+
     ulong wksp_aligned_footprint = fd_ulong_align_up( total_wksp_footprint, page_sz );
 
     /* Give any leftover space in the underlying shared memory to the

src/util/shmem/fd_shmem.h

@@ -346,13 +346,36 @@ fd_shmem_create_multi( char const *  name,         /* Should point to cstr with
                        ulong const * sub_cpu_idx,  /* Indexed [0,sub_cnt), each should be in [0,fd_shmem_cpu_cnt()) */
                        ulong         mode );       /* E.g. 0660 for user rw, group rw, world none */
 
+/* fd_shmem_create_multi_unlocked creates a shared memory region whose
+   name is given by the cstr pointed to by name backed by page_sz pages.
+   It functions the same as fd_shmem_create_multi, but the pages are not
+   locked, not pinned to any particular numa node, and have the default numa
+   mempolicy.
+
+   mode specifies the permissions for this region (the usual POSIX open
+   umask caveats apply).
+
+   Returns 0 on success and an strerror friendly error code on failure
+   (also logs extensive details on error).  Reasons for failure include
+   name is invalid (EINVAL), page_sz is invalid (EINVAL), page_cnt is
+   zero (EINVAL), cnt*page_sz overflows an off_t (EINVAL), open fails
+   (errno of the open, e.g. region with the same name and page_sz in the
+   thread domain already exists), ftruncate fails (errno of ftruncate,
+   e.g. no suitable memory available near cpu_idx), etc.
+*/
+int
+fd_shmem_create_multi_unlocked( char const *  name,
+                                ulong         page_sz,
+                                ulong         page_cnt,
+                                ulong         mode );
+
 /* fd_shmem_update_multi updates a shared memory region created by
    fd_shmem_create_multi in place, to be as-if it was created with
    the provided parameters instead.
-   
+
    This can be preferable to deleting and recreating the shmem region
    because it prevents needing to zero all of the underlying memory.
-   
+
    WARNING: The memory returned will not be zeroed and the user will
    be able to read any contents that were in the previous workspace. */
 
@@ -449,7 +472,7 @@ fd_shmem_acquire( ulong page_sz,
    by fd_shmem_acquire.  This always succeeds from the caller's POV but
    logs details if there is any wonkiness under the hood.  It is fine to
    release subregions of individual previous acquisitions.
-   
+
    Returns 0 if successful, -1 for any errors. */
 
 int

src/util/shmem/fd_shmem_admin.c

@@ -407,6 +407,82 @@ fd_shmem_update_multi( char const *  name,
   return fd_shmem_create_multi_flags( name, page_sz, sub_cnt, _sub_page_cnt, _sub_cpu_idx, mode, O_RDWR );
 }
 
+int
+fd_shmem_create_multi_unlocked( char const *  name,
+                                ulong         page_sz,
+                                ulong         page_cnt,
+                                ulong         mode ) {
+
+  /* Check input args */
+
+  if( FD_UNLIKELY( !fd_shmem_name_len( name ) ) )      { FD_LOG_WARNING(( "bad name (%s)", name ? name : "NULL" )); return EINVAL; }
+
+  if( FD_UNLIKELY( !page_cnt       ) )                 { FD_LOG_WARNING(( "zero page_cnt"      )); return EINVAL; }
+
+  if( FD_UNLIKELY( !fd_shmem_is_page_sz( page_sz ) ) ) { FD_LOG_WARNING(( "bad page_sz (%lu)", page_sz )); return EINVAL; }
+
+
+
+# define ERROR( cleanup ) do { err = errno; goto cleanup; } while(0)
+
+  int err = 0;
+
+  char   path[ FD_SHMEM_PRIVATE_PATH_BUF_MAX ];
+  void * shmem;
+
+  ulong      sz = page_cnt*page_sz;
+
+  /* Acquire the pages at a random address */
+
+  /* Create the region */
+  int open_flags = O_RDWR | O_CREAT | O_TRUNC;
+  int fd = open( fd_shmem_private_path( name, page_sz, path ), open_flags, (mode_t)mode );
+  if( FD_UNLIKELY( fd==-1 ) ) {
+    FD_LOG_WARNING(( "open(\"%s\",%#x,0%03lo) failed (%i-%s)", path, (uint)open_flags, mode, errno, fd_io_strerror( errno ) ));
+    ERROR( done );
+  }
+
+  /* Size the region */
+
+  if( FD_UNLIKELY( ftruncate( fd, (off_t)sz ) ) ) {
+    FD_LOG_WARNING(( "ftruncate(\"%s\",%lu KiB) failed (%i-%s)", path, sz>>10, errno, fd_io_strerror( errno ) ));
+    ERROR( close );
+  }
+
+  /* Map the region into our address space. */
+
+  shmem = mmap( NULL, sz, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, (off_t)0);
+  if( FD_UNLIKELY( shmem==MAP_FAILED ) ) {
+    FD_LOG_WARNING(( "mmap(NULL,%lu KiB,PROT_READ|PROT_WRITE,MAP_SHARED,\"%s\",0) failed (%i-%s)",
+                     sz>>10, path, errno, fd_io_strerror( errno ) ));
+    ERROR( close );
+  }
+
+  /* Validate the mapping */
+
+  if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)shmem, page_sz ) ) ) {
+    FD_LOG_WARNING(( "misaligned memory mapping for unpinned shmem region \"%s\"", name ));
+    errno = EFAULT; /* ENOMEM is arguable */
+    ERROR( unmap );
+  }
+
+# undef ERROR
+
+unmap:
+  if( FD_UNLIKELY( err ) && FD_UNLIKELY( munmap( shmem, sz ) ) )
+    FD_LOG_ERR(( "munmap(\"%s\",%lu KiB) failed (%i-%s)",
+                 path, sz>>10, errno, fd_io_strerror( errno ) ));
+
+close:
+  if( FD_UNLIKELY( err ) && FD_UNLIKELY( unlink( path ) ) )
+    FD_LOG_ERR(( "unlink(\"%s\") failed (%i-%s)", path, errno, fd_io_strerror( errno ) ));
+  if( FD_UNLIKELY( err ) && FD_UNLIKELY( close( fd ) ) )
+    FD_LOG_ERR(( "close(\"%s\") failed (%i-%s)", path, errno, fd_io_strerror( errno ) ));
+
+done:
+  return err;
+}
+
 int
 fd_shmem_unlink( char const * name,
                  ulong        page_sz ) {