Skip to content

Commit 427977f

Browse files
riptlriptl
committed
funk: clean up join API
Adds a 'join' struct containing local address space pointers to sub objects. Improves insert performance by about 20ns per record. Reverts 'ele_max' to ulong to avoid integer truncation issues.
1 parent 16fd3a3 commit 427977f

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

45 files changed

+854
-881
lines changed

‎src/app/ledger/main.c

+7-7
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ struct fd_ledger_args {
2222
fd_wksp_t * status_cache_wksp; /* wksp for status cache. */
2323
fd_blockstore_t blockstore_ljoin;
2424
fd_blockstore_t * blockstore; /* blockstore for replay */
25-
fd_funk_t * funk; /* handle to funk */
25+
fd_funk_t funk[1]; /* handle to funk */
2626
fd_alloc_t * alloc; /* handle to alloc */
2727
char const * cmd; /* user passed command to fd_ledger */
2828
ulong start_slot; /* start slot for offline replay */
@@ -894,13 +894,13 @@ void
894894
init_funk( fd_ledger_args_t * args ) {
895895
fd_funk_t * funk;
896896
if( args->restore_funk ) {
897-
funk = fd_funk_recover_checkpoint( args->funk_file, 1, args->restore_funk, &args->funk_close_args );
897+
funk = fd_funk_recover_checkpoint( args->funk, args->funk_file, 1, args->restore_funk, &args->funk_close_args );
898898
} else {
899-
funk = fd_funk_open_file( args->funk_file, 1, args->hashseed, args->txns_max, args->index_max, args->funk_page_cnt*(1UL<<30), FD_FUNK_OVERWRITE, &args->funk_close_args );
899+
funk = fd_funk_open_file( args->funk, args->funk_file, 1, args->hashseed, args->txns_max, args->index_max, args->funk_page_cnt*(1UL<<30), FD_FUNK_OVERWRITE, &args->funk_close_args );
900900
}
901-
args->funk = funk;
901+
if( FD_UNLIKELY( !funk ) ) FD_LOG_ERR(( "Failed to join funk" ));
902902
args->funk_wksp = fd_funk_wksp( funk );
903-
FD_LOG_NOTICE(( "funky at global address 0x%016lx", fd_wksp_gaddr_fast( args->funk_wksp, funk ) ));
903+
FD_LOG_NOTICE(( "Funk database is at %s:0x%lx", fd_wksp_name( args->wksp ), fd_wksp_gaddr_fast( args->funk_wksp, funk ) ));
904904
}
905905

906906
void
@@ -1171,9 +1171,9 @@ ingest( fd_ledger_args_t * args ) {
11711171

11721172
#ifdef FD_FUNK_HANDHOLDING
11731173
if( args->verify_funk ) {
1174-
FD_LOG_NOTICE(( "verifying funky" ));
1174+
FD_LOG_NOTICE(( "fd_funk_verify() start" ));
11751175
if( fd_funk_verify( funk ) ) {
1176-
FD_LOG_ERR(( "verification failed" ));
1176+
FD_LOG_ERR(( "fd_funk_verify() failed" ));
11771177
}
11781178
}
11791179
#endif

‎src/app/rpcserver/main.c

+10-7
Original file line numberDiff line numberDiff line change
@@ -32,9 +32,9 @@ init_args( int * argc, char *** argv, fd_rpcserver_args_t * args ) {
3232
char const * funk_file = fd_env_strip_cmdline_cstr( argc, argv, "--funk-file", NULL, NULL );
3333
if( FD_UNLIKELY( !funk_file ))
3434
FD_LOG_ERR(( "--funk-file argument is required" ));
35-
args->funk = fd_funk_open_file( funk_file, 1, 0, 0, 0, 0, FD_FUNK_READONLY, NULL );
36-
if( args->funk == NULL ) {
37-
FD_LOG_ERR(( "failed to join a funky" ));
35+
fd_funk_t * funk = fd_funk_open_file( args->funk, funk_file, 1, 0, 0, 0, 0, FD_FUNK_READONLY, NULL );
36+
if( !funk ) {
37+
FD_LOG_ERR(( "failed to join funk" ));
3838
}
3939

4040
char const * blockstore_file = fd_env_strip_cmdline_cstr( argc, argv, "--blockstore-file", NULL, NULL );
@@ -107,10 +107,13 @@ init_args_offline( int * argc, char *** argv, fd_rpcserver_args_t * args ) {
107107
if( FD_UNLIKELY( !funk_file ))
108108
FD_LOG_ERR(( "--funk-file argument is required" ));
109109
char const * restore = fd_env_strip_cmdline_cstr ( argc, argv, "--restore-funk", NULL, NULL );
110-
if( restore != NULL )
111-
args->funk = fd_funk_recover_checkpoint( funk_file, 1, restore, NULL );
112-
else
113-
args->funk = fd_funk_open_file( funk_file, 1, 0, 0, 0, 0, FD_FUNK_READONLY, NULL );
110+
fd_funk_t * funk = NULL;
111+
if( restore != NULL ) {
112+
funk = fd_funk_recover_checkpoint( args->funk, funk_file, 1, restore, NULL );
113+
} else {
114+
funk = fd_funk_open_file( args->funk, funk_file, 1, 0, 0, 0, 0, FD_FUNK_READONLY, NULL );
115+
}
116+
if( FD_UNLIKELY( !funk ) ) FD_LOG_ERR(( "Failed to join funk database" ));
114117

115118
fd_wksp_t * wksp;
116119
const char * wksp_name = fd_env_strip_cmdline_cstr ( argc, argv, "--wksp-name-blockstore", NULL, NULL );

‎src/choreo/forks/fd_forks.c

+3-3
Original file line numberDiff line numberDiff line change
@@ -193,7 +193,7 @@ slot_ctx_restore( ulong slot,
193193
fd_exec_epoch_ctx_t * epoch_ctx,
194194
fd_spad_t * runtime_spad,
195195
fd_exec_slot_ctx_t * slot_ctx_out ) {
196-
fd_funk_txn_map_t txn_map = fd_funk_txn_map( funk, fd_funk_wksp( funk ) );
196+
fd_funk_txn_map_t * txn_map = fd_funk_txn_map( funk );
197197
bool block_exists = fd_blockstore_shreds_complete( blockstore, slot );
198198

199199
FD_LOG_DEBUG( ( "Current slot %lu", slot ) );
@@ -204,11 +204,11 @@ slot_ctx_restore( ulong slot,
204204
fd_funk_rec_key_t id = fd_runtime_slot_bank_key();
205205
for( ; ; ) {
206206
fd_funk_txn_start_read( funk );
207-
fd_funk_txn_t * txn = fd_funk_txn_query( &xid, &txn_map );
207+
fd_funk_txn_t * txn = fd_funk_txn_query( &xid, txn_map );
208208
if( !txn ) {
209209
memset( xid.uc, 0, sizeof( fd_funk_txn_xid_t ) );
210210
xid.ul[0] = slot;
211-
txn = fd_funk_txn_query( &xid, &txn_map );
211+
txn = fd_funk_txn_query( &xid, txn_map );
212212
if( !txn ) {
213213
FD_LOG_ERR( ( "missing txn, parent slot %lu", slot ) );
214214
}

‎src/discof/batch/fd_batch_tile.c

+6-11
Original file line numberDiff line numberDiff line change
@@ -442,20 +442,15 @@ after_credit( fd_snapshot_tile_ctx_t * ctx,
442442
if( FD_UNLIKELY( !ctx->is_funk_active ) ) {
443443
/* Setting these parameters are not required because we are joining the
444444
funk that was setup in the replay tile. */
445-
ctx->funk = fd_funk_open_file( ctx->funk_file,
446-
1UL,
447-
0UL,
448-
0UL,
449-
0UL,
450-
0UL,
451-
FD_FUNK_READ_WRITE,
452-
NULL );
453-
if( FD_UNLIKELY( !ctx->funk ) ) {
454-
FD_LOG_ERR(( "failed to join a funky" ));
445+
fd_funk_t * funk = fd_funk_open_file(
446+
ctx->funk, ctx->funk_file,
447+
1UL, 0UL, 0UL, 0UL, 0UL, FD_FUNK_READ_WRITE, NULL );
448+
if( FD_UNLIKELY( !funk ) ) {
449+
FD_LOG_ERR(( "Failed to join a funk database" ));
455450
}
456451
ctx->is_funk_active = 1;
457452

458-
FD_LOG_WARNING(( "Just joined funk at file=%s", ctx->funk_file ));
453+
FD_LOG_WARNING(( "Joined funk database at file=%s", ctx->funk_file ));
459454
}
460455

461456
if( fd_batch_fseq_is_snapshot( batch_fseq ) ) {

‎src/discof/consensus/test_consensus.c

+4-3
Original file line numberDiff line numberDiff line change
@@ -444,13 +444,14 @@ main( void ) {
444444
// /**********************************************************************/
445445

446446
// fd_wksp_tag_query_info_t funk_info;
447-
// fd_funk_t * funk = NULL;
447+
// fd_funk_t funk_[1];
448+
// fd_funk_t * funk = NULL;
448449
// ulong funk_tag = FD_FUNK_MAGIC;
449450
// if( fd_wksp_tag_query( wksp, &funk_tag, 1, &funk_info, 1 ) > 0 ) {
450451
// void * shmem = fd_wksp_laddr_fast( wksp, funk_info.gaddr_lo );
451-
// funk = fd_funk_join( shmem );
452+
// funk = fd_funk_join( funk_, shmem );
452453
// }
453-
// if( funk == NULL ) FD_LOG_ERR( ( "failed to join a funky" ) );
454+
// if( !funk ) FD_LOG_ERR( ( "failed to join a funky" ) );
454455

455456
// /**********************************************************************/
456457
// /* blockstore */

‎src/discof/exec/fd_exec_tile.c

+9-15
Original file line numberDiff line numberDiff line change
@@ -228,13 +228,13 @@ prepare_new_slot_execution( fd_exec_tile_ctx_t * ctx,
228228
fd_spad_push( ctx->exec_spad );
229229
ctx->pending_slot_pop = 1;
230230

231-
fd_funk_txn_map_t txn_map = fd_funk_txn_map( ctx->funk, ctx->funk_wksp );
232-
if( FD_UNLIKELY( !txn_map.map ) ) {
231+
fd_funk_txn_map_t * txn_map = fd_funk_txn_map( ctx->funk );
232+
if( FD_UNLIKELY( !txn_map->map ) ) {
233233
FD_LOG_ERR(( "Could not find valid funk transaction map" ));
234234
}
235235
fd_funk_txn_xid_t xid = { .ul = { slot_msg->slot, slot_msg->slot } };
236236
fd_funk_txn_start_read( ctx->funk );
237-
fd_funk_txn_t * funk_txn = fd_funk_txn_query( &xid, &txn_map );
237+
fd_funk_txn_t * funk_txn = fd_funk_txn_query( &xid, txn_map );
238238
if( FD_UNLIKELY( !funk_txn ) ) {
239239
FD_LOG_ERR(( "Could not find valid funk transaction" ));
240240
}
@@ -616,19 +616,13 @@ unprivileged_init( fd_topo_t * topo,
616616
the funk that was setup in the replay tile. */
617617
FD_LOG_NOTICE(( "Trying to join funk at file=%s", tile->exec.funk_file ));
618618
fd_funk_txn_start_write( NULL );
619-
ctx->funk = fd_funk_open_file( tile->exec.funk_file,
620-
1UL,
621-
0UL,
622-
0UL,
623-
0UL,
624-
0UL,
625-
FD_FUNK_READONLY,
626-
NULL );
619+
if( FD_UNLIKELY( !fd_funk_open_file(
620+
ctx->funk, tile->exec.funk_file,
621+
1UL, 0UL, 0UL, 0UL, 0UL, FD_FUNK_READONLY, NULL ) ) ) {
622+
FD_LOG_ERR(( "fd_funk_open_file(%s) failed", tile->exec.funk_file ));
623+
}
627624
fd_funk_txn_end_write( NULL );
628625
ctx->funk_wksp = fd_funk_wksp( ctx->funk );
629-
if( FD_UNLIKELY( !ctx->funk ) ) {
630-
FD_LOG_ERR(( "failed to join a funk" ));
631-
}
632626

633627
FD_LOG_NOTICE(( "Just joined funk at file=%s", tile->exec.funk_file ));
634628

@@ -641,7 +635,7 @@ unprivileged_init( fd_topo_t * topo,
641635
ctx->pending_slot_pop = 0;
642636
uchar * txn_ctx_mem = fd_spad_alloc( ctx->exec_spad, FD_EXEC_TXN_CTX_ALIGN, FD_EXEC_TXN_CTX_FOOTPRINT );
643637
ctx->txn_ctx = fd_exec_txn_ctx_join( fd_exec_txn_ctx_new( txn_ctx_mem ), ctx->exec_spad, ctx->exec_spad_wksp );
644-
ctx->txn_ctx->funk = ctx->funk;
638+
*ctx->txn_ctx->funk = *ctx->funk;
645639

646640
ctx->txn_ctx->runtime_pub_wksp = ctx->runtime_public_wksp;
647641
if( FD_UNLIKELY( !ctx->txn_ctx->runtime_pub_wksp ) ) {

‎src/discof/geyser/fd_geyser.c

+5-5
Original file line numberDiff line numberDiff line change
@@ -72,9 +72,9 @@ fd_geyser_new( void * mem, fd_geyser_args_t * args ) {
7272
ulong scratch_top = FD_SCRATCH_ALLOC_FINI( l, 1UL );
7373
FD_TEST( scratch_top <= (ulong)mem + fd_geyser_footprint() );
7474

75-
self->funk = fd_funk_open_file( args->funk_file, 1, 0, 0, 0, 0, FD_FUNK_READONLY, NULL );
76-
if( self->funk == NULL ) {
77-
FD_LOG_ERR(( "failed to join a funky" ));
75+
fd_funk_t * funk = fd_funk_open_file( self->funk, args->funk_file, 1, 0, 0, 0, 0, FD_FUNK_READONLY, NULL );
76+
if( !funk ) {
77+
FD_LOG_ERR(( "fd_funk_open_file(%s) failed", args->funk_file ));
7878
}
7979

8080
fd_wksp_t * wksp = fd_wksp_attach( args->blockstore_wksp );
@@ -216,8 +216,8 @@ replay_sham_link_during_frag( fd_geyser_t * ctx, fd_replay_notif_msg_t * state,
216216

217217
static const void *
218218
read_account_with_xid( fd_geyser_t * ctx, fd_funk_rec_key_t * recid, fd_funk_txn_xid_t * xid, ulong * result_len ) {
219-
fd_funk_txn_map_t txn_map = fd_funk_txn_map( ctx->funk, fd_funk_wksp( ctx->funk ) );
220-
fd_funk_txn_t * txn = fd_funk_txn_query( xid, &txn_map );
219+
fd_funk_txn_map_t * txn_map = fd_funk_txn_map( ctx->funk );
220+
fd_funk_txn_t * txn = fd_funk_txn_query( xid, txn_map );
221221
return fd_funk_rec_query_copy( ctx->funk, txn, recid, fd_scratch_virtual(), result_len );
222222
}
223223

‎src/discof/replay/fd_replay_tile.c

+22-21
Original file line numberDiff line numberDiff line change
@@ -199,7 +199,7 @@ struct fd_replay_tile_ctx {
199199

200200
fd_alloc_t * alloc;
201201
fd_valloc_t valloc;
202-
fd_funk_t * funk;
202+
fd_funk_t funk[1];
203203
fd_exec_epoch_ctx_t * epoch_ctx;
204204
fd_epoch_t * epoch;
205205
fd_forks_t * forks;
@@ -826,9 +826,9 @@ checkpt( fd_replay_tile_ctx_t * ctx ) {
826826
static void FD_FN_UNUSED
827827
funk_cancel( fd_replay_tile_ctx_t * ctx, ulong mismatch_slot ) {
828828
fd_funk_txn_start_write( ctx->funk );
829-
fd_funk_txn_xid_t xid = { .ul = { mismatch_slot, mismatch_slot } };
830-
fd_funk_txn_map_t txn_map = fd_funk_txn_map( ctx->funk, fd_funk_wksp( ctx->funk ) );
831-
fd_funk_txn_t * mismatch_txn = fd_funk_txn_query( &xid, &txn_map );
829+
fd_funk_txn_xid_t xid = { .ul = { mismatch_slot, mismatch_slot } };
830+
fd_funk_txn_map_t * txn_map = fd_funk_txn_map( ctx->funk );
831+
fd_funk_txn_t * mismatch_txn = fd_funk_txn_query( &xid, txn_map );
832832
FD_TEST( fd_funk_txn_cancel( ctx->funk, mismatch_txn, 1 ) );
833833
fd_funk_txn_end_write( ctx->funk );
834834
}
@@ -860,7 +860,7 @@ txncache_publish( fd_replay_tile_ctx_t * ctx,
860860
fd_funk_txn_start_read( ctx->funk );
861861

862862
fd_funk_txn_t * txn = to_root_txn;
863-
fd_funk_txn_pool_t txn_pool = fd_funk_txn_pool( ctx->funk, fd_funk_wksp( ctx->funk ) );
863+
fd_funk_txn_pool_t * txn_pool = fd_funk_txn_pool( ctx->funk );
864864
while( txn!=rooted_txn ) {
865865
ulong slot = txn->xid.ul[0];
866866
if( FD_LIKELY( !fd_txncache_get_is_constipated( ctx->slot_ctx->status_cache ) ) ) {
@@ -870,7 +870,7 @@ txncache_publish( fd_replay_tile_ctx_t * ctx,
870870
FD_LOG_INFO(( "Registering constipated slot %lu", slot ));
871871
fd_txncache_register_constipated_slot( ctx->slot_ctx->status_cache, slot );
872872
}
873-
txn = fd_funk_txn_parent( txn, &txn_pool );
873+
txn = fd_funk_txn_parent( txn, txn_pool );
874874
}
875875

876876
fd_funk_txn_end_read( ctx->funk );
@@ -962,7 +962,7 @@ funk_publish( fd_replay_tile_ctx_t * ctx,
962962
fd_funk_txn_start_write( ctx->funk );
963963

964964
fd_epoch_bank_t * epoch_bank = fd_exec_epoch_ctx_epoch_bank( ctx->slot_ctx->epoch_ctx );
965-
fd_funk_txn_pool_t txn_pool = fd_funk_txn_pool( ctx->funk, fd_funk_wksp( ctx->funk ) );
965+
fd_funk_txn_pool_t * txn_pool = fd_funk_txn_pool( ctx->funk );
966966

967967
/* Try to publish into Funk */
968968
if( is_constipated ) {
@@ -982,7 +982,7 @@ funk_publish( fd_replay_tile_ctx_t * ctx,
982982
if( FD_UNLIKELY( fd_funk_txn_publish_into_parent( ctx->funk, txn, 0 ) ) ) {
983983
FD_LOG_ERR(( "Can't publish funk transaction" ));
984984
}
985-
txn = fd_funk_txn_parent( txn, &txn_pool );
985+
txn = fd_funk_txn_parent( txn, txn_pool );
986986
}
987987

988988
} else {
@@ -1003,7 +1003,7 @@ funk_publish( fd_replay_tile_ctx_t * ctx,
10031003
we will calculate the epoch account hash for. */
10041004

10051005
fd_funk_txn_t * txn = to_root_txn;
1006-
fd_funk_txn_t * parent_txn = fd_funk_txn_parent( txn, &txn_pool );
1006+
fd_funk_txn_t * parent_txn = fd_funk_txn_parent( txn, txn_pool );
10071007
while( parent_txn ) {
10081008
/* We need to be careful here because the eah start slot may be skipped
10091009
so the actual slot that we calculate the eah for may be greater than
@@ -1018,7 +1018,7 @@ funk_publish( fd_replay_tile_ctx_t * ctx,
10181018
break;
10191019
}
10201020
txn = parent_txn;
1021-
parent_txn = fd_funk_txn_parent( txn, &txn_pool );
1021+
parent_txn = fd_funk_txn_parent( txn, txn_pool );
10221022
}
10231023

10241024
/* At this point, we know txn is the funk txn that we will want to
@@ -1069,17 +1069,17 @@ get_rooted_txn( fd_replay_tile_ctx_t * ctx,
10691069
we must also register it into the status cache because we don't register
10701070
the root in txncache_publish to avoid registering the same slot multiple times. */
10711071

1072-
fd_funk_txn_pool_t txn_pool = fd_funk_txn_pool( ctx->funk, fd_funk_wksp( ctx->funk ) );
1072+
fd_funk_txn_pool_t * txn_pool = fd_funk_txn_pool( ctx->funk );
10731073

10741074
if( is_constipated ) {
10751075

10761076
if( FD_UNLIKELY( !ctx->false_root ) ) {
10771077

10781078
fd_funk_txn_t * txn = to_root_txn;
1079-
fd_funk_txn_t * parent_txn = fd_funk_txn_parent( txn, &txn_pool );
1079+
fd_funk_txn_t * parent_txn = fd_funk_txn_parent( txn, txn_pool );
10801080
while( parent_txn ) {
10811081
txn = parent_txn;
1082-
parent_txn = fd_funk_txn_parent( txn, &txn_pool );
1082+
parent_txn = fd_funk_txn_parent( txn, txn_pool );
10831083
}
10841084

10851085
ctx->false_root = txn;
@@ -1143,8 +1143,8 @@ funk_and_txncache_publish( fd_replay_tile_ctx_t * ctx, ulong wmk, fd_funk_txn_xi
11431143
/* Handle updates to funk and the status cache. */
11441144

11451145
fd_funk_txn_start_read( ctx->funk );
1146-
fd_funk_txn_map_t txn_map = fd_funk_txn_map( ctx->funk, fd_funk_wksp( ctx->funk ) );
1147-
fd_funk_txn_t * to_root_txn = fd_funk_txn_query( xid, &txn_map );
1146+
fd_funk_txn_map_t * txn_map = fd_funk_txn_map( ctx->funk );
1147+
fd_funk_txn_t * to_root_txn = fd_funk_txn_query( xid, txn_map );
11481148
if( FD_UNLIKELY( !to_root_txn ) ) {
11491149
FD_LOG_ERR(( "Unable to find funk transaction for xid %lu", xid->ul[0] ));
11501150
}
@@ -2918,26 +2918,27 @@ privileged_init( fd_topo_t * topo,
29182918
if( strcmp( snapshot, "funk" ) == 0 ) {
29192919
/* Funk database already exists. The parameters are actually mostly ignored. */
29202920
funk = fd_funk_open_file(
2921-
tile->replay.funk_file, 1, ctx->funk_seed, tile->replay.funk_txn_max,
2921+
ctx->funk,
2922+
tile->replay.funk_file, 1, ctx->funk_seed, tile->replay.funk_txn_max,
29222923
tile->replay.funk_rec_max, tile->replay.funk_sz_gb * (1UL<<30),
29232924
FD_FUNK_READ_WRITE, NULL );
29242925
} else if( strncmp( snapshot, "wksp:", 5 ) == 0) {
29252926
/* Recover funk database from a checkpoint. */
2926-
funk = fd_funk_recover_checkpoint( tile->replay.funk_file, 1, snapshot+5, NULL );
2927+
funk = fd_funk_recover_checkpoint( ctx->funk, tile->replay.funk_file, 1, snapshot+5, NULL );
29272928
} else {
29282929
FD_LOG_NOTICE(( "Trying to create new funk at file=%s", tile->replay.funk_file ));
29292930
/* Create new funk database */
29302931
funk = fd_funk_open_file(
2931-
tile->replay.funk_file, 1, ctx->funk_seed, tile->replay.funk_txn_max,
2932+
ctx->funk,
2933+
tile->replay.funk_file, 1, ctx->funk_seed, tile->replay.funk_txn_max,
29322934
tile->replay.funk_rec_max, tile->replay.funk_sz_gb * (1UL<<30),
29332935
FD_FUNK_OVERWRITE, NULL );
29342936
FD_LOG_NOTICE(( "Opened funk file at %s", tile->replay.funk_file ));
29352937
}
2936-
if( FD_UNLIKELY( funk == NULL ) ) {
2937-
FD_LOG_ERR(( "no funk loaded" ));
2938+
if( FD_UNLIKELY( !funk ) ) {
2939+
FD_LOG_ERR(( "Failed to join funk database" ));
29382940
}
29392941
fd_funk_txn_end_write( NULL );
2940-
ctx->funk = funk;
29412942
ctx->funk_wksp = fd_funk_wksp( funk );
29422943
if( FD_UNLIKELY( ctx->funk_wksp == NULL ) ) {
29432944
FD_LOG_ERR(( "no funk wksp" ));

0 commit comments

Comments
 (0)