Snapshot Loader 2.0

src/app/firedancer-dev/main.c

@@ -91,6 +91,8 @@ extern fd_topo_run_tile_t fd_tile_archiver_feeder;
 extern fd_topo_run_tile_t fd_tile_archiver_writer;
 extern fd_topo_run_tile_t fd_tile_archiver_playback;
 
+extern fd_topo_run_tile_t fd_tile_snapshot_restore_FileRd;
+
 fd_topo_run_tile_t * TILES[] = {
   &fd_tile_net,
   &fd_tile_netlnk,
@@ -120,6 +122,7 @@ fd_topo_run_tile_t * TILES[] = {
   &fd_tile_archiver_feeder,
   &fd_tile_archiver_writer,
   &fd_tile_archiver_playback,
+  &fd_tile_snapshot_restore_FileRd,
   NULL,
 };
 

src/disco/stem/fd_stem.h

@@ -8,7 +8,7 @@
 struct fd_stem_context {
    fd_frag_meta_t ** mcaches;
    ulong *           seqs;
-   ulong *           depths;
+   ulong const *     depths;
 
    ulong *           cr_avail;
    ulong             cr_decrement_amount;

src/disco/topo/fd_topo.h

@@ -421,6 +421,10 @@ typedef struct {
       int archive_fd;
     } archiver;
 
+    struct {
+      char file_path[ PATH_MAX ];
+    } filerd;
+
   };
 } fd_topo_tile_t;
 

src/disco/topo/fd_topob.c

@@ -64,7 +64,7 @@ fd_topob_obj( fd_topo_t *  topo,
   return obj;
 }
 
-void
+fd_topo_link_t *
 fd_topob_link( fd_topo_t *  topo,
                char const * link_name,
                char const * wksp_name,
@@ -100,6 +100,8 @@ fd_topob_link( fd_topo_t *  topo,
     FD_TEST( fd_pod_insertf_ulong( topo->props, mtu,   "obj.%lu.mtu",   obj->id ) );
   }
   topo->link_cnt++;
+
+  return link;
 }
 
 void

src/disco/topo/fd_topob.h

@@ -72,7 +72,7 @@ fd_topob_tile_uses( fd_topo_t *      topo,
    can have no backing data buffer, a dcache, or a reassembly buffer
    behind it. */
 
-void
+fd_topo_link_t *
 fd_topob_link( fd_topo_t *  topo,
                char const * link_name,
                char const * wksp_name,

src/discof/restore/Local.mk

@@ -0,0 +1 @@
+$(call add-objs,fd_filerd_tile,fd_discof)

src/discof/restore/README.md

@@ -0,0 +1,35 @@
+# Snapshot Restore
+
+## Stream link conventions
+
+Various snapshot components use byte streams, not packet streams.
+
+These require custom conventions.
+
+**Stream fragment descriptors**
+
+Byte streams use `fd_frag_stream_meta_t` (defined in `fd_restore_base.h`).
+
+These have the following changes:
+- `chunk` is replaced by `goff` and `loff`, which are 64-bit offsets
+  describing the stream offset and dcache offset respectively
+- `tsorig` / `tspub` are removed (latency is less relevant)
+- `sig` is removed (cannot filter without looking at stream data)
+- `sz` is widened to 32 bits.
+
+`**Dcache allocations**
+
+Payloads in stream dcaches are unaligned.  Payloads are addressed with
+uncompressed byte offsets relative to the workspace start.
+
+(Compare this to the usual compact packet dcaches, which use 64 byte
+aligned chunks with compressed addressing.)
+
+**Stream backpressure**
+
+Byte streams naturally require a reliable transport.
+
+Consumers periodically publish their progress in `fseq`.
+- `fseq[0]` is the lowest sequence number not yet consumed (standard)
+- `fseq[1]` is the stream offset of the next byte not yet consumed
+`