Line data Source code
1 : #ifndef HEADER_fd_src_disco_topo_fd_topo_h
2 : #define HEADER_fd_src_disco_topo_fd_topo_h
3 :
4 : #include "../stem/fd_stem.h"
5 : #include "../../tango/fd_tango.h"
6 : #include "../../waltz/xdp/fd_xdp1.h"
7 : #include "../../ballet/base58/fd_base58.h"
8 : #include "../../flamenco/fd_flamenco_base.h"
9 : #include "../../util/net/fd_net_headers.h"
10 : #include "../../util/net/fd_ip6.h"
11 : #include "../pack/fd_pack.h" /* for FD_PACK_ACCT_BLOCKLIST_MAX */
12 :
13 : /* Maximum number of workspaces that may be present in a topology. */
14 : #define FD_TOPO_MAX_WKSPS (256UL)
15 : /* Maximum number of links that may be present in a topology. */
16 0 : #define FD_TOPO_MAX_LINKS (256UL)
17 : /* Maximum number of tiles that may be present in a topology. */
18 0 : #define FD_TOPO_MAX_TILES (256UL)
19 : /* Maximum number of objects that may be present in a topology. */
20 : #define FD_TOPO_MAX_OBJS (4096UL)
21 : /* Maximum number of links that may go into any one tile in the
22 : topology. */
23 : #define FD_TOPO_MAX_TILE_IN_LINKS ( 128UL)
24 : /* Maximum number of links that a tile may write to. */
25 : #define FD_TOPO_MAX_TILE_OUT_LINKS ( 32UL)
26 : /* Maximum number of objects that a tile can use. */
27 : #define FD_TOPO_MAX_TILE_OBJS ( 256UL)
28 :
29 : /* Maximum number of additional ip addresses */
30 : #define FD_NET_MAX_SRC_ADDR 4
31 :
32 : /* Maximum number of additional destinations for leader shreds and for retransmitted shreds */
33 : #define FD_TOPO_ADTL_DESTS_MAX ( 32UL)
34 :
35 0 : #define FD_TOPO_CORE_DUMP_LEVEL_DISABLED (0)
36 0 : #define FD_TOPO_CORE_DUMP_LEVEL_MINIMAL (1)
37 6 : #define FD_TOPO_CORE_DUMP_LEVEL_REGULAR (2)
38 0 : #define FD_TOPO_CORE_DUMP_LEVEL_FULL (3)
39 0 : #define FD_TOPO_CORE_DUMP_LEVEL_NEVER (4)
40 :
41 : /* A workspace is a Firedancer specific memory management structure that
42 : sits on top of 1 or more memory mapped gigantic or huge pages mounted
43 : to the hugetlbfs. */
44 : typedef struct {
45 : ulong id; /* The ID of this workspace. Indexed from [0, wksp_cnt). When placed in a topology, the ID must be the index of the workspace in the workspaces list. */
46 : char name[ 14UL ]; /* The name of this workspace, like "pack". There can be at most one of each workspace name in a topology. */
47 :
48 : ulong numa_idx; /* The index of the NUMA node on the system that this workspace should be allocated from. */
49 :
50 : ulong min_part_max; /* Artificially raise part_max */
51 : ulong min_loose_sz; /* Artificially raise loose footprint */
52 :
53 : /* Computed fields. These are not supplied as configuration but calculated as needed. */
54 : struct {
55 : ulong page_sz; /* The size of the pages that this workspace is backed by. One of FD_PAGE_SIZE_*. */
56 : ulong page_cnt; /* The number of pages that must be mapped to this workspace to store all the data needed by consumers. */
57 : ulong part_max; /* The maximum number of partitions in the underlying workspace. There can only be this many allocations made at any one time. */
58 :
59 : int core_dump_level; /* The core dump level required to be set in the application configuration to have this workspace appear in core dumps. */
60 :
61 : fd_wksp_t * wksp; /* The workspace memory in the local process. */
62 : ulong known_footprint; /* Total size in bytes of all data in Firedancer that will be stored in this workspace at startup. */
63 : ulong total_footprint; /* Total size in bytes of all data in Firedancer that could be stored in this workspace, includes known data and loose data. */
64 : };
65 : } fd_topo_wksp_t;
66 :
67 : /* A link is an mcache in a workspace that has one producer and one or
68 : more consumers. A link may optionally also have a dcache, that holds
69 : fragments referred to by the mcache entries.
70 :
71 : A link belongs to exactly one workspace. A link has exactly one
72 : producer, and 1 or more consumers. Each consumer is either reliable
73 : or not reliable. A link has a depth and a MTU, which correspond to
74 : the depth and MTU of the mcache and dcache respectively. A MTU of
75 : zero means no dcache is needed, as there is no data. */
76 : typedef struct {
77 : ulong id; /* The ID of this link. Indexed from [0, link_cnt). When placed in a topology, the ID must be the index of the link in the links list. */
78 : char name[ 14UL ]; /* The name of this link, like "pack_execle". There can be multiple of each link name in a topology. */
79 : ulong kind_id; /* The ID of this link within its name. If there are N links of a particular name, they have IDs [0, N). The pair (name, kind_id) uniquely identifies a link, as does "id" on its own. */
80 :
81 : ulong depth; /* The depth of the mcache representing the link. */
82 : ulong mtu; /* The MTU of data fragments in the mcache. A value of 0 means there is no dcache. */
83 : ulong burst; /* The max amount of MTU sized data fragments that might be bursted to the dcache. */
84 :
85 : ulong mcache_obj_id;
86 : ulong dcache_obj_id;
87 :
88 : /* Computed fields. These are not supplied as configuration but calculated as needed. */
89 : struct {
90 : fd_frag_meta_t * mcache; /* The mcache of this link. */
91 : void * dcache; /* The dcache of this link, if it has one. */
92 : };
93 :
94 : uint permit_no_consumers : 1; /* Permit a topology where this link has no consumers */
95 : uint permit_no_producers : 1; /* Permit a topology where this link has no producers */
96 : } fd_topo_link_t;
97 :
98 : /* Be careful: ip and host are in different byte order */
99 : typedef struct {
100 : uint ip; /* in network byte order */
101 : ushort port; /* in host byte order */
102 : } fd_topo_ip_port_t;
103 :
104 : struct fd_topo_net_tile {
105 : ulong umem_dcache_obj_id; /* dcache for XDP UMEM frames */
106 : uint bind_address;
107 :
108 : ushort shred_listen_port;
109 : ushort quic_transaction_listen_port;
110 : ushort legacy_transaction_listen_port;
111 : ushort gossip_listen_port;
112 : ushort repair_client_listen_port;
113 : ushort repair_serve_listen_port;
114 : ushort txsend_src_port;
115 : };
116 : typedef struct fd_topo_net_tile fd_topo_net_tile_t;
117 :
118 : /* A tile is a unique process that is spawned by Firedancer to represent
119 : one thread of execution. Firedancer sandboxes all tiles to their own
120 : process for security reasons.
121 :
122 : A tile belongs to exactly one workspace. A tile is a consumer of 0
123 : or more links, it's inputs. A tile is a producer of 0 or more output
124 : links.
125 :
126 : All input links will be automatically polled by the tile
127 : infrastructure, and output links will automatically source and manage
128 : credits from consumers. */
129 : struct fd_topo_tile {
130 : ulong id; /* The ID of this tile. Indexed from [0, tile_cnt). When placed in a topology, the ID must be the index of the tile in the tiles list. */
131 : char name[ 7UL ]; /* The name of this tile. There can be multiple of each tile name in a topology. */
132 : ulong kind_id; /* The ID of this tile within its name. If there are n tile of a particular name, they have IDs [0, N). The pair (name, kind_id) uniquely identifies a tile, as does "id" on its own. */
133 : int is_agave; /* If the tile needs to run in the Agave (Anza) address space or not. */
134 : int allow_shutdown; /* If the tile is allowed to shutdown gracefully. If false, when the tile exits it will tear down the entire application. */
135 :
136 : ulong cpu_idx; /* The CPU index to pin the tile on. A value of ULONG_MAX or more indicates the tile should be floating and not pinned to a core. */
137 :
138 : ulong in_cnt; /* The number of links that this tile reads from. */
139 : ulong in_link_id[ FD_TOPO_MAX_TILE_IN_LINKS ]; /* The link_id of each link that this tile reads from, indexed in [0, in_cnt). */
140 : int in_link_reliable[ FD_TOPO_MAX_TILE_IN_LINKS ]; /* If each link that this tile reads from is a reliable or unreliable consumer, indexed in [0, in_cnt). */
141 : int in_link_poll[ FD_TOPO_MAX_TILE_IN_LINKS ]; /* If each link that this tile reads from should be polled by the tile infrastructure, indexed in [0, in_cnt).
142 : If the link is not polled, the tile will not receive frags for it and the tile writer is responsible for
143 : reading from the link. The link must be marked as unreliable as it is not flow controlled. */
144 :
145 : ulong out_cnt; /* The number of links that this tile writes to. */
146 : ulong out_link_id[ FD_TOPO_MAX_TILE_OUT_LINKS ]; /* The link_id of each link that this tile writes to, indexed in [0, link_cnt). */
147 :
148 : ulong event_link_id; /* If not ULONG_MAX, the link_id of a dedicated unreliable link to the event tile that this tile reports
149 : telemetry events on via the thread-local fd_event_report_* macros. This link is deliberately NOT part
150 : of out_link_id[] / out_cnt: it is written directly (outside fd_stem) by the thread-local reporter. */
151 :
152 : ulong tile_obj_id;
153 : ulong metrics_obj_id;
154 : ulong id_keyswitch_obj_id; /* keyswitch object id for identity key updates */
155 : ulong av_keyswitch_obj_id; /* keyswitch object id for authority key updates */
156 : ulong in_link_fseq_obj_id[ FD_TOPO_MAX_TILE_IN_LINKS ];
157 :
158 : ulong uses_obj_cnt;
159 : ulong uses_obj_id[ FD_TOPO_MAX_TILE_OBJS ];
160 : int uses_obj_mode[ FD_TOPO_MAX_TILE_OBJS ];
161 :
162 : /* Computed fields. These are not supplied as configuration but calculated as needed. */
163 : struct {
164 : ulong * metrics; /* The shared memory for metrics that this tile should write. Consumer by monitoring and metrics writing tiles. */
165 :
166 : /* The fseq of each link that this tile reads from. Multiple fseqs
167 : may point to the link, if there are multiple consumers. An fseq
168 : can be uniquely identified via (link_id, tile_id), or (link_kind,
169 : link_kind_id, tile_kind, tile_kind_id) */
170 : ulong * in_link_fseq[ FD_TOPO_MAX_TILE_IN_LINKS ];
171 : };
172 :
173 : /* Configuration fields. These are required to be known by the topology so it can determine the
174 : total size of Firedancer in memory. */
175 : union {
176 : fd_topo_net_tile_t net;
177 :
178 : struct {
179 : fd_topo_net_tile_t net;
180 :
181 : char if_virt[ 16 ]; /* device name (virtual, for routing) */
182 : char if_phys[ 16 ]; /* device name (physical, for RX/TX) */
183 : uint if_queue; /* device queue index */
184 :
185 : /* xdp specific options */
186 : ulong xdp_rx_queue_size;
187 : ulong xdp_tx_queue_size;
188 : ulong free_ring_depth;
189 : long tx_flush_timeout_ns;
190 : char xdp_mode[8];
191 : int zero_copy;
192 :
193 : char poll_mode[ 16 ]; /* "softirq" or "prefbusy" */
194 :
195 : ulong netdev_tbl_obj_id;
196 :
197 : ulong route_max;
198 : ulong route_peer_max;
199 : ulong route_peer_seed;
200 : ulong neigh4_obj_id; /* neigh4 hash map */
201 :
202 : int xsk_core_dump;
203 : } xdp;
204 :
205 : struct {
206 : fd_topo_net_tile_t net;
207 : /* sock specific options */
208 : int so_sndbuf;
209 : int so_rcvbuf;
210 : } sock;
211 :
212 : struct {
213 : ulong netdev_tbl_obj_id;
214 : ulong route_max;
215 : ulong route_peer_max;
216 : char neigh_if[ 16 ]; /* neigh4 interface name */
217 : ulong neigh4_obj_id; /* neigh4 hash map */
218 : } netlink;
219 :
220 : struct {
221 : char identity_key_path[ PATH_MAX ];
222 : } admin;
223 :
224 0 : #define FD_TOPO_GOSSIP_ENTRYPOINTS_MAX 16UL
225 :
226 : struct {
227 : char identity_key_path[ PATH_MAX ];
228 :
229 : ulong entrypoints_cnt;
230 : char entrypoints[ FD_TOPO_GOSSIP_ENTRYPOINTS_MAX ][ 262 ];
231 :
232 : long boot_timestamp_nanos;
233 :
234 : ulong tcache_depth;
235 :
236 : ushort shred_version;
237 : int allow_private_address;
238 :
239 : char gossip_host[ 256 ];
240 : fd_ip4_port_t gossip_addr;
241 : fd_ip4_port_t src_addr;
242 : } gossvf;
243 :
244 : struct {
245 : char identity_key_path[ PATH_MAX ];
246 :
247 : ulong entrypoints_cnt;
248 : char entrypoints[ FD_TOPO_GOSSIP_ENTRYPOINTS_MAX ][ 262 ];
249 :
250 : long boot_timestamp_nanos;
251 :
252 : char gossip_host[ 256 ];
253 : uint net_ip_addr; /* net.ip_addr fallback when gossip_host empty */
254 : uint ip_addr;
255 : uint bind_ip_addr;
256 : ushort shred_version;
257 :
258 : ulong max_entries;
259 : ulong max_purged;
260 : ulong max_failed;
261 :
262 : fd_hash_t wait_for_supermajority_with_bank_hash;
263 :
264 : struct {
265 : ushort gossip;
266 : ushort tvu;
267 : ushort tvu_quic;
268 : ushort tpu;
269 : ushort tpu_quic;
270 : ushort repair;
271 : ushort rserve;
272 : } ports;
273 : } gossip;
274 :
275 : struct {
276 : uint out_depth;
277 : uint reasm_cnt;
278 : ulong max_concurrent_connections;
279 : ulong max_concurrent_handshakes;
280 : ushort quic_transaction_listen_port;
281 : long idle_timeout_millis;
282 : uint ack_delay_millis;
283 : int retry;
284 : char key_log_path[ PATH_MAX ];
285 : } quic;
286 :
287 : struct {
288 : ulong tcache_depth;
289 : } verify;
290 :
291 : struct {
292 : ulong tcache_depth;
293 : } dedup;
294 :
295 : struct {
296 : char url[ 256 ];
297 : ulong url_len;
298 : char sni[ 256 ];
299 : ulong sni_len;
300 : char identity_key_path[ PATH_MAX ];
301 : char key_log_path[ PATH_MAX ];
302 : ulong buf_sz;
303 : ulong out_depth;
304 : ulong ssl_heap_sz;
305 : ulong keepalive_interval_nanos;
306 : uchar tls_cert_verify : 1;
307 : } bundle;
308 :
309 : struct {
310 : char url[ 256 ];
311 : char identity_key_path[ PATH_MAX ];
312 : char action[ 16 ];
313 : uchar genesis_hash[ 32 ];
314 : ushort shred_version;
315 : } event;
316 :
317 : struct {
318 : ulong max_pending_transactions;
319 : ulong execle_tile_count;
320 : int larger_max_cost_per_block;
321 : int larger_shred_limits_per_block;
322 : int use_consumed_cus;
323 : int schedule_strategy;
324 : struct {
325 : int enabled;
326 : uchar tip_distribution_program_addr[ 32 ];
327 : uchar tip_payment_program_addr[ 32 ];
328 : uchar tip_distribution_authority[ 32 ];
329 : ulong commission_bps;
330 : char identity_key_path[ PATH_MAX ];
331 : char vote_account_path[ PATH_MAX ]; /* or pubkey is okay */
332 : } bundle;
333 : ulong acct_blocklist_cnt;
334 : fd_pubkey_t acct_blocklist[ FD_PACK_ACCT_BLOCKLIST_MAX ];
335 : } pack;
336 :
337 : struct {
338 : int lagged_consecutive_leader_start;
339 : int plugins_enabled;
340 : ulong execle_cnt;
341 : char identity_key_path[ PATH_MAX ];
342 : struct {
343 : int enabled;
344 : uchar tip_payment_program_addr[ 32 ];
345 : uchar tip_distribution_program_addr[ 32 ];
346 : char vote_account_path[ PATH_MAX ];
347 : } bundle;
348 : } pohh;
349 :
350 : struct {
351 : ulong execle_cnt;
352 : char identity_key_path[ PATH_MAX ];
353 : } poh;
354 :
355 : struct {
356 : ulong depth;
357 : ulong fec_resolver_depth;
358 : char identity_key_path[ PATH_MAX ];
359 : ushort shred_listen_port;
360 : int larger_shred_limits_per_block;
361 : ushort expected_shred_version;
362 : ulong adtl_dests_retransmit_cnt;
363 : fd_topo_ip_port_t adtl_dests_retransmit[ FD_TOPO_ADTL_DESTS_MAX ];
364 : ulong adtl_dests_leader_cnt;
365 : fd_topo_ip_port_t adtl_dests_leader[ FD_TOPO_ADTL_DESTS_MAX ];
366 : } shred;
367 :
368 : struct {
369 : ulong disable_blockstore_from_slot;
370 : } store;
371 :
372 : struct {
373 : char identity_key_path[ PATH_MAX ];
374 : ulong authorized_voter_paths_cnt;
375 : char authorized_voter_paths[ 16 ][ PATH_MAX ];
376 : } sign;
377 :
378 : struct {
379 : uint listen_addr;
380 : ushort listen_port;
381 :
382 : int is_voting;
383 :
384 : char cluster[ 32 ];
385 : char identity_key_path[ PATH_MAX ];
386 : char vote_key_path[ PATH_MAX ];
387 : char accounts_database_path[ PATH_MAX ];
388 : char gui_database_path[ PATH_MAX ];
389 :
390 : ulong max_http_connections;
391 : ulong max_websocket_connections;
392 : ulong max_http_request_length;
393 : ulong send_buffer_size_mb;
394 : ulong db_size_gib;
395 : int schedule_strategy;
396 :
397 : int websocket_compression;
398 : ulong tile_cnt;
399 :
400 : char wfs_bank_hash[ FD_BASE58_ENCODED_32_SZ ];
401 : ushort expected_shred_version;
402 : ulong cache_size_gib;
403 : ulong accdb_obj_id;
404 : } gui;
405 :
406 : struct {
407 : fd_ip6_addr_t listen_addr;
408 : ushort listen_port;
409 :
410 : ulong max_http_connections;
411 : ulong max_websocket_connections;
412 : ulong send_buffer_size_mb;
413 : ulong max_http_request_length;
414 :
415 : ulong max_live_slots;
416 :
417 : ulong accdb_obj_id;
418 : ulong accdb_epoch_fseq_obj_id;
419 :
420 : char identity_key_path[ PATH_MAX ];
421 : int delay_startup;
422 :
423 : int snapshot_server_enabled;
424 : char snapshot_server_host[ 256 ];
425 : ushort snapshot_server_port;
426 : } rpc;
427 :
428 : struct {
429 : uint prometheus_listen_addr;
430 : ushort prometheus_listen_port;
431 : } metric;
432 :
433 : struct {
434 : int is_voting;
435 : } diag;
436 :
437 : struct {
438 : ulong fec_max;
439 :
440 : ulong accdb_obj_id;
441 : ulong txncache_obj_id;
442 :
443 : char shred_cap[ PATH_MAX ];
444 :
445 : char identity_key_path[ PATH_MAX ];
446 : uint ip_addr;
447 : char vote_account_path[ PATH_MAX ];
448 :
449 : fd_hash_t wait_for_supermajority_with_bank_hash;
450 : ushort expected_shred_version;
451 : int wait_for_vote_to_start_leader;
452 :
453 : ulong heap_size_gib;
454 : ulong sched_depth;
455 : ulong max_live_slots;
456 : ulong full_snapshot_interval_slots;
457 : ulong incremental_snapshot_interval_slots;
458 :
459 : /* not specified in TOML */
460 :
461 : long boot_timestamp_nanos;
462 :
463 : ulong enable_features_cnt;
464 : char enable_features[ 16 ][ FD_BASE58_ENCODED_32_SZ ];
465 :
466 : char genesis_path[ PATH_MAX ];
467 :
468 : int larger_max_cost_per_block;
469 :
470 : ulong capture_start_slot;
471 : char solcap_capture[ PATH_MAX ];
472 : char dump_proto_dir[ PATH_MAX ];
473 : int dump_block_to_pb;
474 :
475 : struct {
476 : int enabled;
477 : uchar tip_payment_program_addr[ 32 ];
478 : uchar tip_distribution_program_addr[ 32 ];
479 : char vote_account_path[ PATH_MAX ];
480 : } bundle;
481 :
482 : } replay;
483 :
484 : struct {
485 : ulong txncache_obj_id;
486 : ulong progcache_obj_id;
487 : ulong accdb_obj_id;
488 :
489 : ulong max_live_slots;
490 :
491 : ulong capture_start_slot;
492 : char solcap_capture[ PATH_MAX ];
493 : char dump_proto_dir[ PATH_MAX ];
494 : char dump_syscall_name_filter[ PATH_MAX ];
495 : char dump_instr_program_id_filter[ FD_BASE58_ENCODED_32_SZ ];
496 : int dump_instr_to_pb;
497 : int dump_txn_to_pb;
498 : int dump_txn_as_fixture;
499 : int dump_syscall_to_pb;
500 : int report_transaction_diffs;
501 : } execrp;
502 :
503 : struct {
504 : ushort send_to_port;
505 : uint send_to_ip_addr;
506 : ulong conn_cnt;
507 : int no_quic;
508 : } benchs;
509 :
510 : struct {
511 : ushort rpc_port;
512 : uint rpc_ip_addr;
513 : } bencho;
514 :
515 : struct {
516 : ulong accounts_cnt;
517 : int mode;
518 : float contending_fraction;
519 : float cu_price_spread;
520 : } benchg;
521 :
522 : struct {
523 : ushort repair_client_listen_port;
524 : char identity_key_path[ PATH_MAX ];
525 : ulong max_pending_shred_sets;
526 : ulong slot_max;
527 :
528 : /* non-config */
529 :
530 : ulong repair_sign_depth;
531 : ulong repair_sign_cnt;
532 : } repair;
533 :
534 : struct {
535 : ushort repair_serve_listen_port;
536 : char identity_key_path[ PATH_MAX ];
537 : char shredb_path[ PATH_MAX ];
538 : ulong shred_storage_limit_gib;
539 : ulong ping_cache_entries;
540 : } rserve;
541 :
542 : struct {
543 : ushort txsend_src_port;
544 :
545 : /* non-config */
546 :
547 : uint ip_addr;
548 : char identity_key_path[ PATH_MAX ];
549 : } txsend;
550 :
551 : struct {
552 : uint fake_dst_ip;
553 : } pktgen;
554 :
555 : struct {
556 : char ledger_format[ 16 ];
557 : char ledger_path[ PATH_MAX ];
558 : ulong end_slot;
559 : ulong root_distance;
560 : } backtest;
561 :
562 : struct {
563 : char ledger_format[ 16 ];
564 : char ledger_path[ PATH_MAX ];
565 : ulong end_slot;
566 : ushort shred_listen_port;
567 : } forktest;
568 :
569 : struct {
570 : ulong accdb_obj_id;
571 :
572 : ulong authorized_voter_paths_cnt;
573 : char authorized_voter_paths[ 16 ][ PATH_MAX ];
574 : int hard_fork_fatal;
575 : int wait_for_supermajority;
576 : ulong max_live_slots;
577 : char identity_key[ PATH_MAX ];
578 : char vote_account[ PATH_MAX ];
579 : char base_path[PATH_MAX];
580 : } tower;
581 :
582 : struct {
583 : ulong accdb_obj_id;
584 : ulong max_live_slots;
585 :
586 : ulong rpc_epoch_obj_id;
587 : ulong resolv_epoch_obj_ids[ 16 ];
588 : ulong resolv_epoch_obj_cnt;
589 : ulong snapmk_epoch_obj_id;
590 : ulong snapzp_epoch_obj_ids[ 64 ];
591 : ulong snapzp_epoch_obj_cnt;
592 : } accdb;
593 :
594 : struct {
595 : ulong max_live_slots;
596 : ulong accdb_obj_id;
597 : ulong accdb_epoch_fseq_obj_id;
598 : } resolv;
599 :
600 :
601 : #define FD_TOPO_SNAPSHOTS_GOSSIP_LIST_MAX (32UL)
602 27 : #define FD_TOPO_SNAPSHOTS_SERVERS_MAX (16UL)
603 27 : #define FD_TOPO_MAX_RESOLVED_ADDRS ( 4UL)
604 27 : #define FD_TOPO_SNAPSHOTS_SERVERS_MAX_RESOLVED (FD_TOPO_MAX_RESOLVED_ADDRS*FD_TOPO_SNAPSHOTS_SERVERS_MAX)
605 :
606 : struct fd_topo_tile_snapct {
607 : char snapshots_path[ PATH_MAX ];
608 :
609 : ulong entrypoints_cnt;
610 : char entrypoints[ FD_TOPO_GOSSIP_ENTRYPOINTS_MAX ][ 262 ];
611 :
612 : struct {
613 : uint max_local_full_effective_age;
614 : uint max_local_incremental_age;
615 :
616 : struct {
617 : int allow_any;
618 : ulong allow_list_cnt;
619 : fd_pubkey_t allow_list[ FD_TOPO_SNAPSHOTS_GOSSIP_LIST_MAX ];
620 : ulong block_list_cnt;
621 : fd_pubkey_t block_list[ FD_TOPO_SNAPSHOTS_GOSSIP_LIST_MAX ];
622 : } gossip;
623 :
624 : ulong servers_cnt;
625 : char servers[ FD_TOPO_SNAPSHOTS_SERVERS_MAX ][ 128 ];
626 : } sources;
627 :
628 : int incremental_snapshots;
629 : uint max_full_snapshots_to_keep;
630 : uint max_incremental_snapshots_to_keep;
631 : uint max_retry_abort;
632 : long wait_for_peers_timeout_nanos;
633 : } snapct;
634 :
635 : struct {
636 : char snapshots_path[ PATH_MAX ];
637 : int incremental_snapshots;
638 : uint min_download_speed_mibs;
639 : } snapld;
640 :
641 : struct {
642 : ulong max_live_slots;
643 : ulong accdb_obj_id;
644 : ulong txncache_obj_id;
645 : ulong banks_obj_id;
646 : } snapin;
647 :
648 : struct {
649 : ulong partition_sz;
650 : } snapwr;
651 :
652 : struct {
653 :
654 : uint bind_address;
655 : ushort bind_port;
656 :
657 : ushort expected_shred_version;
658 : ulong entrypoints_cnt;
659 : char entrypoints[ FD_TOPO_GOSSIP_ENTRYPOINTS_MAX ][ 262 ];
660 : } ipecho;
661 :
662 : struct {
663 : ulong max_live_slots;
664 : ulong txncache_obj_id;
665 : ulong progcache_obj_id;
666 : ulong accdb_obj_id;
667 : int report_transaction_diffs;
668 : } execle;
669 :
670 : struct {
671 : int validate_genesis_hash;
672 : int allow_download;
673 :
674 : ushort expected_shred_version;
675 : ulong entrypoints_cnt;
676 : char entrypoints[ FD_TOPO_GOSSIP_ENTRYPOINTS_MAX ][ 262 ];
677 :
678 : int has_expected_genesis_hash;
679 : uchar expected_genesis_hash[ 32UL ];
680 :
681 : char genesis_path[ PATH_MAX ];
682 :
683 : uint target_gid;
684 : uint target_uid;
685 :
686 : ulong max_live_slots;
687 : ulong accdb_obj_id;
688 : } genesi;
689 :
690 : struct {
691 : ulong capture_start_slot;
692 : char solcap_capture[ PATH_MAX ];
693 : int recent_only;
694 : ulong recent_slots_per_file;
695 : } solcap;
696 :
697 : struct {
698 : ulong accdb_obj_id;
699 : ulong accdb_epoch_obj_id;
700 : ulong visited_set_obj_id;
701 : ulong banks_obj_id;
702 : ulong zp_fseq_id;
703 : ulong txncache_obj_id;
704 : ulong max_accounts;
705 : ulong max_live_slots;
706 : uint max_full_snapshots_to_keep;
707 : char snapshots_path[ PATH_MAX ];
708 : uint max_incremental_snapshots_to_keep;
709 : } snapmk;
710 :
711 : struct {
712 : ulong accdb_obj_id;
713 : ulong accdb_epoch_obj_id;
714 : ulong visited_set_obj_id;
715 : ulong zp_fseq_id;
716 : ulong max_live_slots;
717 : uint snap_fd_cnt;
718 : } snapzp;
719 :
720 : struct {
721 : ulong accdb_obj_id;
722 : } snaprd;
723 :
724 : struct {
725 : ulong snap_max;
726 : ulong conn_max;
727 : uint io_worker_cnt;
728 : ulong idle_timeout_millis;
729 : ulong send_timeout_millis;
730 : ulong send_buffer_size_kib;
731 : fd_ip6_addr_t listen_addr;
732 : ushort listen_port;
733 : } snapsv;
734 : };
735 : };
736 :
737 : typedef struct fd_topo_tile fd_topo_tile_t;
738 :
739 : typedef struct {
740 : ulong id;
741 : char name[ 13UL ]; /* object type */
742 : ulong wksp_id;
743 :
744 : /* Optional label for object */
745 : char label[ 13UL ]; /* object label */
746 : ulong label_idx; /* index of object for this label (ULONG_MAX if not labelled) */
747 :
748 : ulong offset;
749 : ulong footprint;
750 : } fd_topo_obj_t;
751 :
752 : /* An fd_topo_t represents the overall structure of a Firedancer
753 : configuration, describing all the workspaces, tiles, and links
754 : between them. */
755 : struct fd_topo {
756 : char app_name[ 256UL ];
757 : uchar props[ 32768UL ];
758 :
759 : ulong wksp_cnt;
760 : ulong link_cnt;
761 : ulong tile_cnt;
762 : ulong obj_cnt;
763 :
764 : fd_topo_wksp_t workspaces[ FD_TOPO_MAX_WKSPS ];
765 : fd_topo_link_t links[ FD_TOPO_MAX_LINKS ];
766 : fd_topo_tile_t tiles[ FD_TOPO_MAX_TILES ];
767 : fd_topo_obj_t objs[ FD_TOPO_MAX_OBJS ];
768 :
769 : ulong agave_affinity_cnt;
770 : ulong agave_affinity_cpu_idx[ FD_TILE_MAX ];
771 : ulong blocklist_cores_cnt;
772 : ulong blocklist_cores_cpu_idx[ FD_TILE_MAX ];
773 :
774 : ulong max_page_size; /* 2^21 or 2^30 */
775 : ulong gigantic_page_threshold; /* see [hugetlbfs.gigantic_page_threshold_mib]*/
776 :
777 : ulong layout_hash;
778 : };
779 : typedef struct fd_topo fd_topo_t;
780 :
781 : typedef struct {
782 : char const * name;
783 :
784 : int keep_host_networking;
785 : int allow_connect;
786 : int allow_renameat;
787 : ulong rlimit_file_cnt;
788 : ulong rlimit_address_space;
789 : ulong rlimit_data;
790 : ulong rlimit_nproc;
791 : int for_tpool;
792 :
793 : ulong (*max_event_sz )( fd_topo_tile_t const * tile );
794 : ulong (*populate_allowed_seccomp)( fd_topo_t const * topo, fd_topo_tile_t const * tile, ulong out_cnt, struct sock_filter * out );
795 : ulong (*populate_allowed_fds )( fd_topo_t const * topo, fd_topo_tile_t const * tile, ulong out_fds_sz, int * out_fds );
796 : ulong (*scratch_align )( void );
797 : ulong (*scratch_footprint )( fd_topo_tile_t const * tile );
798 : ulong (*loose_footprint )( fd_topo_tile_t const * tile );
799 : void (*privileged_init )( fd_topo_t const * topo, fd_topo_tile_t const * tile );
800 : void (*unprivileged_init )( fd_topo_t const * topo, fd_topo_tile_t const * tile );
801 : void (*run )( fd_topo_t * topo, fd_topo_tile_t * tile );
802 : ulong (*rlimit_file_cnt_fn )( fd_topo_t const * topo, fd_topo_tile_t const * tile );
803 : } fd_topo_run_tile_t;
804 :
805 : struct fd_topo_obj_callbacks {
806 : char const * name;
807 : ulong (* footprint )( fd_topo_t const * topo, fd_topo_obj_t const * obj );
808 : ulong (* align )( fd_topo_t const * topo, fd_topo_obj_t const * obj );
809 : ulong (* loose )( fd_topo_t const * topo, fd_topo_obj_t const * obj );
810 : void (* new )( fd_topo_t const * topo, fd_topo_obj_t const * obj );
811 : };
812 :
813 : typedef struct fd_topo_obj_callbacks fd_topo_obj_callbacks_t;
814 :
815 : FD_PROTOTYPES_BEGIN
816 :
817 : FD_FN_CONST static inline ulong
818 0 : fd_topo_workspace_align( void ) {
819 : /* This needs to be the max( align ) of all the child members that
820 : could be aligned into this workspace, otherwise our footprint
821 : calculation will not be correct. For now just set to 4096 but this
822 : should probably be calculated dynamically, or we should reduce
823 : those child aligns if we can. */
824 0 : return 4096UL;
825 0 : }
826 :
827 : void *
828 : fd_topo_obj_laddr( fd_topo_t const * topo,
829 : ulong obj_id );
830 :
831 : /* Returns a pointer in the local address space to the base address of
832 : the workspace out of which the given object was allocated. */
833 :
834 : static inline void *
835 : fd_topo_obj_wksp_base( fd_topo_t const * topo,
836 0 : ulong obj_id ) {
837 0 : FD_TEST( obj_id<FD_TOPO_MAX_OBJS );
838 0 : fd_topo_obj_t const * obj = &topo->objs[ obj_id ];
839 0 : FD_TEST( obj->id == obj_id );
840 0 : ulong const wksp_id = obj->wksp_id;
841 :
842 0 : FD_TEST( wksp_id<FD_TOPO_MAX_WKSPS );
843 0 : fd_topo_wksp_t const * wksp = &topo->workspaces[ wksp_id ];
844 0 : FD_TEST( wksp->id == wksp_id );
845 0 : return wksp->wksp;
846 0 : }
847 :
848 : FD_FN_PURE static inline ulong
849 : fd_topo_tile_name_cnt( fd_topo_t const * topo,
850 3 : char const * name ) {
851 3 : ulong cnt = 0;
852 6 : for( ulong i=0; i<topo->tile_cnt; i++ ) {
853 3 : if( FD_UNLIKELY( !strcmp( topo->tiles[ i ].name, name ) ) ) cnt++;
854 3 : }
855 3 : return cnt;
856 3 : }
857 :
858 : /* Finds the workspace of a given name in the topology. Returns
859 : ULONG_MAX if there is no such workspace. There can be at most one
860 : workspace of a given name. */
861 :
862 : FD_FN_PURE static inline ulong
863 : fd_topo_find_wksp( fd_topo_t const * topo,
864 60 : char const * name ) {
865 60 : for( ulong i=0; i<topo->wksp_cnt; i++ ) {
866 60 : if( FD_UNLIKELY( !strcmp( topo->workspaces[ i ].name, name ) ) ) return i;
867 60 : }
868 0 : return ULONG_MAX;
869 60 : }
870 :
871 : /* Find the tile of a given name and kind_id in the topology, there will
872 : be at most one such tile, since kind_id is unique among the name.
873 : Returns ULONG_MAX if there is no such tile. */
874 :
875 : FD_FN_PURE static inline ulong
876 : fd_topo_find_tile( fd_topo_t const * topo,
877 : char const * name,
878 21 : ulong kind_id ) {
879 21 : for( ulong i=0; i<topo->tile_cnt; i++ ) {
880 21 : if( FD_UNLIKELY( !strcmp( topo->tiles[ i ].name, name ) ) && topo->tiles[ i ].kind_id == kind_id ) return i;
881 21 : }
882 0 : return ULONG_MAX;
883 21 : }
884 :
885 : /* Find the link of a given name and kind_id in the topology, there will
886 : be at most one such link, since kind_id is unique among the name.
887 : Returns ULONG_MAX if there is no such link. */
888 :
889 : FD_FN_PURE static inline ulong
890 : fd_topo_find_link( fd_topo_t const * topo,
891 : char const * name,
892 18 : ulong kind_id ) {
893 39 : for( ulong i=0; i<topo->link_cnt; i++ ) {
894 39 : if( FD_UNLIKELY( !strcmp( topo->links[ i ].name, name ) ) && topo->links[ i ].kind_id == kind_id ) return i;
895 39 : }
896 0 : return ULONG_MAX;
897 18 : }
898 :
899 : FD_FN_PURE static inline ulong
900 : fd_topo_find_tile_in_link( fd_topo_t const * topo,
901 : fd_topo_tile_t const * tile,
902 : char const * name,
903 0 : ulong kind_id ) {
904 0 : for( ulong i=0; i<tile->in_cnt; i++ ) {
905 0 : if( FD_UNLIKELY( !strcmp( topo->links[ tile->in_link_id[ i ] ].name, name ) )
906 0 : && topo->links[ tile->in_link_id[ i ] ].kind_id == kind_id ) return i;
907 0 : }
908 0 : return ULONG_MAX;
909 0 : }
910 :
911 : FD_FN_PURE static inline ulong
912 : fd_topo_find_tile_out_link( fd_topo_t const * topo,
913 : fd_topo_tile_t const * tile,
914 : char const * name,
915 0 : ulong kind_id ) {
916 0 : for( ulong i=0; i<tile->out_cnt; i++ ) {
917 0 : if( FD_UNLIKELY( !strcmp( topo->links[ tile->out_link_id[ i ] ].name, name ) )
918 0 : && topo->links[ tile->out_link_id[ i ] ].kind_id == kind_id ) return i;
919 0 : }
920 0 : return ULONG_MAX;
921 0 : }
922 :
923 : /* Find the id of the tile which is a producer for the given link. If
924 : no tile is a producer for the link, returns ULONG_MAX. This should
925 : not be possible for a well formed and validated topology. */
926 : FD_FN_PURE static inline ulong
927 : fd_topo_find_link_producer( fd_topo_t const * topo,
928 0 : fd_topo_link_t const * link ) {
929 0 : for( ulong i=0; i<topo->tile_cnt; i++ ) {
930 0 : fd_topo_tile_t const * tile = &topo->tiles[ i ];
931 :
932 0 : for( ulong j=0; j<tile->out_cnt; j++ ) {
933 0 : if( FD_UNLIKELY( tile->out_link_id[ j ] == link->id ) ) return i;
934 0 : }
935 0 : }
936 0 : return ULONG_MAX;
937 0 : }
938 :
939 : /* Given a link, count the number of consumers of that link among all
940 : the tiles in the topology. */
941 : FD_FN_PURE static inline ulong
942 : fd_topo_link_consumer_cnt( fd_topo_t const * topo,
943 0 : fd_topo_link_t const * link ) {
944 0 : ulong cnt = 0;
945 0 : for( ulong i=0; i<topo->tile_cnt; i++ ) {
946 0 : fd_topo_tile_t const * tile = &topo->tiles[ i ];
947 0 : for( ulong j=0; j<tile->in_cnt; j++ ) {
948 0 : if( FD_UNLIKELY( tile->in_link_id[ j ] == link->id ) ) cnt++;
949 0 : }
950 0 : }
951 :
952 0 : return cnt;
953 0 : }
954 :
955 : /* Given a link, count the number of reliable consumers of that link
956 : among all the tiles in the topology. */
957 : FD_FN_PURE static inline ulong
958 : fd_topo_link_reliable_consumer_cnt( fd_topo_t const * topo,
959 0 : fd_topo_link_t const * link ) {
960 0 : ulong cnt = 0;
961 0 : for( ulong i=0; i<topo->tile_cnt; i++ ) {
962 0 : fd_topo_tile_t const * tile = &topo->tiles[ i ];
963 0 : for( ulong j=0; j<tile->in_cnt; j++ ) {
964 0 : if( FD_UNLIKELY( tile->in_link_id[ j ] == link->id && tile->in_link_reliable[ j ] ) ) cnt++;
965 0 : }
966 0 : }
967 0 :
968 0 : return cnt;
969 0 : }
970 :
971 : FD_FN_PURE static inline ulong
972 : fd_topo_tile_consumer_cnt( fd_topo_t const * topo,
973 0 : fd_topo_tile_t const * tile ) {
974 0 : (void)topo;
975 0 : return tile->out_cnt;
976 0 : }
977 :
978 : FD_FN_PURE static inline ulong
979 : fd_topo_tile_reliable_consumer_cnt( fd_topo_t const * topo,
980 0 : fd_topo_tile_t const * tile ) {
981 0 : ulong reliable_cons_cnt = 0UL;
982 0 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
983 0 : fd_topo_tile_t const * consumer_tile = &topo->tiles[ i ];
984 0 : for( ulong j=0UL; j<consumer_tile->in_cnt; j++ ) {
985 0 : for( ulong k=0UL; k<tile->out_cnt; k++ ) {
986 0 : if( FD_UNLIKELY( consumer_tile->in_link_id[ j ]==tile->out_link_id[ k ] && consumer_tile->in_link_reliable[ j ] ) ) {
987 0 : reliable_cons_cnt++;
988 0 : }
989 0 : }
990 0 : }
991 0 : }
992 0 : return reliable_cons_cnt;
993 0 : }
994 :
995 : FD_FN_PURE static inline ulong
996 : fd_topo_tile_producer_cnt( fd_topo_t const * topo,
997 0 : fd_topo_tile_t const * tile ) {
998 0 : (void)topo;
999 0 : ulong in_cnt = 0UL;
1000 0 : for( ulong i=0UL; i<tile->in_cnt; i++ ) {
1001 0 : if( FD_UNLIKELY( !tile->in_link_poll[ i ] ) ) continue;
1002 0 : in_cnt++;
1003 0 : }
1004 0 : return in_cnt;
1005 0 : }
1006 :
1007 : FD_FN_PURE FD_FN_UNUSED static ulong
1008 : fd_topo_obj_cnt( fd_topo_t const * topo,
1009 : char const * obj_type,
1010 0 : char const * label ) {
1011 0 : ulong cnt = 0UL;
1012 0 : for( ulong i=0UL; i<topo->obj_cnt; i++ ) {
1013 0 : fd_topo_obj_t const * obj = &topo->objs[ i ];
1014 0 : if( strncmp( obj->name, obj_type, sizeof(obj->name) ) ) continue;
1015 0 : if( label &&
1016 0 : strncmp( obj->label, label, sizeof(obj->label) ) ) continue;
1017 0 : cnt++;
1018 0 : }
1019 0 : return cnt;
1020 0 : }
1021 :
1022 : FD_FN_PURE FD_FN_UNUSED static fd_topo_obj_t const *
1023 : fd_topo_find_obj( fd_topo_t const * topo,
1024 : char const * obj_type,
1025 : char const * label,
1026 0 : ulong label_idx ) {
1027 0 : for( ulong i=0UL; i<topo->obj_cnt; i++ ) {
1028 0 : fd_topo_obj_t const * obj = &topo->objs[ i ];
1029 0 : if( strncmp( obj->name, obj_type, sizeof(obj->name) ) ) continue;
1030 0 : if( label &&
1031 0 : strncmp( obj->label, label, sizeof(obj->label) ) ) continue;
1032 0 : if( label_idx != ULONG_MAX && obj->label_idx != label_idx ) continue;
1033 0 : return obj;
1034 0 : }
1035 0 : return NULL;
1036 0 : }
1037 :
1038 : FD_FN_PURE FD_FN_UNUSED static fd_topo_obj_t const *
1039 : fd_topo_find_tile_obj( fd_topo_t const * topo,
1040 : fd_topo_tile_t const * tile,
1041 0 : char const * obj_type ) {
1042 0 : for( ulong i=0UL; i<(tile->uses_obj_cnt); i++ ) {
1043 0 : fd_topo_obj_t const * obj = &topo->objs[ tile->uses_obj_id[ i ] ];
1044 0 : if( strncmp( obj->name, obj_type, sizeof(obj->name) ) ) continue;
1045 0 : return obj;
1046 0 : }
1047 0 : return NULL;
1048 0 : }
1049 :
1050 : /* Join (map into the process) all shared memory (huge/gigantic pages)
1051 : needed by the tile, in the given topology. All memory associated
1052 : with the tile (aka. used by links that the tile either produces to or
1053 : consumes from, or used by the tile itself for its cnc) will be
1054 : attached (mapped into the process).
1055 :
1056 : This is needed to play nicely with the sandbox. Once a process is
1057 : sandboxed we can no longer map any memory. */
1058 : void
1059 : fd_topo_join_tile_workspaces( fd_topo_t * topo,
1060 : fd_topo_tile_t * tile,
1061 : int core_dump_level );
1062 :
1063 : /* Join (map into the process) the shared memory (huge/gigantic pages)
1064 : for the given workspace. Mode is one of
1065 : FD_SHMEM_JOIN_MODE_READ_WRITE or FD_SHMEM_JOIN_MODE_READ_ONLY and
1066 : determines the prot argument that will be passed to mmap when mapping
1067 : the pages in (PROT_WRITE or PROT_READ respectively).
1068 :
1069 : Dump should be set to 1 if the workspace memory should be dumpable
1070 : when the process crashes, or 0 if not. */
1071 : void
1072 : fd_topo_join_workspace( fd_topo_t * topo,
1073 : fd_topo_wksp_t * wksp,
1074 : int mode,
1075 : int dump );
1076 :
1077 : /* Join (map into the process) all shared memory (huge/gigantic pages)
1078 : needed by all tiles in the topology. Mode is one of
1079 : FD_SHMEM_JOIN_MODE_READ_WRITE or FD_SHMEM_JOIN_MODE_READ_ONLY and
1080 : determines the prot argument that will be passed to mmap when
1081 : mapping the pages in (PROT_WRITE or PROT_READ respectively). */
1082 : void
1083 : fd_topo_join_workspaces( fd_topo_t * topo,
1084 : int mode,
1085 : int core_dump_level );
1086 :
1087 : /* Leave (unmap from the process) the shared memory needed for the
1088 : given workspace in the topology, if it was previously mapped.
1089 :
1090 : topo and wksp are assumed non-NULL. It is OK if the workspace
1091 : has not been previously joined, in which case this is a no-op. */
1092 :
1093 : void
1094 : fd_topo_leave_workspace( fd_topo_t * topo,
1095 : fd_topo_wksp_t * wksp );
1096 :
1097 : /* Leave (unmap from the process) all shared memory needed by all
1098 : tiles in the topology, if each of them was mapped.
1099 :
1100 : topo is assumed non-NULL. Only workspaces which were previously
1101 : joined are unmapped. */
1102 :
1103 : void
1104 : fd_topo_leave_workspaces( fd_topo_t * topo );
1105 :
1106 : /* Create the given workspace needed by the topology on the system.
1107 : This does not "join" the workspaces (map their memory into the
1108 : process), but only creates the .wksp file and formats it correctly
1109 : as a workspace.
1110 :
1111 : Returns 0 on success and -1 on failure, with errno set to the error.
1112 : The only reason for failure currently that will be returned is
1113 : ENOMEM, as other unexpected errors will cause the program to exit.
1114 :
1115 : If update_existing is 1, the workspace will not be created from
1116 : scratch but it will be assumed that it already exists from a prior
1117 : run and needs to be maybe resized and then have the header
1118 : structures reinitialized. This can save a very expensive operation
1119 : of zeroing all of the workspace pages. This is dangerous in
1120 : production because it can leave stray memory from prior runs around,
1121 : and should only be used in development environments. */
1122 :
1123 : int
1124 : fd_topo_create_workspace( fd_topo_t * topo,
1125 : fd_topo_wksp_t * wksp,
1126 : int update_existing );
1127 :
1128 : /* Join the standard IPC objects needed by the topology of this particular
1129 : tile */
1130 :
1131 : void
1132 : fd_topo_fill_tile( fd_topo_t * topo,
1133 : fd_topo_tile_t * tile );
1134 :
1135 : /* Same as fd_topo_fill_tile but fills in all the objects for a
1136 : particular workspace with the given mode. */
1137 : void
1138 : fd_topo_workspace_fill( fd_topo_t * topo,
1139 : fd_topo_wksp_t * wksp );
1140 :
1141 : /* Apply a new function to every object that is resident in the given
1142 : workspace in the topology. */
1143 :
1144 : void
1145 : fd_topo_wksp_new( fd_topo_t const * topo,
1146 : fd_topo_wksp_t const * wksp,
1147 : fd_topo_obj_callbacks_t ** callbacks );
1148 :
1149 : /* Same as fd_topo_fill_tile but fills in all tiles in the topology. */
1150 :
1151 : void
1152 : fd_topo_fill( fd_topo_t * topo );
1153 :
1154 : /* fd_topo_tile_stack_join joins a huge page optimized stack for the
1155 : provided tile. The stack is assumed to already exist at a known
1156 : path in the hugetlbfs mount. */
1157 :
1158 : void *
1159 : fd_topo_tile_stack_join( char const * app_name,
1160 : char const * tile_name,
1161 : ulong tile_kind_id );
1162 :
1163 : /* fd_topo_run_single_process runs all the tiles in a single process
1164 : (the calling process). This spawns a thread for each tile, switches
1165 : that thread to the given UID and GID and then runs the tile in it.
1166 : Each thread will never exit, as tiles are expected to run forever.
1167 : An error is logged and the application will exit if a tile exits.
1168 : The function itself does return after spawning all the threads.
1169 :
1170 : The threads will not be sandboxed in any way, except switching to the
1171 : provided UID and GID, so they will share the same address space, and
1172 : not have any seccomp restrictions or use any Linux namespaces. The
1173 : calling thread will also switch to the provided UID and GID before
1174 : it returns.
1175 :
1176 : In production, when running with an Agave child process this is
1177 : used for spawning certain tiles inside the Agave address space.
1178 : It's also useful for tooling and debugging, but is not how the main
1179 : production Firedancer process runs. For production, each tile is run
1180 : in its own address space with a separate process and full security
1181 : sandbox.
1182 :
1183 : The agave argument determines which tiles are started. If the
1184 : argument is 0 or 1, only non-agave (or only agave) tiles are started.
1185 : If the argument is any other value, all tiles in the topology are
1186 : started regardless of if they are Agave tiles or not. */
1187 :
1188 : void
1189 : fd_topo_run_single_process( fd_topo_t * topo,
1190 : int agave,
1191 : uint uid,
1192 : uint gid,
1193 : fd_topo_run_tile_t (* tile_run )( fd_topo_tile_t const * tile ) );
1194 :
1195 : /* fd_topo_run_tile runs the given tile directly within the current
1196 : process (and thread). The function will never return, as tiles are
1197 : expected to run forever. An error is logged and the application will
1198 : exit if the tile exits.
1199 :
1200 : The sandbox argument determines if the current process will be
1201 : sandboxed fully before starting the tile. The thread will switch to
1202 : the UID and GID provided before starting the tile, even if the thread
1203 : is not being sandboxed. Although POSIX specifies that all threads in
1204 : a process must share a UID and GID, this is not the case on Linux.
1205 : The thread will switch to the provided UID and GID without switching
1206 : the other threads in the process.
1207 :
1208 : If keep_controlling_terminal is set to 0, and the sandbox is enabled
1209 : the controlling terminal will be detached as an additional sandbox
1210 : measure, but you will not be able to send Ctrl+C or other signals
1211 : from the terminal. See fd_sandbox.h for more information.
1212 :
1213 : The allow_fd argument is only used if sandbox is true, and is a file
1214 : descriptor which will be allowed to exist in the process. Normally
1215 : the sandbox code rejects and aborts if there is an unexpected file
1216 : descriptor present on boot. This is helpful to allow a parent
1217 : process to be notified on termination of the tile by waiting for a
1218 : pipe file descriptor to get closed.
1219 :
1220 : wait and debugger are both used in debugging. If wait is non-NULL,
1221 : the runner will wait until the value pointed to by wait is non-zero
1222 : before launching the tile. Likewise, if debugger is non-NULL, the
1223 : runner will wait until a debugger is attached before setting the
1224 : value pointed to by debugger to non-zero. These are intended to be
1225 : used as a pair, where many tiles share a waiting reference, and then
1226 : one of the tiles (a tile you want to attach the debugger to) has the
1227 : same reference provided as the debugger, so all tiles will stop and
1228 : wait for the debugger to attach to it before proceeding. */
1229 :
1230 : void
1231 : fd_topo_run_tile( fd_topo_t * topo,
1232 : fd_topo_tile_t * tile,
1233 : int sandbox,
1234 : int keep_controlling_terminal,
1235 : int dumpable,
1236 : uint uid,
1237 : uint gid,
1238 : int allow_fd,
1239 : fd_topo_run_tile_t * tile_run );
1240 :
1241 : /* This is for determining the value of RLIMIT_MLOCK that we need to
1242 : successfully run all tiles in separate processes. The value returned
1243 : is the maximum amount of memory that will be locked with mlock() by
1244 : any individual process in the tree. Specifically, if we have three
1245 : tile processes, and they each need to lock 5, 9, and 2 MiB of memory
1246 : respectively, RLIMIT_MLOCK needs to be 9 MiB to allow all three
1247 : process mlock() calls to succeed.
1248 :
1249 : Tiles lock memory in three ways. Any workspace they are using, they
1250 : lock the entire workspace. Then each tile uses huge pages for the
1251 : stack which are also locked, and finally some tiles use private
1252 : locked mmaps outside the workspace for storing key material. The
1253 : results here include all of this memory together.
1254 :
1255 : The result is not necessarily the amount of memory used by the tile
1256 : process, although it will be quite close. Tiles could potentially
1257 : allocate memory (eg, with brk) without needing to lock it, which
1258 : would not need to included, and some kernel memory that tiles cause
1259 : to be allocated (for example XSK buffers) is also not included. The
1260 : actual amount of memory used will not be less than this value. */
1261 : FD_FN_PURE ulong
1262 : fd_topo_mlock_max_tile( fd_topo_t const * topo );
1263 :
1264 : /* Same as fd_topo_mlock_max_tile, but for loading the entire topology
1265 : into one process, rather than a separate process per tile. This is
1266 : used, for example, by the configuration code when it creates all the
1267 : workspaces, or the monitor that maps the entire system into one
1268 : address space. */
1269 : FD_FN_PURE ulong
1270 : fd_topo_mlock( fd_topo_t const * topo );
1271 :
1272 : /* This returns the number of gigantic pages needed by the topology on
1273 : the provided numa node. It includes pages needed by the workspaces,
1274 : as well as additional allocations like huge pages for process stacks
1275 : and private key storage. */
1276 :
1277 : FD_FN_PURE ulong
1278 : fd_topo_gigantic_page_cnt( fd_topo_t const * topo,
1279 : ulong numa_idx );
1280 :
1281 : /* This returns the number of huge pages in the application needed by
1282 : the topology on the provided numa node. It includes pages needed by
1283 : things placed in the hugetlbfs (workspaces, process stacks). If
1284 : include_anonymous is true, it also includes anonymous hugepages which
1285 : are needed but are not placed in the hugetlbfs. */
1286 :
1287 : FD_FN_PURE ulong
1288 : fd_topo_huge_page_cnt( fd_topo_t const * topo,
1289 : ulong numa_idx,
1290 : int include_anonymous );
1291 :
1292 : /* Returns the number of normal (4 KiB) pages needed by the topology
1293 : for extra allocations like private key storage and XSK rings. */
1294 :
1295 : FD_FN_PURE ulong
1296 : fd_topo_normal_page_cnt( fd_topo_t const * topo );
1297 :
1298 : /* Prints a message describing the topology to an output stream. If
1299 : stdout is true, will be written to stdout, otherwise will be written
1300 : as a NOTICE log message to the log file. */
1301 : void
1302 : fd_topo_print_log( int stdout,
1303 : fd_topo_t * topo );
1304 :
1305 : /* fd_topo_print_json prints the same topology description as
1306 : fd_topo_print_log to stdout, as a JSON document. */
1307 : void
1308 : fd_topo_print_json( fd_topo_t * topo );
1309 :
1310 : FD_PROTOTYPES_END
1311 :
1312 : #endif /* HEADER_fd_src_disco_topo_fd_topo_h */
|
