Line data Source code
1 : #ifndef HEADER_fd_src_disco_gui_fd_gui_h
2 : #define HEADER_fd_src_disco_gui_fd_gui_h
3 :
4 : #include "fd_gui_peers.h"
5 :
6 : #include "../topo/fd_topo.h"
7 :
8 : #include "../../ballet/txn/fd_txn.h"
9 : #include "../../disco/tiles.h"
10 : #include "../../disco/fd_txn_p.h"
11 : #include "../../disco/bundle/fd_bundle_tile.h"
12 : #include "../../discof/restore/fd_snapct_tile.h"
13 : #include "../../discof/tower/fd_tower_tile.h"
14 : #include "../../discof/replay/fd_replay_tile.h"
15 : #include "../../choreo/tower/fd_tower.h"
16 : #include "../../choreo/tower/fd_tower_serdes.h"
17 : #include "../../flamenco/leaders/fd_leaders.h"
18 : #include "../../flamenco/types/fd_types_custom.h"
19 : #include "../../util/fd_util_base.h"
20 : #include "../../util/hist/fd_histf.h"
21 : #include "../../waltz/http/fd_http_server.h"
22 :
23 : /* frankendancer only */
24 : #define FD_GUI_MAX_PEER_CNT ( 40200UL)
25 :
26 : /* frankendancer only */
27 0 : #define FD_GUI_START_PROGRESS_TYPE_INITIALIZING ( 0)
28 0 : #define FD_GUI_START_PROGRESS_TYPE_SEARCHING_FOR_FULL_SNAPSHOT ( 1)
29 0 : #define FD_GUI_START_PROGRESS_TYPE_DOWNLOADING_FULL_SNAPSHOT ( 2)
30 0 : #define FD_GUI_START_PROGRESS_TYPE_SEARCHING_FOR_INCREMENTAL_SNAPSHOT ( 3)
31 0 : #define FD_GUI_START_PROGRESS_TYPE_DOWNLOADING_INCREMENTAL_SNAPSHOT ( 4)
32 0 : #define FD_GUI_START_PROGRESS_TYPE_CLEANING_BLOCK_STORE ( 5)
33 0 : #define FD_GUI_START_PROGRESS_TYPE_CLEANING_ACCOUNTS ( 6)
34 0 : #define FD_GUI_START_PROGRESS_TYPE_LOADING_LEDGER ( 7)
35 0 : #define FD_GUI_START_PROGRESS_TYPE_PROCESSING_LEDGER ( 8)
36 0 : #define FD_GUI_START_PROGRESS_TYPE_STARTING_SERVICES ( 9)
37 0 : #define FD_GUI_START_PROGRESS_TYPE_HALTED (10)
38 0 : #define FD_GUI_START_PROGRESS_TYPE_WAITING_FOR_SUPERMAJORITY (11)
39 0 : #define FD_GUI_START_PROGRESS_TYPE_RUNNING (12)
40 :
41 : /* frankendancer only */
42 : struct fd_gui_gossip_peer {
43 : fd_pubkey_t pubkey[ 1 ];
44 : ulong wallclock;
45 : ushort shred_version;
46 :
47 : int has_version;
48 : struct {
49 : ushort major;
50 : ushort minor;
51 : ushort patch;
52 :
53 : int has_commit;
54 : uint commit;
55 :
56 : uint feature_set;
57 : } version;
58 :
59 : struct {
60 : uint ipv4;
61 : ushort port;
62 : } sockets[ 12 ];
63 : };
64 :
65 : /* frankendancer only */
66 : struct fd_gui_vote_account {
67 : fd_pubkey_t pubkey[ 1 ];
68 : fd_pubkey_t vote_account[ 1 ];
69 :
70 : ulong activated_stake;
71 : ulong last_vote;
72 : ulong root_slot;
73 : ulong epoch_credits;
74 : uchar commission;
75 : int delinquent;
76 : };
77 :
78 : /* frankendancer only */
79 : struct fd_gui_validator_info {
80 : fd_pubkey_t pubkey[ 1 ];
81 :
82 : char name[ 64 ];
83 : char website[ 128 ];
84 : char details[ 256 ];
85 : char icon_uri[ 128 ];
86 : };
87 :
88 : /* frankendancer only */
89 : #define FD_GUI_SLOT_LEADER_UNSTARTED (0UL)
90 : #define FD_GUI_SLOT_LEADER_STARTED (1UL)
91 : #define FD_GUI_SLOT_LEADER_ENDED (2UL)
92 :
93 0 : #define FD_GUI_SLOTS_CNT (864000UL) /* 2x 432000 */
94 0 : #define FD_GUI_LEADER_CNT (4096UL)
95 :
96 0 : #define FD_GUI_TPS_HISTORY_WINDOW_DURATION_SECONDS (10L)
97 0 : #define FD_GUI_TPS_HISTORY_SAMPLE_CNT (150UL)
98 :
99 0 : #define FD_GUI_TILE_TIMER_SNAP_CNT (512UL)
100 : #define FD_GUI_TILE_TIMER_LEADER_DOWNSAMPLE_CNT (50UL) /* 500ms / 10ms */
101 0 : #define FD_GUI_SCHEDULER_COUNT_SNAP_CNT (512UL)
102 : #define FD_GUI_SCHEDULER_COUNT_LEADER_DOWNSAMPLE_CNT (50UL) /* 500ms / 10ms */
103 : #define FD_GUI_TILE_TIMER_TILE_CNT (256UL)
104 :
105 0 : #define FD_GUI_VOTE_STATE_NON_VOTING (0)
106 0 : #define FD_GUI_VOTE_STATE_VOTING (1)
107 0 : #define FD_GUI_VOTE_STATE_DELINQUENT (2)
108 :
109 0 : #define FD_GUI_BOOT_PROGRESS_TYPE_JOINING_GOSSIP (1)
110 0 : #define FD_GUI_BOOT_PROGRESS_TYPE_LOADING_FULL_SNAPSHOT (2)
111 0 : #define FD_GUI_BOOT_PROGRESS_TYPE_LOADING_INCREMENTAL_SNAPSHOT (3)
112 0 : #define FD_GUI_BOOT_PROGRESS_TYPE_CATCHING_UP (4)
113 0 : #define FD_GUI_BOOT_PROGRESS_TYPE_RUNNING (5)
114 :
115 0 : #define FD_GUI_BOOT_PROGRESS_FULL_SNAPSHOT_IDX (0UL)
116 0 : #define FD_GUI_BOOT_PROGRESS_INCREMENTAL_SNAPSHOT_IDX (1UL)
117 0 : #define FD_GUI_BOOT_PROGRESS_SNAPSHOT_CNT (2UL)
118 :
119 0 : #define FD_GUI_SLOT_LEVEL_INCOMPLETE (0)
120 0 : #define FD_GUI_SLOT_LEVEL_COMPLETED (1)
121 0 : #define FD_GUI_SLOT_LEVEL_OPTIMISTICALLY_CONFIRMED (2)
122 0 : #define FD_GUI_SLOT_LEVEL_ROOTED (3)
123 0 : #define FD_GUI_SLOT_LEVEL_FINALIZED (4)
124 :
125 : /* Ideally, we would store an entire epoch's worth of transactions. If
126 : we assume any given validator will have at most 5% stake, and average
127 : transactions per slot is around 10_000, then an epoch will have about
128 : 432_000*10_000*0.05 transactions (~2^28).
129 :
130 : Unfortunately, the transaction struct is 100+ bytes. If we sized the
131 : array to 2^28 entries then the memory required would be ~26GB. In
132 : order to keep memory usage to a more reasonable level, we'll
133 : arbitrarily use a fourth of that size. */
134 0 : #define FD_GUI_TXN_HISTORY_SZ (1UL<<26UL)
135 :
136 0 : #define FD_GUI_TXN_FLAGS_STARTED ( 1U)
137 0 : #define FD_GUI_TXN_FLAGS_ENDED ( 2U)
138 0 : #define FD_GUI_TXN_FLAGS_IS_SIMPLE_VOTE ( 4U)
139 0 : #define FD_GUI_TXN_FLAGS_FROM_BUNDLE ( 8U)
140 0 : #define FD_GUI_TXN_FLAGS_LANDED_IN_BLOCK (16U)
141 :
142 0 : #define FD_GUI_TURBINE_RECV_TIMESTAMPS (750UL)
143 :
144 : /* One use case for tracking ingress shred slot is to estimate when we
145 : have caught up to the tip of the blockchain. A naive approach would
146 : be to track the maximum seen slot.
147 :
148 : maximum_seen_slot = fd_ulong_max( maximum_seen_slot, new_slot_from_shred_tile );
149 :
150 : Unfortunately, this doesn't always work because a validator can send
151 : a slot number that is arbitrarily large on a false fork. Also, these
152 : shreds can be for a repair response, which can be arbitrarily small.
153 :
154 : The prospects here seem bleak, but not all hope is lost! We know
155 : that for a sufficiently large historical time window there is a high
156 : probability that at least some of the slots we observe will be valid
157 : recent turbine slots. For a sufficiently small window there is a high
158 : probability that all the observed shred slots are non-malicious (i.e.
159 : not arbitrarily large).
160 :
161 : In practice shred slots are almost always non-malicious. We can keep
162 : a history of the 12 largest slots we've seen in the past 4.8 seconds.
163 : We'll consider the "tip" of the blockchain to be the maximum slot in
164 : our history. This way, if we receive maliciously large slot number,
165 : it will be evicted after 4.8 seconds. If we receive a small slot from
166 : a repair response it will be ignored because we've seen other larger
167 : slots, meaning that our estimate is eventually consistent. For
168 : monitoring purposes this is sufficient.
169 :
170 : The worst case scenario is that this validator receives an incorrect
171 : shred slot slot more than once every 3 leader rotations. Before the
172 : previous incorrect slot is evicted from the history, a new one takes
173 : it's place and we wouldn't never get a correct estimate of the tip of
174 : the chain. We also would indefinitely think that that we haven't
175 : caught up. This would require the chain having perpetually malicious
176 : leaders with adjacent rotations. If this happens, Solana has bigger
177 : problems. */
178 0 : #define FD_GUI_TURBINE_SLOT_HISTORY_SZ ( 12UL )
179 :
180 : /* Like the turbine slot, the latest repair slot can also swing to
181 : arbitrarily large values due to a malicious fork switch. The gui
182 : provides the same guarantees for freshness and accuracy. This
183 : history is somewhat larger to handle the increased repair bandwidth
184 : during catch up. */
185 0 : #define FD_GUI_REPAIR_SLOT_HISTORY_SZ ( 512UL )
186 :
187 : /* FD_GUI_*_CATCH_UP_HISTORY_SZ is the capacity of the record of slots
188 : seen from repair or turbine during the catch up stage at startup.
189 : These buffers are run-length encoded, so they will typically be very
190 : small. The worst-case scenario is unbounded, so bounds here are
191 : determined heuristically. */
192 0 : #define FD_GUI_REPAIR_CATCH_UP_HISTORY_SZ (4096UL)
193 0 : #define FD_GUI_TURBINE_CATCH_UP_HISTORY_SZ (4096UL)
194 :
195 : /* FD_GUI_SHREDS_STAGING_SZ is number of shred events we'll retain in
196 : in a small staging area. The lifecycle of a shred looks something
197 : like the following
198 :
199 : states] turbine -> repairing (optional) -> processing -> waiting_for_siblings -> slot_complete
200 : events] ^-repair_requested ^-shred_received/shred_repaired ^-shred_replayed ^-max(shred_replayed)
201 :
202 : We're interested in recording timestamps for state transitions (which
203 : these docs call "shred events"). Unfortunately, due to forking,
204 : duplicate packets, etc we can't make any guarantees about ordering or
205 : uniqueness for these event timestamps. Instead the GUI just records
206 : timestamps for all events as they occur and put them into an array.
207 : Newly recorded event timestamps are also broadcast live to WebSocket
208 : consumers.
209 :
210 : The amount of shred events for non-finalized blocks can't really be
211 : bounded, so we use generous estimates here to set a memory bound. */
212 0 : #define FD_GUI_MAX_SHREDS_PER_BLOCK (32UL*1024UL)
213 0 : #define FD_GUI_MAX_EVENTS_PER_SHRED ( 32UL)
214 0 : #define FD_GUI_SHREDS_STAGING_SZ (32UL * FD_GUI_MAX_SHREDS_PER_BLOCK * FD_GUI_MAX_EVENTS_PER_SHRED)
215 :
216 : /* FD_GUI_SHREDS_HISTORY_SZ the number of shred events in our historical
217 : shred store. Shred events here belong to finalized slots which means
218 : we won't record any additional shred updates for these slots.
219 :
220 : All shred events for a given slot will be places in a contiguous
221 : chunk in the array, and the bounding indicies are stored in the
222 : fd_gui_slot_t slot history. Within a slot chunk, shred events are
223 : ordered in the ordered they were recorded by the gui tile.
224 :
225 : Ideally, we have enough space to store an epoch's worth of events,
226 : but we are limited by realistic memory consumption. Instead, we pick
227 : bound heuristically. */
228 0 : #define FD_GUI_SHREDS_HISTORY_SZ (432000UL*2000UL*4UL / 6UL)
229 :
230 0 : #define FD_GUI_SLOT_SHRED_REPAIR_REQUEST (0UL)
231 0 : #define FD_GUI_SLOT_SHRED_SHRED_RECEIVED_TURBINE (1UL)
232 0 : #define FD_GUI_SLOT_SHRED_SHRED_RECEIVED_REPAIR (2UL)
233 0 : #define FD_GUI_SLOT_SHRED_SHRED_REPLAY_EXEC_DONE (3UL)
234 0 : #define FD_GUI_SLOT_SHRED_SHRED_SLOT_COMPLETE (4UL)
235 : /* #define FD_GUI_SLOT_SHRED_SHRED_REPLAY_EXEC_START (5UL) // UNUSED */
236 0 : #define FD_GUI_SLOT_SHRED_SHRED_PUBLISHED (6UL)
237 :
238 0 : #define FD_GUI_SLOT_RANKINGS_SZ (100UL)
239 0 : #define FD_GUI_SLOT_RANKING_TYPE_ASC (0)
240 0 : #define FD_GUI_SLOT_RANKING_TYPE_DESC (1)
241 :
242 : struct fd_gui_tile_timers {
243 : ulong timers[ FD_METRICS_ENUM_TILE_REGIME_CNT ];
244 : ulong sched_timers[ FD_METRICS_ENUM_CPU_REGIME_CNT ];
245 :
246 : int in_backp;
247 : ushort last_cpu;
248 : uchar status;
249 : ulong heartbeat;
250 : ulong backp_cnt;
251 : ulong nvcsw;
252 : ulong nivcsw;
253 : ulong minflt;
254 : ulong majflt;
255 : };
256 :
257 : typedef struct fd_gui_tile_timers fd_gui_tile_timers_t;
258 :
259 : struct fd_gui_scheduler_counts {
260 : long sample_time_ns;
261 : ulong regular;
262 : ulong votes;
263 : ulong conflicting;
264 : ulong bundles;
265 : };
266 :
267 : typedef struct fd_gui_scheduler_counts fd_gui_scheduler_counts_t;
268 :
269 : struct fd_gui_network_stats {
270 : /* total bytes accumulated */
271 : struct {
272 : ulong turbine;
273 : ulong gossip;
274 : ulong tpu;
275 : ulong repair;
276 : ulong metric;
277 : } in, out;
278 : };
279 :
280 : typedef struct fd_gui_network_stats fd_gui_network_stats_t;
281 :
282 : struct fd_gui_leader_slot {
283 : ulong slot;
284 : fd_hash_t block_hash;
285 : long leader_start_time; /* UNIX timestamp of when we first became leader in this slot */
286 : long leader_end_time; /* UNIX timestamp of when we stopped being leader in this slot */
287 :
288 : /* Stem tiles can exist in one of 8 distinct activity regimes at any
289 : given moment. One of these regimes, caughtup_postfrag, is the
290 : only regime where a tile is in a spin loop without doing any
291 : useful work. This info is useful from a monitoring perspective
292 : because it lets us estimate CPU utilization on a pinned core.
293 :
294 : Every 10ms, the gui tile samples the amount of time tiles spent
295 : in each regime in the past 10ms. This sample is used to infer
296 : the CPU utilization in the past 10ms. This utilization is
297 : streamed live to WebSocket clients.
298 :
299 : In additional to live utilization, we are interested in recording
300 : utilization during one of this validator's leader slots. The gui
301 : tile is continuously recording samples to storage with capacity
302 : FD_GUI_TILE_TIMER_SNAP_CNT. The sample index is recorded at the
303 : start and end of a leader slot, and the number of samples is
304 : downsampled to be at most FD_GUI_TILE_TIMER_LEADER_DOWNSAMPLE_CNT
305 : samples (e.g. if there was an unusually long leader slot) and
306 : inserted into historical storage with capacity FD_GUI_LEADER_CNT.
307 : FD_GUI_TILE_TIMER_TILE_CNT is the maximum number of tiles supported. */
308 : fd_gui_tile_timers_t tile_timers[ FD_GUI_TILE_TIMER_LEADER_DOWNSAMPLE_CNT ][ FD_GUI_TILE_TIMER_TILE_CNT ];
309 : ulong tile_timers_sample_cnt;
310 :
311 : fd_gui_scheduler_counts_t scheduler_counts[ FD_GUI_SCHEDULER_COUNT_LEADER_DOWNSAMPLE_CNT ][ 1 ];
312 : ulong scheduler_counts_sample_cnt;
313 :
314 : struct {
315 : uint microblocks_upper_bound; /* An upper bound on the number of microblocks in the slot. If the number of
316 : microblocks observed is equal to this, the slot can be considered over.
317 : Generally, the bound is set to a "final" state by a done packing message,
318 : which sets it to the exact number of microblocks, but sometimes this message
319 : is not sent, if the max upper bound published by poh was already correct. */
320 : uint begin_microblocks; /* The number of microblocks we have seen be started (sent) from pack to banks. */
321 : uint end_microblocks; /* The number of microblocks we have seen be ended (sent) from banks to poh. The
322 : slot is only considered over if the begin and end microblocks seen are both equal
323 : to the microblock upper bound. */
324 :
325 : ulong start_offset; /* The smallest pack transaction index for this slot. The first transaction for this slot will
326 : be written to gui->txs[ start_offset%FD_GUI_TXN_HISTORY_SZ ]. */
327 : ulong end_offset; /* The largest pack transaction index for this slot, plus 1. The last transaction for this
328 : slot will be written to gui->txs[ (end_offset-1)%FD_GUI_TXN_HISTORY_SZ ]. */
329 : } txs;
330 :
331 : fd_done_packing_t scheduler_stats[ 1 ];
332 :
333 : uchar unbecame_leader: 1;
334 : };
335 :
336 : typedef struct fd_gui_leader_slot fd_gui_leader_slot_t;
337 :
338 : struct fd_gui_turbine_slot {
339 : ulong slot;
340 : long timestamp;
341 : };
342 :
343 : typedef struct fd_gui_turbine_slot fd_gui_turbine_slot_t;
344 :
345 : struct fd_gui_slot_staged_shred_event {
346 : long timestamp;
347 : ulong slot;
348 : ushort shred_idx;
349 : uchar event;
350 : };
351 :
352 : typedef struct fd_gui_slot_staged_shred_event fd_gui_slot_staged_shred_event_t;
353 :
354 : struct __attribute__((packed)) fd_gui_slot_history_shred_event {
355 : long timestamp;
356 : ushort shred_idx;
357 : uchar event;
358 : };
359 :
360 : typedef struct fd_gui_slot_history_shred_event fd_gui_slot_history_shred_event_t;
361 :
362 : struct fd_gui_slot_ranking {
363 : ulong slot;
364 : ulong value;
365 : int type;
366 : };
367 : typedef struct fd_gui_slot_ranking fd_gui_slot_ranking_t;
368 :
369 : struct fd_gui_slot_rankings {
370 : fd_gui_slot_ranking_t largest_tips [ FD_GUI_SLOT_RANKINGS_SZ+1UL ];
371 : fd_gui_slot_ranking_t largest_fees [ FD_GUI_SLOT_RANKINGS_SZ+1UL ];
372 : fd_gui_slot_ranking_t largest_rewards [ FD_GUI_SLOT_RANKINGS_SZ+1UL ];
373 : fd_gui_slot_ranking_t largest_duration [ FD_GUI_SLOT_RANKINGS_SZ+1UL ];
374 : fd_gui_slot_ranking_t largest_compute_units [ FD_GUI_SLOT_RANKINGS_SZ+1UL ];
375 : fd_gui_slot_ranking_t largest_skipped [ FD_GUI_SLOT_RANKINGS_SZ+1UL ];
376 : fd_gui_slot_ranking_t largest_rewards_per_cu [ FD_GUI_SLOT_RANKINGS_SZ+1UL ];
377 : fd_gui_slot_ranking_t smallest_tips [ FD_GUI_SLOT_RANKINGS_SZ+1UL ];
378 : fd_gui_slot_ranking_t smallest_fees [ FD_GUI_SLOT_RANKINGS_SZ+1UL ];
379 : fd_gui_slot_ranking_t smallest_rewards [ FD_GUI_SLOT_RANKINGS_SZ+1UL ];
380 : fd_gui_slot_ranking_t smallest_rewards_per_cu[ FD_GUI_SLOT_RANKINGS_SZ+1UL ];
381 : fd_gui_slot_ranking_t smallest_duration [ FD_GUI_SLOT_RANKINGS_SZ+1UL ];
382 : fd_gui_slot_ranking_t smallest_compute_units [ FD_GUI_SLOT_RANKINGS_SZ+1UL ];
383 : fd_gui_slot_ranking_t smallest_skipped [ FD_GUI_SLOT_RANKINGS_SZ+1UL ];
384 : };
385 :
386 : typedef struct fd_gui_slot_rankings fd_gui_slot_rankings_t;
387 :
388 : struct fd_gui_ephemeral_slot {
389 : ulong slot; /* ULONG_MAX indicates invalid/evicted */
390 : long timestamp_arrival_nanos;
391 : };
392 : typedef struct fd_gui_ephemeral_slot fd_gui_ephemeral_slot_t;
393 :
394 : struct __attribute__((packed)) fd_gui_txn {
395 : uchar signature[ FD_TXN_SIGNATURE_SZ ];
396 : ulong transaction_fee;
397 : ulong priority_fee;
398 : ulong tips;
399 : long timestamp_arrival_nanos;
400 :
401 : /* compute_units_requested has both execution and non-execution cus */
402 : uint compute_units_requested : 21; /* <= 1.4M */
403 : uint compute_units_consumed : 21; /* <= 1.4M */
404 : uint bank_idx : 6; /* in [0, 64) */
405 : uint error_code : 6; /* in [0, 64) */
406 : int timestamp_delta_start_nanos;
407 : int timestamp_delta_end_nanos;
408 :
409 : /* txn_{}_pct is used as a fraction of the total microblock
410 : duration. For example, txn_load_end_pct can be used to find the
411 : time when this transaction started executing:
412 :
413 : timestamp_delta_start_exec_nanos = (
414 : (timestamp_delta_end_nanos-timestamp_delta_start_nanos) *
415 : ((double)txn_{}_pct/UCHAR_MAX)
416 : ) */
417 : uchar txn_start_pct;
418 : uchar txn_load_end_pct;
419 : uchar txn_end_pct;
420 : uchar txn_preload_end_pct;
421 : uchar flags; /* assigned with the FD_GUI_TXN_FLAGS_* macros */
422 : uchar source_tpu; /* FD_TXN_M_TPU_SOURCE_* */
423 : uint source_ipv4;
424 : uint microblock_idx;
425 : };
426 :
427 : typedef struct fd_gui_txn fd_gui_txn_t;
428 :
429 : struct fd_gui_txn_waterfall {
430 : struct {
431 : ulong quic;
432 : ulong udp;
433 : ulong gossip;
434 : ulong block_engine;
435 : ulong pack_cranked;
436 : } in;
437 :
438 : struct {
439 : ulong net_overrun;
440 : ulong quic_overrun;
441 : ulong quic_frag_drop;
442 : ulong quic_abandoned;
443 : ulong tpu_quic_invalid;
444 : ulong tpu_udp_invalid;
445 : ulong verify_overrun;
446 : ulong verify_parse;
447 : ulong verify_failed;
448 : ulong verify_duplicate;
449 : ulong dedup_duplicate;
450 : ulong resolv_lut_failed;
451 : ulong resolv_expired;
452 : ulong resolv_ancient;
453 : ulong resolv_no_ledger;
454 : ulong resolv_retained;
455 : ulong pack_invalid;
456 : ulong pack_invalid_bundle;
457 : ulong pack_expired;
458 : ulong pack_already_executed;
459 : ulong pack_retained;
460 : ulong pack_wait_full;
461 : ulong pack_leader_slow;
462 : ulong bank_invalid;
463 : ulong bank_nonce_already_advanced;
464 : ulong bank_nonce_advance_failed;
465 : ulong bank_nonce_wrong_blockhash;
466 : ulong block_success;
467 : ulong block_fail;
468 : } out;
469 : };
470 :
471 : typedef struct fd_gui_txn_waterfall fd_gui_txn_waterfall_t;
472 :
473 : struct fd_gui_tile_stats {
474 : long sample_time_nanos;
475 :
476 : ulong net_in_rx_bytes; /* Number of bytes received by the net or sock tile*/
477 : ulong quic_conn_cnt; /* Number of active QUIC connections */
478 : fd_histf_t bundle_rx_delay_hist; /* Histogram of bundle rx delay */
479 : ulong bundle_rtt_smoothed_nanos; /* RTT (nanoseconds) moving average */
480 : ulong verify_drop_cnt; /* Number of transactions dropped by verify tiles */
481 : ulong verify_total_cnt; /* Number of transactions received by verify tiles */
482 : ulong dedup_drop_cnt; /* Number of transactions dropped by dedup tile */
483 : ulong dedup_total_cnt; /* Number of transactions received by dedup tile */
484 : ulong pack_buffer_cnt; /* Number of buffered transactions in the pack tile */
485 : ulong pack_buffer_capacity; /* Total size of the pack transaction buffer */
486 : ulong bank_txn_exec_cnt; /* Number of transactions processed by the bank tile */
487 : ulong net_out_tx_bytes; /* Number of bytes sent by the net or sock tile */
488 : };
489 :
490 : typedef struct fd_gui_tile_stats fd_gui_tile_stats_t;
491 :
492 : struct fd_gui_slot {
493 : ulong slot;
494 : ulong parent_slot;
495 : ulong vote_slot;
496 : ulong reset_slot;
497 : long completed_time;
498 : uint max_compute_units;
499 : int mine;
500 : int skipped;
501 : int must_republish;
502 : int level;
503 : uint compute_units;
504 : ulong transaction_fee;
505 : ulong priority_fee;
506 : ulong tips;
507 : uint shred_cnt;
508 : uchar vote_latency;
509 :
510 : uint vote_success;
511 : uint vote_failed;
512 : uint nonvote_success;
513 : uint nonvote_failed;
514 :
515 : /* Some slot info is only tracked for our own leader slots. These
516 : slots are kept in a separate buffer. */
517 : ulong leader_history_idx;
518 :
519 : fd_gui_txn_waterfall_t waterfall_begin[ 1 ];
520 : fd_gui_txn_waterfall_t waterfall_end[ 1 ];
521 :
522 : fd_gui_tile_stats_t tile_stats_begin[ 1 ];
523 : fd_gui_tile_stats_t tile_stats_end[ 1 ];
524 :
525 : struct {
526 : ulong start_offset; /* gui->shreds.history[ start_offset % FD_GUI_SHREDS_HISTORY_SZ ] is the first shred event in
527 : contiguous chunk of events in the shred history corresponding to this slot. */
528 : ulong end_offset; /* One past the last shred event in the contiguous chunk of events in the shred history
529 : corresponding to this slot. */
530 : } shreds;
531 : };
532 :
533 : typedef struct fd_gui_slot fd_gui_slot_t;
534 :
535 : struct fd_gui {
536 : fd_http_server_t * http;
537 : fd_topo_t * topo;
538 :
539 : long next_sample_400millis;
540 : long next_sample_100millis;
541 : long next_sample_50millis;
542 : long next_sample_25millis;
543 : long next_sample_10millis;
544 :
545 : ulong leader_slot;
546 :
547 : struct {
548 : fd_pubkey_t identity_key[ 1 ];
549 : int has_vote_key;
550 : fd_pubkey_t vote_key[ 1 ];
551 : char vote_key_base58[ FD_BASE58_ENCODED_32_SZ ];
552 : char identity_key_base58[ FD_BASE58_ENCODED_32_SZ ];
553 :
554 : int is_full_client;
555 : char const * version;
556 : char const * cluster;
557 :
558 : ulong vote_distance;
559 : int vote_state;
560 :
561 : long startup_time_nanos;
562 :
563 : union {
564 : struct { /* frankendancer only */
565 : uchar phase;
566 : int startup_got_full_snapshot;
567 :
568 : ulong startup_incremental_snapshot_slot;
569 : uint startup_incremental_snapshot_peer_ip_addr;
570 : ushort startup_incremental_snapshot_peer_port;
571 : double startup_incremental_snapshot_elapsed_secs;
572 : double startup_incremental_snapshot_remaining_secs;
573 : double startup_incremental_snapshot_throughput;
574 : ulong startup_incremental_snapshot_total_bytes;
575 : ulong startup_incremental_snapshot_current_bytes;
576 :
577 : ulong startup_full_snapshot_slot;
578 : uint startup_full_snapshot_peer_ip_addr;
579 : ushort startup_full_snapshot_peer_port;
580 : double startup_full_snapshot_elapsed_secs;
581 : double startup_full_snapshot_remaining_secs;
582 : double startup_full_snapshot_throughput;
583 : ulong startup_full_snapshot_total_bytes;
584 : ulong startup_full_snapshot_current_bytes;
585 :
586 : ulong startup_ledger_slot;
587 : ulong startup_ledger_max_slot;
588 :
589 : ulong startup_waiting_for_supermajority_slot;
590 : ulong startup_waiting_for_supermajority_stake_pct;
591 : } startup_progress;
592 : struct { /* used in the full client */
593 : uchar phase;
594 : long joining_gossip_time_nanos;
595 : struct {
596 : ulong slot;
597 : uint peer_addr;
598 : ushort peer_port;
599 : ulong total_bytes_compressed;
600 : long reset_time_nanos; /* UNIX nanosecond timestamp */
601 : long sample_time_nanos;
602 : ulong reset_cnt;
603 :
604 : ulong read_bytes_compressed;
605 : char read_path[ PATH_MAX+30UL ]; /* URL or filesystem path. 30 is fd_cstr_nlen( "https://255.255.255.255:12345/", ULONG_MAX ) */
606 :
607 : ulong decompress_bytes_decompressed;
608 : ulong decompress_bytes_compressed;
609 :
610 : ulong insert_bytes_decompressed;
611 : char insert_path[ PATH_MAX ];
612 : ulong insert_accounts_current;
613 : } loading_snapshot[ FD_GUI_BOOT_PROGRESS_SNAPSHOT_CNT ];
614 :
615 : long catching_up_time_nanos;
616 : ulong catching_up_first_replay_slot;
617 : } boot_progress;
618 : };
619 :
620 : int schedule_strategy;
621 :
622 : ulong identity_account_balance;
623 : ulong vote_account_balance;
624 : ulong estimated_slot_duration_nanos;
625 :
626 : ulong sock_tile_cnt;
627 : ulong net_tile_cnt;
628 : ulong quic_tile_cnt;
629 : ulong verify_tile_cnt;
630 : ulong resolh_tile_cnt;
631 : ulong resolv_tile_cnt;
632 : ulong bank_tile_cnt;
633 : ulong execle_tile_cnt;
634 : ulong shred_tile_cnt;
635 :
636 : ulong slot_rooted;
637 : ulong slot_optimistically_confirmed;
638 : ulong slot_completed;
639 : ulong slot_estimated;
640 : ulong slot_caught_up;
641 : ulong slot_repair;
642 : ulong slot_turbine;
643 : ulong slot_reset;
644 : ulong slot_storage;
645 : ulong active_fork_cnt;
646 :
647 : fd_gui_ephemeral_slot_t slots_max_turbine[ FD_GUI_TURBINE_SLOT_HISTORY_SZ+1UL ];
648 : fd_gui_ephemeral_slot_t slots_max_repair [ FD_GUI_REPAIR_SLOT_HISTORY_SZ +1UL ];
649 :
650 : /* catchup_* and late_votes are run-length encoded. i.e. adjacent
651 : pairs represent contiguous runs */
652 : ulong catch_up_turbine[ FD_GUI_TURBINE_CATCH_UP_HISTORY_SZ ];
653 : ulong catch_up_turbine_sz;
654 :
655 : ulong catch_up_repair[ FD_GUI_REPAIR_CATCH_UP_HISTORY_SZ ];
656 : ulong catch_up_repair_sz;
657 :
658 : ulong late_votes[ MAX_SLOTS_PER_EPOCH ];
659 : ulong late_votes_sz;
660 :
661 : ulong estimated_tps_history_idx;
662 : struct {
663 : ulong vote_failed;
664 : ulong vote_success;
665 : ulong nonvote_success;
666 : ulong nonvote_failed;
667 : } estimated_tps_history[ FD_GUI_TPS_HISTORY_SAMPLE_CNT ];
668 :
669 : fd_gui_network_stats_t network_stats_current[ 1 ];
670 :
671 : fd_gui_txn_waterfall_t txn_waterfall_reference[ 1 ];
672 : fd_gui_txn_waterfall_t txn_waterfall_current[ 1 ];
673 :
674 : fd_gui_tile_stats_t tile_stats_reference[ 1 ];
675 : fd_gui_tile_stats_t tile_stats_current[ 1 ];
676 :
677 : ulong tile_timers_snap_idx;
678 : ulong tile_timers_snap_idx_slot_start;
679 : /* Temporary storage for samples. Will be downsampled into leader history on slot end. */
680 : fd_gui_tile_timers_t tile_timers_snap[ FD_GUI_TILE_TIMER_SNAP_CNT ][ FD_GUI_TILE_TIMER_TILE_CNT ];
681 :
682 : ulong scheduler_counts_snap_idx;
683 : ulong scheduler_counts_snap_idx_slot_start;
684 : /* Temporary storage for samples. Will be downsampled into leader history on slot end. */
685 : fd_gui_scheduler_counts_t scheduler_counts_snap[ FD_GUI_SCHEDULER_COUNT_SNAP_CNT ][ 1 ];
686 : } summary;
687 :
688 : fd_gui_slot_t slots[ FD_GUI_SLOTS_CNT ][ 1 ];
689 :
690 : /* used for estimating slot duration */
691 : fd_gui_turbine_slot_t turbine_slots[ FD_GUI_TURBINE_RECV_TIMESTAMPS ];
692 :
693 : fd_gui_leader_slot_t leader_slots[ FD_GUI_LEADER_CNT ][ 1 ];
694 : ulong leader_slots_cnt;
695 :
696 : fd_gui_txn_t txs[ FD_GUI_TXN_HISTORY_SZ ][ 1 ];
697 : ulong pack_txn_idx; /* The pack index of the most recently received transaction */
698 :
699 : ulong tower_cnt;
700 : fd_vote_acc_vote_t tower[ FD_TOWER_VOTE_MAX ];
701 :
702 : struct {
703 : int has_block_engine;
704 : char name[ 16 ];
705 : char url[ 256 ];
706 : char ip_cstr[ 40 ]; /* IPv4 or IPv6 cstr */
707 : int status;
708 : } block_engine;
709 :
710 : struct {
711 : int has_epoch[ 2 ];
712 :
713 : struct {
714 : ulong epoch;
715 : long start_time;
716 : long end_time;
717 :
718 : ulong my_total_slots;
719 : ulong my_skipped_slots;
720 :
721 : ulong start_slot;
722 : ulong end_slot;
723 : ulong excluded_stake;
724 : fd_epoch_leaders_t * lsched;
725 : uchar __attribute__((aligned(FD_EPOCH_LEADERS_ALIGN))) _lsched[ FD_EPOCH_LEADERS_FOOTPRINT(MAX_STAKED_LEADERS, MAX_SLOTS_PER_EPOCH) ];
726 : fd_vote_stake_weight_t stakes[ MAX_STAKED_LEADERS ];
727 :
728 : ulong rankings_slot; /* One more than the largest slot we've processed into our rankings */
729 : fd_gui_slot_rankings_t rankings[ 1 ]; /* global slot rankings */
730 : fd_gui_slot_rankings_t my_rankings[ 1 ]; /* my slots only */
731 : } epochs[ 2 ];
732 : } epoch;
733 :
734 : struct { /* frankendancer only */
735 : ulong peer_cnt;
736 : struct fd_gui_gossip_peer peers[ FD_GUI_MAX_PEER_CNT ];
737 : } gossip;
738 :
739 : struct { /* frankendancer only */
740 : ulong vote_account_cnt;
741 : struct fd_gui_vote_account vote_accounts[ FD_GUI_MAX_PEER_CNT ];
742 : } vote_account;
743 :
744 : struct { /* frankendancer only */
745 : ulong info_cnt;
746 : struct fd_gui_validator_info info[ FD_GUI_MAX_PEER_CNT ];
747 : } validator_info;
748 :
749 : fd_gui_peers_ctx_t * peers; /* full-client */
750 :
751 : struct {
752 : ulong leader_shred_cnt; /* A gauge counting the number of leader shreds seen on the SHRED_OUT link. Resets at
753 : the end of a leader slot. This works because leader fecs are published in order. */
754 : ulong staged_next_broadcast; /* staged[ staged_next_broadcast % FD_GUI_SHREDS_STAGING_SZ ] is the first shred event
755 : that hasn't yet been broadcast to WebSocket clients */
756 : ulong staged_head; /* staged_head % FD_GUI_SHREDS_STAGING_SZ is the first valid event in staged */
757 : ulong staged_tail; /* staged_tail % FD_GUI_SHREDS_STAGING_SZ is one past the last valid event in staged */
758 : fd_gui_slot_staged_shred_event_t staged [ FD_GUI_SHREDS_STAGING_SZ ];
759 :
760 : ulong history_slot; /* the largest slot store in history */
761 : ulong history_tail; /* history_tail % FD_GUI_SHREDS_HISTORY_SZ is one past the last valid event in history */
762 : fd_gui_slot_history_shred_event_t history[ FD_GUI_SHREDS_HISTORY_SZ ];
763 :
764 : /* scratch space for stable sorts */
765 : fd_gui_slot_staged_shred_event_t _staged_scratch [ FD_GUI_SHREDS_STAGING_SZ ];
766 : fd_gui_slot_staged_shred_event_t _staged_scratch2[ FD_GUI_SHREDS_STAGING_SZ ];
767 : } shreds; /* full client */
768 : };
769 :
770 : typedef struct fd_gui fd_gui_t;
771 :
772 : FD_PROTOTYPES_BEGIN
773 :
774 : FD_FN_CONST ulong
775 : fd_gui_align( void );
776 :
777 : FD_FN_CONST ulong
778 : fd_gui_footprint( void );
779 :
780 : void *
781 : fd_gui_new( void * shmem,
782 : fd_http_server_t * http,
783 : char const * version,
784 : char const * cluster,
785 : uchar const * identity_key,
786 : int has_vote_key,
787 : uchar const * vote_key,
788 : int is_full_client,
789 : int snapshots_enabled,
790 : int is_voting,
791 : int schedule_strategy,
792 : fd_topo_t * topo,
793 : long now );
794 :
795 : fd_gui_t *
796 : fd_gui_join( void * shmem );
797 :
798 : void
799 : fd_gui_set_identity( fd_gui_t * gui,
800 : uchar const * identity_pubkey );
801 :
802 : void
803 : fd_gui_ws_open( fd_gui_t * gui,
804 : ulong conn_id,
805 : long now );
806 :
807 : int
808 : fd_gui_ws_message( fd_gui_t * gui,
809 : ulong ws_conn_id,
810 : uchar const * data,
811 : ulong data_len );
812 :
813 : void
814 : fd_gui_plugin_message( fd_gui_t * gui,
815 : ulong plugin_msg,
816 : void const * msg,
817 : long now );
818 :
819 : void
820 : fd_gui_became_leader( fd_gui_t * gui,
821 : ulong slot,
822 : long start_time_nanos,
823 : long end_time_nanos,
824 : ulong max_compute_units,
825 : ulong max_microblocks );
826 :
827 : void
828 : fd_gui_unbecame_leader( fd_gui_t * gui,
829 : ulong _slot,
830 : fd_done_packing_t const * done_packing,
831 : long now );
832 :
833 : void
834 : fd_gui_microblock_execution_begin( fd_gui_t * gui,
835 : long now,
836 : ulong _slot,
837 : fd_txn_e_t * txns,
838 : ulong txn_cnt,
839 : uint microblock_idx,
840 : ulong pack_txn_idx );
841 :
842 : void
843 : fd_gui_microblock_execution_end( fd_gui_t * gui,
844 : long now,
845 : ulong bank_idx,
846 : ulong _slot,
847 : ulong txn_cnt,
848 : fd_txn_p_t * txns,
849 : ulong pack_txn_idx,
850 : uchar txn_start_pct,
851 : uchar txn_load_end_pct,
852 : uchar txn_end_pct,
853 : uchar txn_preload_end_pct,
854 : ulong tips );
855 :
856 : int
857 : fd_gui_poll( fd_gui_t * gui, long now );
858 :
859 : void
860 : fd_gui_handle_block_engine_update( fd_gui_t * gui,
861 : fd_bundle_block_engine_update_t const * update );
862 :
863 : void
864 : fd_gui_handle_shred( fd_gui_t * gui,
865 : ulong slot,
866 : ulong shred_idx,
867 : int is_turbine,
868 : long tsorig );
869 :
870 : void
871 : fd_gui_handle_leader_fec( fd_gui_t * gui,
872 : ulong slot,
873 : ulong fec_shred_cnt,
874 : int is_end_of_slot,
875 : long tsorig );
876 :
877 : void
878 : fd_gui_handle_exec_txn_done( fd_gui_t * gui,
879 : ulong slot,
880 : ulong start_shred_idx,
881 : ulong end_shred_idx,
882 : long tsorig_ns,
883 : long tspub_ns );
884 :
885 : void
886 : fd_gui_handle_repair_slot( fd_gui_t * gui, ulong slot, long now );
887 :
888 : void
889 : fd_gui_handle_repair_request( fd_gui_t * gui, ulong slot, ulong shred_idx, long now );
890 :
891 : void
892 : fd_gui_handle_snapshot_update( fd_gui_t * gui,
893 : fd_snapct_update_t const * msg );
894 :
895 : void
896 : fd_gui_handle_leader_schedule( fd_gui_t * gui,
897 : fd_stake_weight_msg_t const * leader_schedule,
898 : long now );
899 :
900 : void
901 : fd_gui_handle_epoch_info( fd_gui_t * gui,
902 : fd_epoch_info_msg_t const * epoch_info,
903 : long now );
904 :
905 : void
906 : fd_gui_handle_notarization_update( fd_gui_t * gui,
907 : fd_tower_slot_confirmed_t const * notar );
908 :
909 : void
910 : fd_gui_handle_tower_update( fd_gui_t * gui,
911 : fd_tower_slot_done_t const * msg,
912 : long now );
913 :
914 : void
915 : fd_gui_handle_replay_update( fd_gui_t * gui,
916 : fd_replay_slot_completed_t const * slot_completed,
917 : ulong vote_slot,
918 : long now );
919 :
920 : void
921 : fd_gui_handle_genesis_hash( fd_gui_t * gui,
922 : fd_hash_t const * msg );
923 :
924 : static inline ulong
925 0 : fd_gui_current_epoch_idx( fd_gui_t * gui ) {
926 0 : ulong epoch_idx = ULONG_MAX;
927 0 : ulong epoch = ULONG_MAX;
928 0 : for( ulong i = 0UL; i<2UL; i++ ) {
929 0 : if( FD_LIKELY( gui->epoch.has_epoch[ i ] ) ) {
930 : /* the "current" epoch is the smaller one */
931 0 : if( FD_LIKELY( gui->epoch.epochs[ i ].epoch<epoch ) ) {
932 0 : epoch = gui->epoch.epochs[ i ].epoch;
933 0 : epoch_idx = i;
934 0 : }
935 0 : }
936 0 : }
937 0 : return epoch_idx;
938 0 : }
939 :
940 : static inline fd_gui_slot_t *
941 0 : fd_gui_get_slot( fd_gui_t const * gui, ulong _slot ) {
942 0 : fd_gui_slot_t const * slot = gui->slots[ _slot % FD_GUI_SLOTS_CNT ];
943 0 : if( FD_UNLIKELY( slot->slot==ULONG_MAX || _slot==ULONG_MAX || slot->slot!=_slot ) ) return NULL;
944 0 : return (fd_gui_slot_t *)slot;
945 0 : }
946 :
947 : static inline fd_gui_slot_t const *
948 0 : fd_gui_get_slot_const( fd_gui_t const * gui, ulong _slot ) {
949 0 : return fd_gui_get_slot( gui, _slot );
950 0 : }
951 :
952 : static inline fd_gui_leader_slot_t *
953 0 : fd_gui_get_leader_slot( fd_gui_t const * gui, ulong _slot ) {
954 0 : fd_gui_slot_t const * slot = fd_gui_get_slot( gui, _slot );
955 0 : if( FD_UNLIKELY( !slot
956 0 : || !slot->mine
957 0 : || slot->leader_history_idx==ULONG_MAX
958 0 : || slot->leader_history_idx + FD_GUI_LEADER_CNT < gui->leader_slots_cnt
959 0 : || gui->leader_slots[ slot->leader_history_idx % FD_GUI_LEADER_CNT ]->slot!=_slot ) ) return NULL;
960 0 : return (fd_gui_leader_slot_t *)gui->leader_slots[ slot->leader_history_idx % FD_GUI_LEADER_CNT ];
961 0 : }
962 :
963 : static inline fd_gui_leader_slot_t const *
964 0 : fd_gui_get_leader_slot_const( fd_gui_t const * gui, ulong _slot ) {
965 0 : return fd_gui_get_leader_slot( gui, _slot );
966 0 : }
967 :
968 : /* fd_gui_get_root_slot returns a handle to the closest ancestor of slot
969 : that is a root, if available, otherwise NULL. */
970 : static inline fd_gui_slot_t *
971 : fd_gui_get_root_slot( fd_gui_t const * gui,
972 0 : ulong slot ) {
973 0 : fd_gui_slot_t * c = fd_gui_get_slot( gui, slot );
974 0 : while( c ) {
975 0 : if( FD_UNLIKELY( c->level>=FD_GUI_SLOT_LEVEL_ROOTED ) ) return c;
976 0 : c = fd_gui_get_slot( gui, c->parent_slot );
977 0 : }
978 0 : return NULL;
979 0 : }
980 :
981 : /* fd_gui_slot_is_ancestor returns 1 if anc is known to be an ancestor
982 : of slot (on the same fork), 0 otherwise. */
983 : static inline int
984 : fd_gui_slot_is_ancestor( fd_gui_t const * gui,
985 : ulong anc,
986 0 : ulong slot ) {
987 0 : fd_gui_slot_t * c = fd_gui_get_slot( gui, slot );
988 0 : while( c ) {
989 0 : if( FD_UNLIKELY( c->slot==anc ) ) return 1;
990 0 : c = fd_gui_get_slot( gui, c->parent_slot );
991 0 : }
992 0 : return 0;
993 0 : }
994 :
995 : /* fd_gui_get_parent_slot_on_fork returns a handle to the parent of slot
996 : on the fork ending on frontier_slot. If slot is unknown or skipped,
997 : the closest (by slot number) valid parent on the fork is returned.
998 :
999 : NULL if slot is not an ancestor of frontier slot or if the parent is
1000 : unknown. */
1001 : static inline fd_gui_slot_t *
1002 : fd_gui_get_parent_slot_on_fork( fd_gui_t const * gui,
1003 : ulong frontier_slot,
1004 0 : ulong slot ) {
1005 0 : fd_gui_slot_t * c = fd_gui_get_slot( gui, frontier_slot );
1006 0 : while( c ) {
1007 0 : if( FD_UNLIKELY( c->slot<=slot ) ) return NULL;
1008 0 : fd_gui_slot_t * p = fd_gui_get_slot( gui, c->parent_slot );
1009 0 : if( FD_UNLIKELY( p && p->slot<=slot-1UL ) ) return p;
1010 0 : c = p;
1011 0 : }
1012 0 : return NULL;
1013 0 : }
1014 :
1015 : /* fd_gui_is_skipped_on_fork returns 1 if slot is skipped on the fork
1016 : starting at anc and ending at des, 0 otherwise. */
1017 : static inline int
1018 : fd_gui_is_skipped_on_fork( fd_gui_t const * gui,
1019 : ulong anc,
1020 : ulong des,
1021 0 : ulong slot ) {
1022 0 : fd_gui_slot_t const * c = fd_gui_get_slot( gui, des );
1023 0 : while( c ) {
1024 0 : if( FD_UNLIKELY( anc==c->slot ) ) return 0; /* on the fork, not skipped */
1025 0 : fd_gui_slot_t const * p = fd_gui_get_slot( gui, c->parent_slot );
1026 0 : if( FD_UNLIKELY( p && p->slot<slot && c->slot>slot ) ) return 1; /* in-between two nodes, skipped */
1027 0 : c = p;
1028 0 : }
1029 :
1030 0 : return 0; /* slot not between anc and des, or is unknown */
1031 0 : }
1032 :
1033 : FD_PROTOTYPES_END
1034 :
1035 : #endif /* HEADER_fd_src_disco_gui_fd_gui_h */
|
