Line data Source code
1 : #ifndef HEADER_fd_src_disco_gui_fd_gui_hist_h
2 : #define HEADER_fd_src_disco_gui_fd_gui_hist_h
3 :
4 : /* fd_gui_hist is the GUI's historical event store semantics layer. It
5 : sits on top of fd_gui_store and owns everything GUI-specific about
6 : the monitoring history: the named DBs (FD_GUI_HIST_* below), their
7 : key/value structs, fixed record sizes, the eviction policy, and the
8 : query API.
9 :
10 : There are two kinds of DB:
11 :
12 : - TIME-SERIES (TS) DBs are append-only streams tagged by a
13 : wallclock window. Written by fd_gui_hist_ts_append, read by
14 : time-window range scan.
15 : - KEY-VALUE (KV) DBs hold one record per key. Written/read by
16 : point key (fd_gui_hist_kv_get_or_create / _get / _get_any).
17 : */
18 :
19 : #include "../../util/fd_util_base.h"
20 : #include "fd_gui_store.h"
21 : #include "../../flamenco/leaders/fd_leaders_base.h" /* MAX_SLOTS_PER_EPOCH */
22 : #include "../../flamenco/runtime/fd_slot_params.h" /* FD_SLOT_PARAMS_200MS */
23 :
24 : struct fd_gui;
25 : typedef struct fd_gui fd_gui_t;
26 :
27 21 : #define FD_GUI_HIST_MAGIC (0xf17e6d09147802UL)
28 :
29 : /* FD_GUI_HIST_TS_SKEW_NS bounds how far a TS record's timestamp may
30 : deviate from the trusted wallclock `now` at append time (in either
31 : direction). fd_gui_hist_ts_append clamps ts_ns into [now-skew, now+skew]
32 : before flooring it to a window. This bounds the out-of-orderness of
33 : the append stream which ensures the underlying ring buffer can efficiently evict old entries. */
34 :
35 6426 : #define FD_GUI_HIST_TS_SKEW_NS (10L*FD_GUI_HIST_RES_1S_NS) /* +/- 10 s */
36 6552 : #define FD_GUI_HIST_RES_1S_NS (1000000000L)
37 :
38 : /* FD_GUI_HIST_MIN_EPOCHS is the minimum number of epochs the store must
39 : hold, plus one more which can be evicted under space pressure. */
40 :
41 24 : #define FD_GUI_HIST_MIN_EPOCHS (3UL)
42 27 : #define FD_GUI_HIST_MAX_LEADER_SLOTS_PER_EPOCH (43200UL) /* 10% capacity should be enough for mainnet/testnet */
43 :
44 : /* FD_GUI_HIST_MAX_EPOCHS caps KV index provisioning at the number of
45 : epochs reachable within the TS index horizon (30 days) +2 for partial
46 : epochs at both ends of the horizon. */
47 :
48 3 : #define FD_GUI_HIST_MAX_EPOCHS ((FD_GUI_STORE_TS_IDX_DEPTH*(ulong)FD_GUI_HIST_RES_1S_NS)/(MAX_SLOTS_PER_EPOCH*((FD_SLOT_PARAMS_200MS).ns_per_slot))+2UL)
49 :
50 444 : #define FD_GUI_HIST_SCHEDULER_COUNTS (0) /* (ts, type) */
51 300 : #define FD_GUI_HIST_TILE_TIMERS (1) /* (ts, type) */
52 10029 : #define FD_GUI_HIST_SHRED_EVENTS (2) /* (ts, type, slot) */
53 3864 : #define FD_GUI_HIST_TXN_START (3) /* (ts, type, bank, txn) */
54 3948 : #define FD_GUI_HIST_TXN_END (4) /* (ts, type, bank, txn) */
55 546 : #define FD_GUI_HIST_TOWER (5) /* (ts, type) */
56 9825 : #define FD_GUI_HIST_SLOT (6) /* (slot, bank_seq) */
57 6945 : #define FD_GUI_HIST_LEADER_SLOT (7) /* (slot, bank_seq) */
58 591 : #define FD_GUI_HIST_EPOCH (8) /* (epoch) */
59 384 : #define FD_GUI_HIST_TILE_STATS (9) /* (ts, type) */
60 468 : #define FD_GUI_HIST_TXN_WATERFALL (10) /* (ts, type) */
61 2514 : #define FD_GUI_HIST_CNT (11)
62 :
63 : struct fd_gui_hist_metrics {
64 : /* Writes that hit MAP_FULL and were dropped. */
65 : ulong map_full[ FD_GUI_HIST_CNT ];
66 : /* Writes that evicted records before succeeding. */
67 : ulong reserves [ FD_GUI_HIST_CNT ];
68 : };
69 : typedef struct fd_gui_hist_metrics fd_gui_hist_metrics_t;
70 :
71 : struct fd_gui_hist_slot_key { ulong slot; ulong bank_seq; };
72 : struct fd_gui_hist_leader_slot_key { ulong slot; ulong bank_seq; };
73 : struct fd_gui_hist_epoch_key { ulong epoch; };
74 :
75 : typedef struct fd_gui_hist_slot_key fd_gui_hist_slot_key_t;
76 : typedef struct fd_gui_hist_leader_slot_key fd_gui_hist_leader_slot_key_t;
77 : typedef struct fd_gui_hist_epoch_key fd_gui_hist_epoch_key_t;
78 :
79 : struct fd_gui_hist_private;
80 : typedef struct fd_gui_hist_private fd_gui_hist_t;
81 :
82 : /* fd_gui_hist_kv_slot_iter_t iterates every record for `slot` (every
83 : fork's block) in a slot-keyed KV DB. Used to enumerate the
84 : equivocating forks of a slot. */
85 : struct fd_gui_hist_kv_slot_iter {
86 : void const * rec; /* current record; NULL when done */
87 : ulong bank_seq; /* current record's bank_seq */
88 : fd_gui_store_kv_iter_t _it;
89 : };
90 : typedef struct fd_gui_hist_kv_slot_iter fd_gui_hist_kv_slot_iter_t;
91 :
92 : /* fd_gui_hist_range_filter_fn is an optional per-record predicate
93 : evaluated during a range scan: called with a pointer to the DB's
94 : record and the caller's `ctx`, it returns non-zero to emit the record
95 : or zero to skip it. A NULL filter emits every record in the window
96 : range. */
97 :
98 : typedef int (*fd_gui_hist_range_filter_fn)( void const * rec, void * ctx );
99 :
100 : /* fd_gui_hist_iter_t iterates the records produced by a time-window
101 : range query.
102 :
103 : Records are emitted oldest-window-first, but the backend does not
104 : strictly order records within a window.
105 : */
106 : struct fd_gui_hist_iter {
107 : void const * rec;
108 : ulong rec_sz;
109 :
110 : int _dbi;
111 : int _emitted;
112 : fd_gui_hist_range_filter_fn _filter;
113 : void * _filter_ctx;
114 : fd_gui_store_ts_iter_t _it;
115 : };
116 :
117 : typedef struct fd_gui_hist_iter fd_gui_hist_iter_t;
118 :
119 : FD_PROTOTYPES_BEGIN
120 :
121 : FD_FN_CONST ulong
122 : fd_gui_hist_align( void );
123 :
124 : FD_FN_CONST ulong
125 : fd_gui_hist_footprint( void );
126 :
127 : /* fd_gui_hist_new formats the workspace region `mem`
128 : (>= fd_gui_hist_footprint bytes, fd_gui_hist_align aligned) as the
129 : history semantics layer over the store `db`. Returns `mem` on success
130 : or NULL on failure (logged). */
131 :
132 : void *
133 : fd_gui_hist_new( void * mem,
134 : fd_gui_store_t const * db );
135 :
136 : /* fd_gui_hist_join joins a region formatted by fd_gui_hist_new,
137 : returning a usable handle (mem may be NULL -> NULL). */
138 :
139 : fd_gui_hist_t *
140 : fd_gui_hist_join( void * mem );
141 :
142 : void *
143 : fd_gui_hist_leave( fd_gui_hist_t * hist );
144 :
145 : void *
146 : fd_gui_hist_delete( void * mem );
147 :
148 : /* fd_gui_hist_db_descs returns a pointer to a static array to be passed
149 : to fd_gui_store_new. */
150 :
151 : fd_gui_store_desc_t const *
152 : fd_gui_hist_db_descs( ulong store_bytes );
153 :
154 : FD_FN_CONST static inline ulong
155 42 : fd_gui_hist_db_cnt( void ) { return FD_GUI_HIST_CNT; }
156 :
157 : fd_gui_hist_metrics_t const *
158 : fd_gui_hist_metrics( fd_gui_t const * gui );
159 :
160 : /* ---- TS read/write -------------------------------------------------- */
161 :
162 : /* fd_gui_hist_ts_append appends one record `val` into time-series
163 : database `dbi`, tagged with wallclock timestamp `ts_ns`. `now` is
164 : the current wallclock at append time.
165 :
166 : Will evict old entries to make room if necessary. Returns 0 on
167 : success, -1 on failure (e.g. a misconfigured DB size). */
168 :
169 : int
170 : fd_gui_hist_ts_append( fd_gui_t * gui,
171 : int dbi,
172 : long now,
173 : long ts_ns,
174 : void const * val );
175 :
176 : int
177 : fd_gui_hist_range_begin( fd_gui_t * gui,
178 : fd_gui_hist_iter_t * iter,
179 : int dbi,
180 : long lo_ns,
181 : long hi_ns,
182 : fd_gui_hist_range_filter_fn filter,
183 : void * filter_ctx );
184 :
185 : int
186 : fd_gui_hist_range_next( fd_gui_hist_iter_t * iter );
187 :
188 : void
189 : fd_gui_hist_range_end( fd_gui_hist_iter_t * iter );
190 :
191 :
192 : /* ---- KV read/write -------------------------------------------------- */
193 :
194 : /* fd_gui_hist_kv_get_or_create reserves (creating if absent) the record
195 : for `*key` in a KV DB and returns a mutable pointer to the record's
196 : value, or NULL on failure.
197 :
198 : Will evict old entries to make room if necessary. Returns the value
199 : pointer on success, or NULL on error. */
200 :
201 : void *
202 : fd_gui_hist_kv_get_or_create( fd_gui_t * gui,
203 : int dbi,
204 : void const * key );
205 :
206 : /* fd_gui_hist_kv_get is like fd_gui_hist_kv_get_or_create, but returns
207 : NULL when the record is not found instead of creating one. */
208 :
209 : void *
210 : fd_gui_hist_kv_get( fd_gui_t * gui,
211 : int dbi,
212 : void const * key );
213 :
214 : /* fd_gui_hist_kv_get_slot_any looks up the slot record for the first
215 : key matching (slot, *) and returns a pointer to its value. */
216 :
217 : void *
218 : fd_gui_hist_kv_get_slot_any( fd_gui_t * gui,
219 : int dbi,
220 : ulong slot );
221 :
222 : fd_gui_hist_kv_slot_iter_t *
223 : fd_gui_hist_kv_iter_begin( fd_gui_t * gui,
224 : fd_gui_hist_kv_slot_iter_t * iter,
225 : int dbi,
226 : ulong slot );
227 :
228 : int
229 : fd_gui_hist_kv_iter_next( fd_gui_hist_kv_slot_iter_t * iter );
230 :
231 : /* ---- Eviction ------------------------------------------------------- */
232 :
233 : /* fd_gui_hist_evict_step does at most one bounded unit of eviction
234 : work.
235 :
236 : The store is bounded by its configured map size. When it grows near
237 : full, the oldest epoch is evicted whole: its EPOCH record, every KV
238 : row for the epoch's slots, and every time-series row in the wallclock
239 : window the epoch spanned. An epoch can hold a lot of data, so the
240 : work is spread across many bounded batches.
241 :
242 : If the store is below the high-water threshold and no eviction
243 : is in progress, it does nothing and returns 0. Otherwise it advances
244 : the current epoch's cascade by one batch (or starts a new cascade on
245 : the oldest epoch), returning 1. No-op (returns 0) if the store is
246 : unavailable. */
247 :
248 : int
249 : fd_gui_hist_evict_step( fd_gui_t * gui );
250 :
251 : /* fd_gui_hist_evict_oldest evicts the single oldest epoch in its
252 : entirety, synchronously and unconditionally.
253 :
254 : It is the slow-path used when a write hits map-full. Returns 1 if an
255 : epoch was evicted, 0 if there was nothing to evict or the store is
256 : unavailable. */
257 :
258 : int
259 : fd_gui_hist_evict_oldest( fd_gui_t * gui );
260 :
261 : /* fd_gui_hist_evict_ts_oldest sheds time-series data oldest-first.
262 :
263 : It is the slow-path used when a write hits map-full. Returns 1 if it
264 : evicted a window's worth of records, 0 if every time-series DB is
265 : already empty or the store is unavailable. */
266 :
267 : int
268 : fd_gui_hist_evict_ts_oldest( fd_gui_t * gui );
269 :
270 : FD_PROTOTYPES_END
271 :
272 : #endif /* HEADER_fd_src_disco_gui_fd_gui_hist_h */
|
