Line data Source code
1 : #ifndef HEADER_fd_src_disco_gui_fd_gui_hist_h
2 : #define HEADER_fd_src_disco_gui_fd_gui_hist_h
3 :
4 : /* fd_gui_hist is the GUI's historical event store semantics layer. It
5 : sits on top of fd_gui_store and owns everything GUI-specific about
6 : the monitoring history: the named DBs (FD_GUI_HIST_* below), their
7 : key/value structs, fixed record sizes, the eviction policy, and the
8 : query API.
9 :
10 : There are two kinds of DB:
11 :
12 : - TIME-SERIES (TS) DBs are append-only streams tagged by a
13 : wallclock window. Written by fd_gui_hist_ts_append, read by
14 : time-window range scan.
15 : - KEY-VALUE (KV) DBs hold one record per key. Written/read by
16 : point key (fd_gui_hist_kv_get_or_create / _get / _get_any).
17 : */
18 :
19 : #include "../../util/fd_util_base.h"
20 : #include "fd_gui_store.h"
21 : #include "../../flamenco/leaders/fd_leaders_base.h" /* MAX_SLOTS_PER_EPOCH */
22 : #include "../../flamenco/runtime/fd_slot_params.h" /* FD_SLOT_PARAMS_200MS */
23 :
24 : struct fd_gui;
25 : typedef struct fd_gui fd_gui_t;
26 :
27 18 : #define FD_GUI_HIST_MAGIC (0xf17e6d09147802UL)
28 :
29 : /* FD_GUI_HIST_TS_SKEW_NS bounds how far a TS record's timestamp may
30 : deviate from the trusted wallclock `now` at append time (in either
31 : direction). fd_gui_hist_ts_append clamps ts_ns into [now-skew, now+skew]
32 : before flooring it to a window. This bounds the out-of-orderness of
33 : the append stream which ensures the underlying ring buffer can efficiently evict old entries. */
34 :
35 6426 : #define FD_GUI_HIST_TS_SKEW_NS (10L*FD_GUI_HIST_RES_1S_NS) /* +/- 10 s */
36 6564 : #define FD_GUI_HIST_RES_1S_NS (1000000000L)
37 :
38 : /* FD_GUI_HIST_MIN_EPOCHS is the minimum number of epochs the store must
39 : hold, plus one more which can be evicted under space pressure. */
40 :
41 21 : #define FD_GUI_HIST_MIN_EPOCHS (3UL)
42 :
43 : /* FD_GUI_HIST_MAX_EPOCHS caps KV index provisioning at the number of
44 : epochs reachable within the TS index horizon (30 days) +2 for partial
45 : epochs at both ends of the horizon. */
46 :
47 3 : #define FD_GUI_HIST_MAX_EPOCHS ((FD_GUI_STORE_TS_IDX_DEPTH*(ulong)FD_GUI_HIST_RES_1S_NS)/(MAX_SLOTS_PER_EPOCH*((FD_SLOT_PARAMS_200MS).ns_per_slot))+2UL)
48 :
49 456 : #define FD_GUI_HIST_SCHEDULER_COUNTS (0) /* (ts, type) */
50 324 : #define FD_GUI_HIST_TILE_TIMERS (1) /* (ts, type) */
51 10101 : #define FD_GUI_HIST_SHRED_EVENTS (2) /* (ts, type, slot) */
52 3948 : #define FD_GUI_HIST_TXN_START (3) /* (ts, type, bank, txn) */
53 4044 : #define FD_GUI_HIST_TXN_END (4) /* (ts, type, bank, txn) */
54 606 : #define FD_GUI_HIST_TOWER (5) /* (ts, type) */
55 9915 : #define FD_GUI_HIST_SLOT (6) /* (slot, bank_seq) */
56 6927 : #define FD_GUI_HIST_LEADER_SLOT (7) /* (slot, bank_seq) */
57 645 : #define FD_GUI_HIST_EPOCH (8) /* (epoch) */
58 420 : #define FD_GUI_HIST_TILE_STATS (9) /* (ts, type) */
59 516 : #define FD_GUI_HIST_TXN_WATERFALL (10) /* (ts, type) */
60 3012 : #define FD_GUI_HIST_CNT (11)
61 :
62 : struct fd_gui_hist_metrics {
63 : /* Writes that hit MAP_FULL and were dropped. */
64 : ulong map_full[ FD_GUI_HIST_CNT ];
65 : /* Writes that evicted records before succeeding. */
66 : ulong reserves [ FD_GUI_HIST_CNT ];
67 : };
68 : typedef struct fd_gui_hist_metrics fd_gui_hist_metrics_t;
69 :
70 : struct fd_gui_hist_slot_key { ulong slot; ulong bank_seq; };
71 : struct fd_gui_hist_leader_slot_key { ulong slot; ulong bank_seq; };
72 : struct fd_gui_hist_epoch_key { ulong epoch; };
73 :
74 : typedef struct fd_gui_hist_slot_key fd_gui_hist_slot_key_t;
75 : typedef struct fd_gui_hist_leader_slot_key fd_gui_hist_leader_slot_key_t;
76 : typedef struct fd_gui_hist_epoch_key fd_gui_hist_epoch_key_t;
77 :
78 : struct fd_gui_hist_private;
79 : typedef struct fd_gui_hist_private fd_gui_hist_t;
80 :
81 : /* fd_gui_hist_kv_slot_iter_t iterates every record for `slot` (every
82 : fork's block) in a slot-keyed KV DB. Used to enumerate the
83 : equivocating forks of a slot. */
84 : struct fd_gui_hist_kv_slot_iter {
85 : void const * rec; /* current record; NULL when done */
86 : ulong bank_seq; /* current record's bank_seq */
87 : fd_gui_store_kv_iter_t _it;
88 : };
89 : typedef struct fd_gui_hist_kv_slot_iter fd_gui_hist_kv_slot_iter_t;
90 :
91 : /* fd_gui_hist_range_filter_fn is an optional per-record predicate
92 : evaluated during a range scan: called with a pointer to the DB's
93 : record and the caller's `ctx`, it returns non-zero to emit the record
94 : or zero to skip it. A NULL filter emits every record in the window
95 : range. */
96 :
97 : typedef int (*fd_gui_hist_range_filter_fn)( void const * rec, void * ctx );
98 :
99 : /* fd_gui_hist_iter_t iterates the records produced by a time-window
100 : range query.
101 :
102 : Records are emitted oldest-window-first, but the backend does not
103 : strictly order records within a window.
104 : */
105 : struct fd_gui_hist_iter {
106 : void const * rec;
107 : ulong rec_sz;
108 :
109 : int _dbi;
110 : int _emitted;
111 : fd_gui_hist_range_filter_fn _filter;
112 : void * _filter_ctx;
113 : fd_gui_store_ts_iter_t _it;
114 : };
115 :
116 : typedef struct fd_gui_hist_iter fd_gui_hist_iter_t;
117 :
118 : FD_PROTOTYPES_BEGIN
119 :
120 : FD_FN_CONST ulong
121 : fd_gui_hist_align( void );
122 :
123 : FD_FN_CONST ulong
124 : fd_gui_hist_footprint( void );
125 :
126 : /* fd_gui_hist_new formats the workspace region `mem`
127 : (>= fd_gui_hist_footprint bytes, fd_gui_hist_align aligned) as the
128 : history semantics layer over the store `db`. Returns `mem` on success
129 : or NULL on failure (logged). */
130 :
131 : void *
132 : fd_gui_hist_new( void * mem,
133 : fd_gui_store_t const * db );
134 :
135 : /* fd_gui_hist_join joins a region formatted by fd_gui_hist_new,
136 : returning a usable handle (mem may be NULL -> NULL). */
137 :
138 : fd_gui_hist_t *
139 : fd_gui_hist_join( void * mem );
140 :
141 : void *
142 : fd_gui_hist_leave( fd_gui_hist_t * hist );
143 :
144 : void *
145 : fd_gui_hist_delete( void * mem );
146 :
147 : /* fd_gui_hist_db_descs returns a pointer to a static array to be passed
148 : to fd_gui_store_new. */
149 :
150 : fd_gui_store_desc_t const *
151 : fd_gui_hist_db_descs( ulong store_bytes );
152 :
153 : FD_FN_CONST static inline ulong
154 36 : fd_gui_hist_db_cnt( void ) { return FD_GUI_HIST_CNT; }
155 :
156 : fd_gui_hist_metrics_t const *
157 : fd_gui_hist_metrics( fd_gui_t const * gui );
158 :
159 : /* ---- TS read/write -------------------------------------------------- */
160 :
161 : /* fd_gui_hist_ts_append appends one record `val` into time-series
162 : database `dbi`, tagged with wallclock timestamp `ts_ns`. `now` is
163 : the current wallclock at append time.
164 :
165 : Will evict old entries to make room if necessary. Returns 0 on
166 : success, -1 on failure (e.g. a misconfigured DB size). */
167 :
168 : int
169 : fd_gui_hist_ts_append( fd_gui_t * gui,
170 : int dbi,
171 : long now,
172 : long ts_ns,
173 : void const * val );
174 :
175 : int
176 : fd_gui_hist_range_begin( fd_gui_t * gui,
177 : fd_gui_hist_iter_t * iter,
178 : int dbi,
179 : long lo_ns,
180 : long hi_ns,
181 : fd_gui_hist_range_filter_fn filter,
182 : void * filter_ctx );
183 :
184 : int
185 : fd_gui_hist_range_next( fd_gui_hist_iter_t * iter );
186 :
187 : void
188 : fd_gui_hist_range_end( fd_gui_hist_iter_t * iter );
189 :
190 :
191 : /* ---- KV read/write -------------------------------------------------- */
192 :
193 : /* fd_gui_hist_kv_get_or_create reserves (creating if absent) the record
194 : for `*key` in a KV DB and returns a mutable pointer to the record's
195 : value, or NULL on failure.
196 :
197 : Will evict old entries to make room if necessary. Returns the value
198 : pointer on success, or NULL on error. */
199 :
200 : void *
201 : fd_gui_hist_kv_get_or_create( fd_gui_t * gui,
202 : int dbi,
203 : void const * key );
204 :
205 : /* fd_gui_hist_kv_get is like fd_gui_hist_kv_get_or_create, but returns
206 : NULL when the record is not found instead of creating one. */
207 :
208 : void *
209 : fd_gui_hist_kv_get( fd_gui_t * gui,
210 : int dbi,
211 : void const * key );
212 :
213 : /* fd_gui_hist_kv_get_slot_any looks up the slot record for the first
214 : key matching (slot, *) and returns a pointer to its value. */
215 :
216 : void *
217 : fd_gui_hist_kv_get_slot_any( fd_gui_t * gui,
218 : int dbi,
219 : ulong slot );
220 :
221 : fd_gui_hist_kv_slot_iter_t *
222 : fd_gui_hist_kv_iter_begin( fd_gui_t * gui,
223 : fd_gui_hist_kv_slot_iter_t * iter,
224 : int dbi,
225 : ulong slot );
226 :
227 : int
228 : fd_gui_hist_kv_iter_next( fd_gui_hist_kv_slot_iter_t * iter );
229 :
230 : /* ---- Eviction ------------------------------------------------------- */
231 :
232 : /* fd_gui_hist_evict_step does at most one bounded unit of eviction
233 : work.
234 :
235 : The store is bounded by its configured map size. When it grows near
236 : full, the oldest epoch is evicted whole: its EPOCH record, every KV
237 : row for the epoch's slots, and every time-series row in the wallclock
238 : window the epoch spanned. An epoch can hold a lot of data, so the
239 : work is spread across many bounded batches.
240 :
241 : If the store is below the high-water threshold and no eviction
242 : is in progress, it does nothing and returns 0. Otherwise it advances
243 : the current epoch's cascade by one batch (or starts a new cascade on
244 : the oldest epoch), returning 1. No-op (returns 0) if the store is
245 : unavailable. */
246 :
247 : int
248 : fd_gui_hist_evict_step( fd_gui_t * gui );
249 :
250 : /* fd_gui_hist_evict_oldest evicts the single oldest epoch in its
251 : entirety, synchronously and unconditionally.
252 :
253 : It is the slow-path used when a write hits map-full. Returns 1 if an
254 : epoch was evicted, 0 if there was nothing to evict or the store is
255 : unavailable. */
256 :
257 : int
258 : fd_gui_hist_evict_oldest( fd_gui_t * gui );
259 :
260 : /* fd_gui_hist_evict_ts_oldest sheds time-series data oldest-first.
261 :
262 : It is the slow-path used when a write hits map-full. Returns 1 if it
263 : evicted a window's worth of records, 0 if every time-series DB is
264 : already empty or the store is unavailable. */
265 :
266 : int
267 : fd_gui_hist_evict_ts_oldest( fd_gui_t * gui );
268 :
269 : FD_PROTOTYPES_END
270 :
271 : #endif /* HEADER_fd_src_disco_gui_fd_gui_hist_h */
|
