Line data Source code
1 : #include "fd_epoch_credits.h"
2 : #include "../runtime/fd_runtime_const.h"
3 :
4 : #include <errno.h>
5 : #include <unistd.h>
6 :
7 201 : #define FD_EPOCH_CREDITS_STORE_MAGIC (0xF17EDA2CEC5E7000UL) /* FIREDANCER EPOCH CREDITS STORE V0 */
8 :
9 : /* set_t is the metadata for each fork. This is always in-memory. */
10 10467 : #define SET_SZ (sizeof(fd_epoch_credits_t)*FD_RUNTIME_MAX_VAT_VOTE_ACCOUNTS)
11 : struct set {
12 : ulong len;
13 : ulong refcnt;
14 : ushort next;
15 : ushort cache_idx; /* USHORT_MAX if the set is not cached */
16 : };
17 : typedef struct set set_t;
18 :
19 : #define POOL_NAME set_pool
20 402 : #define POOL_T set_t
21 : #define POOL_IDX_T ushort
22 : #include "../../util/tmpl/fd_pool.c"
23 :
24 : /* cache_ent_t is in-memory slot holding the epoch credits of one set */
25 : struct cache_ent {
26 : ulong set_idx; /* ULONG_MAX if the entry is empty */
27 : ulong pin_cnt;
28 : ulong lru;
29 : fd_epoch_credits_t credits[ FD_RUNTIME_MAX_VAT_VOTE_ACCOUNTS ];
30 : };
31 : typedef struct cache_ent cache_ent_t;
32 :
33 : struct fd_epoch_credits_store {
34 : ulong magic;
35 : ulong cache_cnt;
36 : int disk_fd;
37 : ulong set_off;
38 : ulong cache_off;
39 : ulong lru;
40 : };
41 : typedef struct fd_epoch_credits_store fd_epoch_credits_store_t;
42 :
43 : static inline set_t *
44 65106 : sets( fd_epoch_credits_store_t * store ) {
45 65106 : return fd_type_pun( (uchar *)store + store->set_off );
46 65106 : }
47 :
48 : static inline cache_ent_t *
49 27912 : cache( fd_epoch_credits_store_t * store ) {
50 27912 : return fd_type_pun( (uchar *)store + store->cache_off );
51 27912 : }
52 :
53 : static inline set_t *
54 : set_query( fd_epoch_credits_store_t * store,
55 43167 : ulong set_idx ) {
56 43167 : set_t * pool = sets( store );
57 43167 : FD_CHECK_CRIT( set_idx<set_pool_max( pool ), "invariant violation: invalid epoch credits set index" );
58 43167 : return pool + set_idx;
59 43167 : }
60 :
61 : static void
62 : disk_read( fd_epoch_credits_store_t * store,
63 : void * dst,
64 : ulong set_idx,
65 144 : ulong sz ) {
66 144 : ulong off = set_idx * SET_SZ;
67 144 : ulong got = 0UL;
68 288 : while( got<sz ) {
69 144 : long n = pread( store->disk_fd, (uchar *)dst+got, sz-got, (long)(off+got) );
70 144 : if( FD_UNLIKELY( n<0L ) ) {
71 0 : if( FD_LIKELY( errno==EINTR ) ) continue;
72 0 : FD_LOG_CRIT(( "pread(epoch credits spill file, fd=%d) failed (%i-%s)", store->disk_fd, errno, fd_io_strerror( errno ) ));
73 0 : }
74 144 : if( FD_UNLIKELY( !n ) ) {
75 0 : FD_LOG_CRIT(( "unexpected EOF in epoch credits spill file (set=%lu offset=%lu size=%lu)", set_idx, off+got, sz-got ));
76 0 : }
77 144 : got += (ulong)n;
78 144 : }
79 144 : }
80 :
81 : static void
82 : disk_write( fd_epoch_credits_store_t * store,
83 : void const * src,
84 : ulong set_idx,
85 183 : ulong sz ) {
86 183 : ulong off = set_idx * SET_SZ;
87 183 : ulong put = 0UL;
88 366 : while( put<sz ) {
89 183 : long n = pwrite( store->disk_fd, (uchar const *)src+put, sz-put, (long)(off+put) );
90 183 : if( FD_LIKELY( n>0L ) ) {
91 183 : put += (ulong)n;
92 183 : continue;
93 183 : }
94 0 : if( FD_UNLIKELY( n<0L && errno==EINTR ) ) continue;
95 0 : if( FD_UNLIKELY( !n ) ) errno = EIO;
96 0 : FD_LOG_CRIT(( "pwrite(epoch credits spill file, fd=%d) failed (%i-%s)", store->disk_fd, errno, fd_io_strerror( errno ) ));
97 0 : }
98 183 : }
99 :
100 : static cache_ent_t *
101 : cache_get( fd_epoch_credits_store_t * store,
102 11349 : ulong set_idx ) {
103 11349 : set_t * set = sets( store );
104 11349 : cache_ent_t * ent = cache( store );
105 11349 : if( set[ set_idx ].cache_idx!=USHORT_MAX ) return ent + set[ set_idx ].cache_idx;
106 :
107 : /* Pick cache eviction victim. Can evict anything that's not pinned
108 : and is the LRU. TODO: consider replacing scan with DLL. */
109 5472 : cache_ent_t * victim = NULL;
110 48111 : for( ulong i=0UL; i<store->cache_cnt; i++ ) {
111 42639 : if( !ent[i].pin_cnt && (!victim || ent[i].lru<victim->lru) ) victim = ent+i;
112 42639 : }
113 5472 : FD_CHECK_CRIT( victim, "every epoch credits cache entry is pinned" );
114 :
115 : /* Write back cache entry to disk */
116 5472 : if( victim->set_idx!=ULONG_MAX ) {
117 183 : disk_write( store, victim->credits, victim->set_idx, set[ victim->set_idx ].len*sizeof(fd_epoch_credits_t) );
118 183 : set[ victim->set_idx ].cache_idx = USHORT_MAX;
119 183 : }
120 5472 : if( set[ set_idx ].len ) disk_read( store, victim->credits, set_idx, set[ set_idx ].len*sizeof(fd_epoch_credits_t) );
121 5472 : victim->set_idx = set_idx;
122 5472 : set[ set_idx ].cache_idx = (ushort)( victim - ent );
123 5472 : return victim;
124 5472 : }
125 :
126 : ulong
127 1926 : fd_epoch_credits_store_align( void ) {
128 1926 : return FD_EPOCH_CREDITS_STORE_ALIGN;
129 1926 : }
130 :
131 : ulong
132 : fd_epoch_credits_store_footprint( ulong max_live_slots,
133 1083 : ulong cache_cnt ) {
134 1083 : if( FD_UNLIKELY( !max_live_slots || max_live_slots>=USHORT_MAX || !cache_cnt ) ) return 0UL;
135 1068 : cache_cnt = fd_ulong_min( cache_cnt, max_live_slots );
136 :
137 1068 : ulong l = FD_LAYOUT_INIT;
138 1068 : l = FD_LAYOUT_APPEND( l, FD_EPOCH_CREDITS_STORE_ALIGN, sizeof(fd_epoch_credits_store_t) );
139 1068 : l = FD_LAYOUT_APPEND( l, set_pool_align(), set_pool_footprint( max_live_slots ) );
140 1068 : l = FD_LAYOUT_APPEND( l, alignof(cache_ent_t), sizeof(cache_ent_t)*cache_cnt );
141 1068 : return FD_LAYOUT_FINI( l, FD_EPOCH_CREDITS_STORE_ALIGN );
142 1083 : }
143 :
144 : void *
145 : fd_epoch_credits_store_new( void * shmem,
146 : int disk_fd,
147 : ulong max_live_slots,
148 216 : ulong cache_cnt ) {
149 216 : if( FD_UNLIKELY( !shmem ) ) {
150 3 : FD_LOG_WARNING(( "NULL shmem" ));
151 3 : return NULL;
152 3 : }
153 :
154 213 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)shmem, fd_epoch_credits_store_align() ) ) ) {
155 3 : FD_LOG_WARNING(( "misaligned shmem" ));
156 3 : return NULL;
157 3 : }
158 :
159 210 : if( FD_UNLIKELY( disk_fd<0 ) ) {
160 3 : FD_LOG_WARNING(( "invalid disk_fd %d", disk_fd ));
161 3 : return NULL;
162 3 : }
163 :
164 207 : if( FD_UNLIKELY( !fd_epoch_credits_store_footprint( max_live_slots, cache_cnt ) ) ) {
165 6 : FD_LOG_WARNING(( "invalid max_live_slots %lu or cache_cnt %lu", max_live_slots, cache_cnt ));
166 6 : return NULL;
167 6 : }
168 :
169 201 : cache_cnt = fd_ulong_min( cache_cnt, max_live_slots );
170 :
171 201 : FD_SCRATCH_ALLOC_INIT( l, shmem );
172 201 : fd_epoch_credits_store_t * store = FD_SCRATCH_ALLOC_APPEND( l, FD_EPOCH_CREDITS_STORE_ALIGN, sizeof(fd_epoch_credits_store_t) );
173 201 : void * set_mem = FD_SCRATCH_ALLOC_APPEND( l, set_pool_align(), set_pool_footprint( max_live_slots ) );
174 201 : cache_ent_t * cache_mem = FD_SCRATCH_ALLOC_APPEND( l, alignof(cache_ent_t), sizeof(cache_ent_t)*cache_cnt );
175 201 : FD_SCRATCH_ALLOC_FINI( l, FD_EPOCH_CREDITS_STORE_ALIGN );
176 :
177 201 : set_t * set_pool = set_pool_join( set_pool_new( set_mem, max_live_slots ) );
178 201 : if( FD_UNLIKELY( !set_pool ) ) {
179 0 : FD_LOG_WARNING(( "failed to create epoch credits set pool" ));
180 0 : return NULL;
181 0 : }
182 :
183 201 : store->cache_cnt = cache_cnt;
184 201 : store->disk_fd = disk_fd;
185 201 : store->set_off = (ulong)set_pool - (ulong)store;
186 201 : store->cache_off = (ulong)cache_mem - (ulong)store;
187 1548 : for( ulong i=0UL; i<cache_cnt; i++ ) cache_mem[ i ].pin_cnt = 0UL;
188 201 : fd_epoch_credits_store_reset( store );
189 :
190 201 : FD_COMPILER_MFENCE();
191 201 : FD_VOLATILE( store->magic ) = FD_EPOCH_CREDITS_STORE_MAGIC;
192 201 : FD_COMPILER_MFENCE();
193 :
194 201 : return shmem;
195 201 : }
196 :
197 : fd_epoch_credits_store_t *
198 : fd_epoch_credits_store_join( void * shmem,
199 504 : int disk_fd ) {
200 504 : fd_epoch_credits_store_t * store = shmem;
201 :
202 504 : if( FD_UNLIKELY( !store ) ) {
203 3 : FD_LOG_WARNING(( "NULL shmem" ));
204 3 : return NULL;
205 3 : }
206 :
207 501 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)store, fd_epoch_credits_store_align() ) ) ) {
208 0 : FD_LOG_WARNING(( "misaligned shmem" ));
209 0 : return NULL;
210 0 : }
211 :
212 501 : if( FD_UNLIKELY( store->magic!=FD_EPOCH_CREDITS_STORE_MAGIC ) ) {
213 3 : FD_LOG_WARNING(( "bad magic" ));
214 3 : return NULL;
215 3 : }
216 :
217 498 : if( FD_UNLIKELY( store->disk_fd!=disk_fd ) ) {
218 3 : FD_LOG_WARNING(( "disk_fd mismatch (store=%d join=%d)", store->disk_fd, disk_fd ));
219 3 : return NULL;
220 3 : }
221 :
222 495 : return store;
223 498 : }
224 :
225 : void
226 5043 : fd_epoch_credits_store_reset( fd_epoch_credits_store_t * store ) {
227 5043 : set_t * pool = sets( store );
228 5043 : ulong max = set_pool_max( pool );
229 86880 : for( ulong i=0UL; i<max; i++ ) pool[ i ] = (set_t){ .cache_idx = USHORT_MAX };
230 5043 : set_pool_reset( pool );
231 5043 : store->lru = 0UL;
232 5043 : cache_ent_t * ent = cache( store );
233 45102 : for( ulong i=0UL; i<store->cache_cnt; i++ ) {
234 40059 : FD_CHECK_CRIT( !ent[i].pin_cnt, "invariant violation: resetting pinned epoch credits cache" );
235 40059 : ent[i].set_idx = ULONG_MAX;
236 40059 : ent[i].lru = 0UL;
237 40059 : }
238 5043 : }
239 :
240 : ushort
241 : fd_epoch_credits_store_new_fork( fd_epoch_credits_store_t * store,
242 5421 : ushort old_fork_id ) {
243 5421 : if( FD_LIKELY( old_fork_id!=USHORT_MAX ) ) fd_epoch_credits_store_release( store, old_fork_id );
244 :
245 5421 : set_t * pool = sets( store );
246 5421 : FD_CHECK_CRIT( set_pool_free( pool ), "invariant violation: no free epoch credits sets" );
247 5421 : ulong set_idx = set_pool_idx_acquire( pool );
248 5421 : pool[ set_idx ].refcnt = 1UL;
249 5421 : return (ushort)set_idx;
250 5421 : }
251 :
252 : void
253 : fd_epoch_credits_store_acquire( fd_epoch_credits_store_t * store,
254 7764 : ushort fork_id ) {
255 7764 : set_t * set = set_query( store, (ulong)fork_id );
256 7764 : FD_CHECK_CRIT( set->refcnt, "invariant violation: acquiring an unreferenced epoch credits set" );
257 7764 : set->refcnt++;
258 7764 : }
259 :
260 : void
261 : fd_epoch_credits_store_release( fd_epoch_credits_store_t * store,
262 12630 : ushort fork_id ) {
263 12630 : set_t * set = set_query( store, (ulong)fork_id );
264 12630 : FD_CHECK_CRIT( set->refcnt, "invariant violation: releasing an unreferenced epoch credits set" );
265 12630 : if( --set->refcnt ) return;
266 :
267 126 : set->len = 0UL;
268 126 : if( set->cache_idx!=USHORT_MAX ) {
269 96 : cache_ent_t * ent = cache( store ) + set->cache_idx;
270 96 : ent->set_idx = ULONG_MAX;
271 96 : ent->lru = 0UL;
272 96 : set->cache_idx = USHORT_MAX;
273 96 : }
274 126 : set_pool_idx_release( sets( store ), (ulong)fork_id );
275 126 : }
276 :
277 : fd_epoch_credits_view_t *
278 : fd_epoch_credits_view_init( fd_epoch_credits_view_t * view,
279 : fd_epoch_credits_store_t * store,
280 11349 : ushort fork_id ) {
281 11349 : if( FD_UNLIKELY( !view || !store ) ) return NULL;
282 :
283 11349 : ulong set_idx = (ulong)fork_id;
284 11349 : set_t * set = set_query( store, set_idx );
285 11349 : cache_ent_t * ent = cache_get( store, set_idx );
286 :
287 11349 : set->refcnt++;
288 11349 : ent->pin_cnt++;
289 11349 : ent->lru = ++store->lru;
290 11349 : if( FD_UNLIKELY( !set->len ) ) fd_memset( ent->credits, 0, SET_SZ );
291 :
292 11349 : *view = (fd_epoch_credits_view_t) {
293 11349 : .credits = ent->credits,
294 11349 : .store = store,
295 11349 : .len = set->len,
296 11349 : .set_idx = set_idx
297 11349 : };
298 11349 : return view;
299 11349 : }
300 :
301 : void
302 11349 : fd_epoch_credits_view_fini( fd_epoch_credits_view_t * view ) {
303 11349 : if( FD_UNLIKELY( !view || !view->credits ) ) return;
304 :
305 11349 : fd_epoch_credits_store_t * store = view->store;
306 11349 : set_t * set = set_query( store, view->set_idx );
307 11349 : FD_CHECK_CRIT( set->cache_idx<store->cache_cnt, "invariant violation: invalid epoch credits view" );
308 11349 : cache_ent_t * ent = cache( store ) + set->cache_idx;
309 11349 : FD_CHECK_CRIT( ent->set_idx==view->set_idx && ent->pin_cnt, "invariant violation: invalid epoch credits view" );
310 :
311 11349 : FD_CHECK_CRIT( view->len<=FD_RUNTIME_MAX_VAT_VOTE_ACCOUNTS, "invariant violation: invalid epoch credits length" );
312 11349 : set->len = view->len;
313 :
314 11349 : ent->pin_cnt--;
315 11349 : fd_epoch_credits_store_release( store, (ushort)view->set_idx );
316 :
317 11349 : view->credits = NULL;
318 11349 : view->store = NULL;
319 11349 : }
320 :
321 : fd_epoch_credits_t const *
322 : fd_epoch_credits_store_peek( fd_epoch_credits_store_t const * store,
323 : ushort fork_id,
324 75 : ulong * len ) {
325 75 : fd_epoch_credits_store_t * s = (fd_epoch_credits_store_t *)store;
326 75 : set_t const * set = set_query( s, (ulong)fork_id );
327 75 : FD_CHECK_CRIT( set->cache_idx<store->cache_cnt, "invariant violation: epoch credits set is not cached" );
328 75 : cache_ent_t const * ent = cache( s ) + set->cache_idx;
329 75 : FD_CHECK_CRIT( ent->set_idx==(ulong)fork_id && ent->pin_cnt, "invariant violation: epoch credits set is not pinned" );
330 75 : *len = set->len;
331 75 : return ent->credits;
332 75 : }
|
