Line data Source code
1 : #include "fd_cost_tracker_store.h"
2 :
3 : #include <errno.h>
4 : #include <unistd.h>
5 :
6 162 : #define FD_COST_TRACKER_STORE_MAGIC (0xF17EDA2CC057B000UL) /* FIREDANCER COST TRACKER STORE V0 */
7 :
8 : /* set_t is the metadata of one fork's cost tracker. An acquired cost
9 : tracker that is not cached is on disk. */
10 : struct set {
11 : ushort next;
12 : ushort cache_idx; /* USHORT_MAX if the cost tracker is not cached */
13 : };
14 : typedef struct set set_t;
15 :
16 : #define POOL_NAME set_pool
17 324 : #define POOL_T set_t
18 : #define POOL_IDX_T ushort
19 : #include "../../util/tmpl/fd_pool.c"
20 :
21 : /* cache_ent_t is an in-memory slot holding one cost tracker, plus the
22 : bookkeeping needed to pin and evict it. */
23 : struct cache_ent {
24 : ulong set_idx; /* ULONG_MAX if the entry is empty */
25 : ulong pin_cnt;
26 : ulong lru;
27 : uchar data[ FD_COST_TRACKER_FOOTPRINT ] __attribute__((aligned(FD_COST_TRACKER_ALIGN)));
28 : };
29 : typedef struct cache_ent cache_ent_t;
30 :
31 : struct fd_cost_tracker_store {
32 : ulong magic;
33 : ulong cache_cnt;
34 : int disk_fd;
35 : ulong bench_max_cost_per_block;
36 : ulong seed;
37 : ulong set_off;
38 : ulong cache_off;
39 : ulong lru;
40 : };
41 :
42 : static inline set_t *
43 36339 : sets( fd_cost_tracker_store_t const * store ) {
44 36339 : return fd_type_pun( (uchar *)store + store->set_off );
45 36339 : }
46 :
47 : static inline cache_ent_t *
48 27939 : cache( fd_cost_tracker_store_t const * store ) {
49 27939 : return fd_type_pun( (uchar *)store + store->cache_off );
50 27939 : }
51 :
52 : static inline set_t *
53 : set_query( fd_cost_tracker_store_t const * store,
54 15672 : ulong set_idx ) {
55 15672 : set_t * pool = sets( store );
56 15672 : FD_CHECK_CRIT( set_idx<set_pool_max( pool ), "invariant violation: invalid cost tracker set index" );
57 15672 : return pool + set_idx;
58 15672 : }
59 :
60 : static void
61 : disk_read( fd_cost_tracker_store_t * store,
62 : void * dst,
63 21 : ulong set_idx ) {
64 21 : ulong off = set_idx * FD_COST_TRACKER_FOOTPRINT;
65 21 : ulong sz = FD_COST_TRACKER_FOOTPRINT;
66 21 : ulong got = 0UL;
67 42 : while( got<sz ) {
68 21 : long n = pread( store->disk_fd, (uchar *)dst+got, sz-got, (long)(off+got) );
69 21 : if( FD_UNLIKELY( n<0L ) ) {
70 0 : if( FD_LIKELY( errno==EINTR ) ) continue;
71 0 : FD_LOG_CRIT(( "pread(cost tracker spill file, fd=%d) failed (%i-%s)", store->disk_fd, errno, fd_io_strerror( errno ) ));
72 0 : }
73 21 : if( FD_UNLIKELY( !n ) ) {
74 0 : FD_LOG_CRIT(( "unexpected EOF in cost tracker spill file (set=%lu offset=%lu size=%lu)", set_idx, off+got, sz-got ));
75 0 : }
76 21 : got += (ulong)n;
77 21 : }
78 21 : }
79 :
80 : static void
81 : disk_write( fd_cost_tracker_store_t * store,
82 : void const * src,
83 33 : ulong set_idx ) {
84 33 : ulong off = set_idx * FD_COST_TRACKER_FOOTPRINT;
85 33 : ulong sz = FD_COST_TRACKER_FOOTPRINT;
86 33 : ulong put = 0UL;
87 66 : while( put<sz ) {
88 33 : long n = pwrite( store->disk_fd, (uchar const *)src+put, sz-put, (long)(off+put) );
89 33 : if( FD_LIKELY( n>0L ) ) {
90 33 : put += (ulong)n;
91 33 : continue;
92 33 : }
93 0 : if( FD_UNLIKELY( n<0L && errno==EINTR ) ) continue;
94 0 : if( FD_UNLIKELY( !n ) ) errno = EIO;
95 0 : FD_LOG_CRIT(( "pwrite(cost tracker spill file, fd=%d) failed (%i-%s)", store->disk_fd, errno, fd_io_strerror( errno ) ));
96 0 : }
97 33 : }
98 :
99 : /* cache_evict frees the least recently used unpinned cache entry for
100 : the cost tracker of set_idx, writing its previous cost tracker back
101 : to disk. */
102 :
103 : static cache_ent_t *
104 : cache_evict( fd_cost_tracker_store_t * store,
105 7374 : ulong set_idx ) {
106 7374 : set_t * set = sets( store );
107 7374 : cache_ent_t * ent = cache( store );
108 :
109 7374 : cache_ent_t * victim = NULL;
110 65964 : for( ulong i=0UL; i<store->cache_cnt; i++ ) {
111 58590 : if( !ent[i].pin_cnt && (!victim || ent[i].lru<victim->lru) ) victim = ent+i;
112 58590 : }
113 7374 : FD_CHECK_CRIT( victim, "every cost tracker cache entry is pinned" );
114 :
115 7374 : if( victim->set_idx!=ULONG_MAX ) {
116 33 : disk_write( store, victim->data, victim->set_idx );
117 33 : set[ victim->set_idx ].cache_idx = USHORT_MAX;
118 33 : }
119 7374 : victim->set_idx = set_idx;
120 7374 : set[ set_idx ].cache_idx = (ushort)( victim - ent );
121 7374 : return victim;
122 7374 : }
123 :
124 : static cache_ent_t *
125 : cache_get( fd_cost_tracker_store_t * store,
126 7158 : ulong set_idx ) {
127 7158 : set_t * set = set_query( store, set_idx );
128 7158 : if( set->cache_idx!=USHORT_MAX ) return cache( store ) + set->cache_idx;
129 :
130 21 : cache_ent_t * ent = cache_evict( store, set_idx );
131 21 : disk_read( store, ent->data, set_idx );
132 21 : return ent;
133 7158 : }
134 :
135 : static cache_ent_t *
136 : cached_ent( fd_cost_tracker_store_t const * store,
137 7491 : ulong set_idx ) {
138 7491 : set_t const * set = set_query( store, set_idx );
139 7491 : FD_CHECK_CRIT( set->cache_idx<store->cache_cnt, "invariant violation: cost tracker is not cached" );
140 7491 : return cache( store ) + set->cache_idx;
141 7491 : }
142 :
143 : ulong
144 1650 : fd_cost_tracker_store_align( void ) {
145 1650 : return FD_COST_TRACKER_STORE_ALIGN;
146 1650 : }
147 :
148 : ulong
149 : fd_cost_tracker_store_footprint( ulong max_live_slots,
150 915 : ulong cache_cnt ) {
151 915 : if( FD_UNLIKELY( !max_live_slots || max_live_slots>=USHORT_MAX || !cache_cnt ) ) return 0UL;
152 906 : cache_cnt = fd_ulong_min( cache_cnt, max_live_slots );
153 :
154 906 : ulong l = FD_LAYOUT_INIT;
155 906 : l = FD_LAYOUT_APPEND( l, FD_COST_TRACKER_STORE_ALIGN, sizeof(fd_cost_tracker_store_t) );
156 906 : l = FD_LAYOUT_APPEND( l, set_pool_align(), set_pool_footprint( max_live_slots ) );
157 906 : l = FD_LAYOUT_APPEND( l, alignof(cache_ent_t), sizeof(cache_ent_t)*cache_cnt );
158 906 : return FD_LAYOUT_FINI( l, FD_COST_TRACKER_STORE_ALIGN );
159 915 : }
160 :
161 : void *
162 : fd_cost_tracker_store_new( void * shmem,
163 : int disk_fd,
164 : ulong max_live_slots,
165 : ulong cache_cnt,
166 : ulong bench_max_cost_per_block,
167 171 : ulong seed ) {
168 171 : if( FD_UNLIKELY( !shmem ) ) {
169 3 : FD_LOG_WARNING(( "NULL shmem" ));
170 3 : return NULL;
171 3 : }
172 :
173 168 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)shmem, fd_cost_tracker_store_align() ) ) ) {
174 3 : FD_LOG_WARNING(( "misaligned shmem" ));
175 3 : return NULL;
176 3 : }
177 :
178 165 : if( FD_UNLIKELY( disk_fd<0 ) ) {
179 3 : FD_LOG_WARNING(( "invalid disk_fd %d", disk_fd ));
180 3 : return NULL;
181 3 : }
182 :
183 162 : if( FD_UNLIKELY( !fd_cost_tracker_store_footprint( max_live_slots, cache_cnt ) ) ) {
184 0 : FD_LOG_WARNING(( "invalid max_live_slots %lu or cache_cnt %lu", max_live_slots, cache_cnt ));
185 0 : return NULL;
186 0 : }
187 :
188 162 : cache_cnt = fd_ulong_min( cache_cnt, max_live_slots );
189 :
190 162 : FD_SCRATCH_ALLOC_INIT( l, shmem );
191 162 : fd_cost_tracker_store_t * store = FD_SCRATCH_ALLOC_APPEND( l, FD_COST_TRACKER_STORE_ALIGN, sizeof(fd_cost_tracker_store_t) );
192 162 : void * set_mem = FD_SCRATCH_ALLOC_APPEND( l, set_pool_align(), set_pool_footprint( max_live_slots ) );
193 162 : cache_ent_t * cache_mem = FD_SCRATCH_ALLOC_APPEND( l, alignof(cache_ent_t), sizeof(cache_ent_t)*cache_cnt );
194 162 : FD_SCRATCH_ALLOC_FINI( l, FD_COST_TRACKER_STORE_ALIGN );
195 :
196 162 : set_t * set_pool = set_pool_join( set_pool_new( set_mem, max_live_slots ) );
197 162 : if( FD_UNLIKELY( !set_pool ) ) {
198 0 : FD_LOG_WARNING(( "failed to create cost tracker set pool" ));
199 0 : return NULL;
200 0 : }
201 :
202 162 : store->cache_cnt = cache_cnt;
203 162 : store->disk_fd = disk_fd;
204 162 : store->bench_max_cost_per_block = bench_max_cost_per_block;
205 162 : store->seed = seed;
206 162 : store->set_off = (ulong)set_pool - (ulong)store;
207 162 : store->cache_off = (ulong)cache_mem - (ulong)store;
208 1242 : for( ulong i=0UL; i<cache_cnt; i++ ) cache_mem[ i ].pin_cnt = 0UL;
209 162 : fd_cost_tracker_store_reset( store );
210 :
211 162 : FD_COMPILER_MFENCE();
212 162 : FD_VOLATILE( store->magic ) = FD_COST_TRACKER_STORE_MAGIC;
213 162 : FD_COMPILER_MFENCE();
214 :
215 162 : return shmem;
216 162 : }
217 :
218 : fd_cost_tracker_store_t *
219 : fd_cost_tracker_store_join( void * shmem,
220 435 : int disk_fd ) {
221 435 : fd_cost_tracker_store_t * store = shmem;
222 :
223 435 : if( FD_UNLIKELY( !store ) ) {
224 3 : FD_LOG_WARNING(( "NULL shmem" ));
225 3 : return NULL;
226 3 : }
227 :
228 432 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)store, fd_cost_tracker_store_align() ) ) ) {
229 3 : FD_LOG_WARNING(( "misaligned shmem" ));
230 3 : return NULL;
231 3 : }
232 :
233 429 : if( FD_UNLIKELY( store->magic!=FD_COST_TRACKER_STORE_MAGIC ) ) {
234 0 : FD_LOG_WARNING(( "bad magic" ));
235 0 : return NULL;
236 0 : }
237 :
238 429 : if( FD_UNLIKELY( store->disk_fd!=disk_fd ) ) {
239 3 : FD_LOG_WARNING(( "disk_fd mismatch (store=%d join=%d)", store->disk_fd, disk_fd ));
240 3 : return NULL;
241 3 : }
242 :
243 426 : return store;
244 429 : }
245 :
246 : void
247 4917 : fd_cost_tracker_store_reset( fd_cost_tracker_store_t * store ) {
248 4917 : set_t * pool = sets( store );
249 4917 : ulong max = set_pool_max( pool );
250 84846 : for( ulong i=0UL; i<max; i++ ) pool[ i ] = (set_t){ .cache_idx = USHORT_MAX };
251 4917 : set_pool_reset( pool );
252 4917 : store->lru = 0UL;
253 4917 : cache_ent_t * ent = cache( store );
254 44004 : for( ulong i=0UL; i<store->cache_cnt; i++ ) {
255 39087 : FD_CHECK_CRIT( !ent[i].pin_cnt, "invariant violation: resetting pinned cost tracker cache" );
256 39087 : ent[i].set_idx = ULONG_MAX;
257 39087 : ent[i].lru = 0UL;
258 39087 : }
259 4917 : }
260 :
261 : ushort
262 : fd_cost_tracker_store_new_fork( fd_cost_tracker_store_t * store,
263 7353 : ushort old_fork_id ) {
264 7353 : if( FD_LIKELY( old_fork_id!=USHORT_MAX ) ) fd_cost_tracker_store_release( store, old_fork_id );
265 :
266 7353 : set_t * pool = sets( store );
267 7353 : FD_CHECK_CRIT( set_pool_free( pool ), "invariant violation: no free cost tracker sets" );
268 7353 : ulong set_idx = set_pool_idx_acquire( pool );
269 7353 : cache_ent_t * ent = cache_evict( store, set_idx );
270 7353 : FD_TEST( fd_cost_tracker_join( fd_cost_tracker_new( ent->data, store->bench_max_cost_per_block, store->seed ) ) );
271 7353 : ent->lru = ++store->lru;
272 7353 : return (ushort)set_idx;
273 7353 : }
274 :
275 : void
276 : fd_cost_tracker_store_release( fd_cost_tracker_store_t * store,
277 1023 : ushort fork_id ) {
278 1023 : set_t * set = set_query( store, (ulong)fork_id );
279 1023 : if( set->cache_idx!=USHORT_MAX ) {
280 1020 : cache_ent_t * ent = cache( store ) + set->cache_idx;
281 1020 : FD_CHECK_CRIT( !ent->pin_cnt, "invariant violation: releasing a pinned cost tracker" );
282 1020 : ent->set_idx = ULONG_MAX;
283 1020 : ent->lru = 0UL;
284 1020 : set->cache_idx = USHORT_MAX;
285 1020 : }
286 1023 : set_pool_idx_release( sets( store ), (ulong)fork_id );
287 1023 : }
288 :
289 : fd_cost_tracker_t *
290 : fd_cost_tracker_store_pin( fd_cost_tracker_store_t * store,
291 7158 : ushort fork_id ) {
292 7158 : cache_ent_t * ent = cache_get( store, (ulong)fork_id );
293 7158 : ent->pin_cnt++;
294 7158 : ent->lru = ++store->lru;
295 7158 : return fd_type_pun( ent->data );
296 7158 : }
297 :
298 : void
299 : fd_cost_tracker_store_unpin( fd_cost_tracker_store_t * store,
300 7158 : ushort fork_id ) {
301 7158 : cache_ent_t * ent = cached_ent( store, (ulong)fork_id );
302 7158 : FD_CHECK_CRIT( ent->pin_cnt, "invariant violation: unpinning an unpinned cost tracker" );
303 7158 : ent->pin_cnt--;
304 7158 : }
305 :
306 : fd_cost_tracker_t *
307 : fd_cost_tracker_store_peek( fd_cost_tracker_store_t const * store,
308 333 : ushort fork_id ) {
309 333 : return fd_type_pun( cached_ent( store, (ulong)fork_id )->data );
310 333 : }
|
