Line data Source code
1 0 : case FD_VINYL_REQ_TYPE_RELEASE: {
2 :
3 0 : ulong req_flags = (ulong)req->flags;
4 0 : fd_vinyl_key_t const * req_key = MAP_REQ_GADDR( req->key_gaddr, fd_vinyl_key_t const, batch_cnt );
5 0 : ulong * req_val_gaddr = MAP_REQ_GADDR( req->val_gaddr_gaddr, ulong, batch_cnt );
6 0 : schar * req_err = MAP_REQ_GADDR( req->err_gaddr, schar, batch_cnt );
7 :
8 0 : int req_flag_modify = fd_vinyl_req_flag_modify( req_flags );
9 0 : int req_flag_ignore = fd_vinyl_req_flag_ignore( req_flags );
10 0 : int req_flag_erase = fd_vinyl_req_flag_erase ( req_flags );
11 0 : int req_flag_by_key = fd_vinyl_req_flag_by_key( req_flags );
12 0 : int req_evict_prio = fd_vinyl_req_evict_prio ( req_flags );
13 :
14 0 : if( FD_UNLIKELY( (!!batch_cnt) & ( ((!req_key ) & req_flag_by_key ) |
15 0 : ((!req_val_gaddr) & (!req_flag_by_key)) |
16 0 : ( !req_err ) ) ) ) {
17 0 : comp_err = FD_VINYL_ERR_INVAL;
18 0 : break;
19 0 : }
20 :
21 0 : for( ulong batch_idx=0UL; batch_idx<batch_cnt; batch_idx++ ) {
22 :
23 0 : # define DONE(err) do { \
24 0 : int _err = (err); \
25 0 : FD_COMPILER_MFENCE(); \
26 0 : req_err[ batch_idx ] = (schar)_err; \
27 0 : FD_COMPILER_MFENCE(); \
28 0 : quota_rem += (ulong) !_err; \
29 0 : fail_cnt += (ulong)!!_err; \
30 0 : goto next_release; /* sigh ... can't use continue */ \
31 0 : } while(0)
32 :
33 : /* If a pair has been acquired, there is a non-zero ref count line
34 : holding it. This line connects the meta element (with the
35 : bstream state at seq_present) to the data object (with the
36 : cached pair data). Determine the line, meta element and data
37 : object associated with the acquire to release. */
38 :
39 0 : fd_vinyl_data_obj_t * obj;
40 0 : ulong line_idx;
41 0 : ulong ele_idx;
42 0 : ulong ver;
43 0 : long ref;
44 :
45 0 : if( FD_LIKELY( !req_flag_by_key ) ) { /* Release by val_gaddr */
46 :
47 0 : void * _obj = (void *)( req_val_gaddr[ batch_idx ]
48 0 : + data_laddr0 - sizeof(fd_vinyl_bstream_phdr_t) - sizeof(fd_vinyl_data_obj_t) );
49 :
50 0 : if( FD_UNLIKELY( !fd_vinyl_data_is_valid_obj( _obj, vol, vol_cnt ) ) ) DONE( FD_VINYL_ERR_INVAL );
51 0 : obj = (fd_vinyl_data_obj_t *)_obj;
52 :
53 0 : if( FD_UNLIKELY( obj->rd_active ) ) DONE( FD_VINYL_ERR_INVAL );
54 :
55 0 : line_idx = obj->line_idx;
56 0 : if( FD_UNLIKELY( line_idx>=line_cnt ) || FD_UNLIKELY( obj!=line[ line_idx ].obj ) ) DONE( FD_VINYL_ERR_INVAL );
57 :
58 0 : ele_idx = line[ line_idx ].ele_idx;
59 0 : if( FD_UNLIKELY( ele_idx>=ele_max ) || FD_UNLIKELY( ele0[ ele_idx ].line_idx!=line_idx ) ) DONE( FD_VINYL_ERR_INVAL );
60 : /* FIXME: MAKE SURE ELE0[ ELE_IDX ] IS IN USE FOR DATA INTEGRITY! */
61 :
62 0 : ulong ctl = line[ line_idx ].ctl;
63 :
64 0 : ver = fd_vinyl_line_ctl_ver( ctl );
65 0 : ref = fd_vinyl_line_ctl_ref( ctl );
66 :
67 0 : if( FD_UNLIKELY( !ref ) ) DONE( FD_VINYL_ERR_INVAL ); /* Pair key exists and is cached ... but not acquired */
68 :
69 0 : } else { /* Release by key */
70 :
71 0 : fd_vinyl_key_t const * key = req_key + batch_idx;
72 :
73 0 : ulong memo = fd_vinyl_key_memo( meta_seed, key ); /* This can be slow which is why releasing by val_gaddr is preferred */
74 :
75 0 : ulong _ele_idx; /* avoid pointer escape */
76 0 : int err = fd_vinyl_meta_query_fast( ele0, ele_max, key, memo, &_ele_idx );
77 0 : ele_idx = _ele_idx; /* in [0,ele_max) */
78 :
79 0 : if( FD_UNLIKELY( err ) ) DONE( FD_VINYL_ERR_INVAL ); /* Pair key does not exist ... can't have been acquired */
80 :
81 0 : line_idx = ele0[ ele_idx ].line_idx;
82 :
83 0 : if( FD_UNLIKELY( line_idx>=line_cnt ) ) { /* Pair key exists but is not cached ... can't have been acquired */
84 0 : FD_CRIT( line_idx==ULONG_MAX, "corruption detected" );
85 0 : DONE( FD_VINYL_ERR_INVAL );
86 0 : }
87 :
88 0 : FD_CRIT( ele_idx==line[ line_idx ].ele_idx, "corruption detected" );
89 :
90 0 : obj = line[ line_idx ].obj;
91 :
92 0 : FD_ALERT( fd_vinyl_data_is_valid_obj( obj, vol, vol_cnt ), "corruption detected" );
93 0 : FD_CRIT ( obj->line_idx==line_idx, "corruption detected" );
94 0 : FD_CRIT ( !obj->rd_active, "corruption detected" );
95 :
96 0 : ulong ctl = line[ line_idx ].ctl;
97 :
98 0 : ver = fd_vinyl_line_ctl_ver( ctl );
99 0 : ref = fd_vinyl_line_ctl_ref( ctl );
100 :
101 0 : if( FD_UNLIKELY( !ref ) ) DONE( FD_VINYL_ERR_INVAL ); /* Pair key exists and is cached ... but not acquired */
102 :
103 0 : }
104 :
105 : /* At this point, we are releasing an acquire of the object obj,
106 : cached at line line_idx with metadata at ele_idx. */
107 :
108 0 : fd_vinyl_bstream_phdr_t * phdr = fd_vinyl_data_obj_phdr( obj );
109 :
110 0 : if( FD_LIKELY( ref>0L ) ) {
111 :
112 : /* At this point, we are releasing an acquire for read. If
113 : the client indicated they modified pair key, we don't have
114 : data integrity anymore and we CRIT. Otherwise, we update
115 : line eviction priority and ref count to do the release. */
116 :
117 0 : if( FD_UNLIKELY( req_flag_modify ) ) FD_LOG_CRIT(( "client modified read only acquire" ));
118 :
119 0 : FD_CRIT( phdr->ctl==fd_vinyl_bstream_ctl( FD_VINYL_BSTREAM_CTL_TYPE_PAIR,
120 0 : FD_VINYL_BSTREAM_CTL_STYLE_RAW,
121 0 : (ulong)ele0[ ele_idx ].phdr.info.val_sz ), "corruption detected" );
122 0 : FD_CRIT( fd_vinyl_key_eq( &phdr->key, &ele0[ ele_idx ].phdr.key ), "corruption detected" );
123 0 : FD_CRIT( !memcmp( &phdr->info, &ele0[ ele_idx ].phdr.info, sizeof(fd_vinyl_info_t) ), "corruption detected" );
124 :
125 0 : fd_vinyl_line_evict_prio( &vinyl->line_idx_lru, line, line_cnt, line_idx, req_evict_prio );
126 :
127 0 : line[ line_idx ].ctl = fd_vinyl_line_ctl( ver, ref-1L ); /* don't bump ver */
128 :
129 0 : DONE( FD_VINYL_SUCCESS );
130 0 : }
131 :
132 : /* At this point, we are releasing an acquire for modify */
133 :
134 0 : ulong phdr_ctl = phdr->ctl;
135 :
136 0 : int modifying_existing = (phdr_ctl!=ULONG_MAX);
137 :
138 0 : if( FD_LIKELY( req_flag_modify & (!req_flag_erase) ) ) {
139 :
140 : /* At this point, we are either finishing up modifying an
141 : existing pair (modifying_existing 1) or finishing up creating
142 : a new pair (modifying_existing 0). Cache the object in the
143 : smallest size class that supports it. Note that the client
144 : could have modified info so we only validate ctl and key
145 : (FIXME: consider validating memo too?). */
146 :
147 0 : FD_CRIT( (!modifying_existing) |
148 0 : (phdr_ctl==fd_vinyl_bstream_ctl( FD_VINYL_BSTREAM_CTL_TYPE_PAIR,
149 0 : FD_VINYL_BSTREAM_CTL_STYLE_RAW,
150 0 : (ulong)ele0[ ele_idx ].phdr.info.val_sz )), "corruption detected" );
151 0 : FD_CRIT( fd_vinyl_key_eq( &phdr->key, &ele0[ ele_idx ].phdr.key ), "corruption detected" );
152 :
153 0 : ulong val_sz_after = (ulong)phdr->info.val_sz;
154 :
155 0 : if( FD_UNLIKELY( val_sz_after > fd_vinyl_data_obj_val_max( obj ) ) ) FD_LOG_CRIT(( "client overran memory" ));
156 :
157 0 : ulong szc_before = (ulong)obj->szc;
158 0 : ulong szc_after = fd_vinyl_data_szc( val_sz_after );
159 :
160 0 : if( FD_UNLIKELY( szc_before!=szc_after ) ) {
161 :
162 0 : FD_CRIT( szc_after<szc_before, "corruption detected" );
163 :
164 0 : fd_vinyl_data_obj_t * obj_after = fd_vinyl_data_alloc( data, szc_after );
165 0 : if( FD_UNLIKELY( !obj_after ) ) FD_LOG_CRIT(( "increase data cache size" ));
166 :
167 0 : fd_vinyl_bstream_phdr_t * phdr_after = fd_vinyl_data_obj_phdr( obj_after );
168 :
169 0 : memcpy( phdr_after, phdr, sizeof(fd_vinyl_bstream_phdr_t) + val_sz_after );
170 :
171 0 : fd_vinyl_data_free( data, obj );
172 :
173 0 : obj = obj_after;
174 0 : phdr = phdr_after;
175 :
176 0 : }
177 :
178 : /* Append to the updated pair key to the bstream. If we are
179 : finishing up modifying an existing pair, this will create 1
180 : item of bstream garbage (the old version of the pair). If we
181 : are finishing up creating a new pair, this will not create
182 : any garbage. Note that this will zero out any pair zero
183 : padding region and populate footer hashes. */
184 :
185 0 : if( FD_LIKELY( modifying_existing ) ) {
186 :
187 0 : ulong val_esz_before = fd_vinyl_bstream_ctl_sz( ele0[ ele_idx ].phdr.ctl );
188 :
189 0 : accum_garbage_cnt++;
190 0 : accum_garbage_sz += fd_vinyl_bstream_pair_sz( val_esz_before );
191 :
192 0 : }
193 :
194 0 : phdr->ctl = fd_vinyl_bstream_ctl( FD_VINYL_BSTREAM_CTL_TYPE_PAIR, FD_VINYL_BSTREAM_CTL_STYLE_RAW, val_sz_after );
195 : /*phdr->key already init */
196 : /*phdr->info already init */
197 :
198 0 : int style_after;
199 0 : ulong val_esz_after;
200 0 : ulong seq_after = fd_vinyl_io_append_pair_inplace( io, vinyl->style, phdr, &style_after, &val_esz_after );
201 0 : append_cnt++;
202 :
203 : /* Update the line and meta to match. Note that setting meta
204 : element ele_idx phdr.ctl to something other than ULONG_MAX
205 : marks a pair that was being created as no longer being
206 : created. For a pair that already existed, we also need to
207 : update phdr.ctl to reflect that we might be storing this in
208 : the stream in a different format than it was stored in
209 : bstream before. Since we are changing shared fields of meta
210 : element ele_idx, we need to use prepare / publish semantics. */
211 :
212 0 : line[ line_idx ].obj = obj; obj->line_idx = line_idx; obj->rd_active = (short)0;
213 : //line[ line_idx ].ele_idx ... already init
214 0 : line[ line_idx ].ctl = fd_vinyl_line_ctl( ver+1L, 0L ); /* bump ver */
215 :
216 0 : fd_vinyl_line_evict_prio( &vinyl->line_idx_lru, line, line_cnt, line_idx, req_evict_prio );
217 :
218 0 : fd_vinyl_meta_prepare_fast( lock, lock_shift, ele_idx );
219 :
220 : //ele0[ ele_idx ].memo = already init
221 0 : ele0[ ele_idx ].phdr.ctl = fd_vinyl_bstream_ctl( FD_VINYL_BSTREAM_CTL_TYPE_PAIR, style_after, val_esz_after );
222 : //ele0[ ele_idx ].phdr.key = already init
223 0 : ele0[ ele_idx ].phdr.info = phdr->info;
224 0 : ele0[ ele_idx ].seq = seq_after;
225 : //ele0[ ele_idx ].line_idx = already init
226 :
227 0 : fd_vinyl_meta_publish_fast( lock, lock_shift, ele_idx );
228 :
229 0 : DONE( FD_VINYL_SUCCESS );
230 :
231 0 : }
232 :
233 : /* At this point, we are either canceling a modification (modify
234 : 0, erase d/c) or the modification is to erase the pair (modify
235 : 1, erase 1). If we are canceling the modification of an
236 : existing pair and the client indicated the cached pair info and
237 : cached pair val are still valid, (i.e. release-cancel of an
238 : acquire-for-modify of an existing pair), we revert the line
239 : state and adjust the line evict priority. (This code path can
240 : be omitted if we don't trust the clients to report correctly.
241 : We do test at least the client is correctly reporting the info
242 : is not modified.) Note that we might have put this in a larged
243 : sized obj when we acquired it for modify. So we also move the
244 : object to the tightest location. */
245 :
246 0 : if( FD_LIKELY( modifying_existing & (!req_flag_modify) & (!req_flag_ignore) ) ) {
247 :
248 : /* FIXME: consider allowing the client to always clobber the
249 : pair info and just restore info from the meta cache? */
250 :
251 0 : if( FD_UNLIKELY( !( (phdr->ctl==fd_vinyl_bstream_ctl( FD_VINYL_BSTREAM_CTL_TYPE_PAIR,
252 0 : FD_VINYL_BSTREAM_CTL_STYLE_RAW,
253 0 : (ulong)ele0[ ele_idx ].phdr.info.val_sz ) ) &
254 0 : (fd_vinyl_key_eq( &phdr->key, &ele0[ ele_idx ].phdr.key ) ) &
255 0 : (!memcmp( &phdr->info, &ele0[ ele_idx ].phdr.info, sizeof(fd_vinyl_info_t) )) ) ) )
256 0 : FD_LOG_CRIT(( "client clobbered pair info" ));
257 :
258 0 : ulong val_sz_before = (ulong)phdr->info.val_sz;
259 :
260 0 : ulong szc_after = (ulong)obj->szc;
261 0 : ulong szc_before = fd_vinyl_data_szc( val_sz_before );
262 :
263 0 : if( FD_UNLIKELY( szc_before!=szc_after ) ) {
264 :
265 0 : FD_CRIT( szc_before<szc_after, "corruption detected" );
266 :
267 0 : fd_vinyl_data_obj_t * obj_before = fd_vinyl_data_alloc( data, szc_before );
268 0 : if( FD_UNLIKELY( !obj_before ) ) FD_LOG_CRIT(( "increase data cache size" ));
269 :
270 0 : fd_vinyl_bstream_phdr_t * phdr_before = fd_vinyl_data_obj_phdr( obj_before );
271 :
272 0 : memcpy( phdr_before, phdr, sizeof(fd_vinyl_bstream_phdr_t) + val_sz_before );
273 :
274 0 : fd_vinyl_data_free( data, obj );
275 :
276 0 : line[ line_idx ].obj = obj_before; obj_before->line_idx = line_idx; obj_before->rd_active = (short)0;
277 :
278 0 : }
279 :
280 0 : line[ line_idx ].ctl = fd_vinyl_line_ctl( ver-1UL, 0L ); /* revert ver */
281 :
282 0 : fd_vinyl_line_evict_prio( &vinyl->line_idx_lru, line, line_cnt, line_idx, req_evict_prio );
283 :
284 0 : DONE( FD_VINYL_SUCCESS );
285 :
286 0 : }
287 :
288 : /* At this point, we are canceling a modification of an existing
289 : pair that no longer has valid cached pair info or cached pair
290 : val, erasing an existing pair, canceling the creation of a new
291 : pair or erasing a pair in the process of being created (which
292 : we treat the same as cancelling the creation).
293 :
294 : Since there was nothing cached originally (canceling / erasing
295 : a pair being created), the cached data is no longer valid
296 : (cancel with ignore of an existing pair) or the the cached data
297 : is no longer needed (erase of an existing pair), we free the
298 : data obj, mark the line as empty, move the line to LRU
299 : position. */
300 :
301 : /* FIXME: INTEGRITY CHECKS ON PHDR HERE? (TRICKY AS WE'D HAVE TO
302 : MAP OUT EXACTLY WHICH FIELDS CAN BE TRUSTED AT THIS POINT AND
303 : IT ISN'T OBVIOUS IT MATTERS) */
304 :
305 0 : fd_vinyl_data_free( data, obj );
306 :
307 0 : line[ line_idx ].obj = NULL;
308 0 : line[ line_idx ].ele_idx = ULONG_MAX; ele0[ ele_idx ].line_idx = ULONG_MAX;
309 0 : line[ line_idx ].ctl = fd_vinyl_line_ctl( ver+1UL, 0L ); /* bump ver */
310 :
311 0 : fd_vinyl_line_evict_prio( &vinyl->line_idx_lru, line, line_cnt, line_idx, FD_VINYL_LINE_EVICT_PRIO_LRU );
312 :
313 : /* If we are erasing an existing pair, append a dead block to
314 : the bstream. This generates two pieces of bstream garbage (the
315 : old pair and the dead block itself). Likewise, if we are
316 : erasing an existing pair or cancelling / erasing a pair
317 : creation, remove the element from the meta. Note that
318 : req_flag_modify==1 implies req_flag_erase==1 but not vice versa
319 : at this point. */
320 :
321 0 : if( FD_LIKELY( req_flag_modify & modifying_existing ) ) {
322 :
323 0 : ulong val_esz_before = fd_vinyl_bstream_ctl_sz( ele0[ ele_idx ].phdr.ctl );
324 :
325 0 : accum_garbage_cnt += 2UL;
326 0 : accum_garbage_sz += fd_vinyl_bstream_pair_sz( val_esz_before ) + FD_VINYL_BSTREAM_BLOCK_SZ;
327 :
328 0 : fd_vinyl_io_append_dead( io, &ele0[ ele_idx ].phdr, NULL, 0UL );
329 0 : append_cnt++;
330 0 : accum_dead_cnt++;
331 :
332 0 : }
333 :
334 0 : if( FD_LIKELY( req_flag_modify | (!modifying_existing) ) ) {
335 0 : fd_vinyl_meta_remove_fast( ele0, ele_max, lock, lock_shift, line, line_cnt, ele_idx );
336 :
337 0 : ulong pair_cnt = vinyl->pair_cnt;
338 0 : FD_CRIT( (0UL<pair_cnt) & (pair_cnt<=pair_max), "corruption detected" );
339 0 : vinyl->pair_cnt = pair_cnt - 1UL;
340 0 : }
341 :
342 0 : DONE( FD_VINYL_SUCCESS );
343 :
344 0 : next_release: /* silly language restriction */;
345 :
346 0 : # undef DONE
347 :
348 0 : } /* for batch_idx */
349 :
350 0 : comp_err = FD_VINYL_SUCCESS;
351 0 : break;
352 0 : }
|
