Line data    Source code

       1             : #include "fd_progcache_reclaim.h"
       2             : #include "fd_progcache_clock.h"
       3             : #include "fd_progcache_user.h"
       4             : #include "../../util/racesan/fd_racesan_target.h"
       5             : 
       6             : 
       7             : /* Shared body of the delete pair.  claim additionally takes the record's write
       8             :    lock inside the chain transaction, so the caller owns the slot outright instead
       9             :    of leaving a zombie.  The lock is taken here, not before, because the order is
      10             :    chain then rec.lock: fd_progcache_push holds a chain and then blocks on a
      11             :    winner's read lock, so the other order deadlocks. */
      12             : 
      13             : static long
      14             : delete_rec_inner( fd_progcache_join_t *          cache,
      15             :                   fd_progcache_rec_t *           rec,
      16             :                   fd_progcache_rec_key_t const * _pair,
      17             :                   int                            claim,
      18      914192 :                   fd_progcache_txn_t *           owner ) {
      19      914192 :   if( !rec ) return -1L;
      20             : 
      21      914192 :   fd_progcache_rec_key_t pair = *_pair;
      22             : 
      23             :   /* Prepare index removal, and bail if rec is not mapped under pair */
      24      914192 :   struct {
      25      914192 :     fd_prog_recm_txn_t txn[1];
      26      914192 :     fd_prog_recm_txn_private_info_t info[1];
      27      914192 :   } _map_txn;
      28      914192 :   fd_prog_recm_txn_t * map_txn = fd_prog_recm_txn_init( _map_txn.txn, cache->rec.map, 1UL );
      29      914192 :   fd_prog_recm_txn_add( map_txn, &pair, 1 );
      30      914192 :   fd_racesan_hook( "prog_delete_rec:post_txn_add" );
      31      914192 :   int txn_err = fd_prog_recm_txn_try( map_txn, FD_MAP_FLAG_BLOCKING );
      32      914192 :   if( FD_UNLIKELY( txn_err!=FD_MAP_SUCCESS ) )
      33           0 :     FD_LOG_CRIT(( "fd_prog_recm_txn_try failed: %i-%s", txn_err, fd_map_strerror( txn_err ) ));
      34      914192 :   fd_prog_recm_query_t query[1];
      35      914192 :   int q_err = fd_prog_recm_txn_query( cache->rec.map, &pair, NULL, query, 0 );
      36             :   /* query->ele!=rec rejects a slot recycled under a different key. */
      37      914192 :   int ok = !( q_err==FD_MAP_ERR_KEY || query->ele!=rec );
      38             : 
      39      914192 :   if( ok && claim ) {
      40      911902 :     ok = !!fd_rwlock_trywrite( &rec->lock );
      41      911902 :     fd_racesan_hook( "prog_delete_rec:post_claim" );
      42      911902 :     if( ok ) {
      43             :       /* The write lock freezes the record, so these reads decide: a rooted claim
      44             :          needs it detached, an owner claim needs it still owner's (xid rejects a
      45             :          recycled txn slot). */
      46      911902 :       uchar st   = __atomic_load_n( &rec->state, __ATOMIC_RELAXED );
      47      911902 :       uint  want = owner ? (uint)( owner - cache->txn.pool ) : UINT_MAX;
      48      911902 :       ok = !!( st & FD_PROGCACHE_REC_LIVE ) & !( st & FD_PROGCACHE_REC_VISITED )
      49      911902 :          & ( atomic_load_explicit( &rec->txn_idx, memory_order_relaxed )==want );
      50      911902 :       if( owner ) ok &= ( owner->xid==rec->pair.xid );
      51      911902 :       if( FD_UNLIKELY( !ok ) ) fd_rwlock_unwrite( &rec->lock );
      52      911902 :     }
      53      911902 :   }
      54             : 
      55      914192 :   if( FD_UNLIKELY( !ok ) ) {
      56          34 :     fd_prog_recm_txn_test( map_txn );
      57          34 :     fd_prog_recm_txn_fini( map_txn );
      58          34 :     return -1L;
      59          34 :   }
      60             : 
      61             :   /* Splice an attached victim out of owner's list (owner->lock held by the
      62             :      caller), only now that the claim is certain. */
      63      914158 :   if( owner ) {
      64      911800 :     ulong rec_max = cache->rec.max;
      65      911800 :     uint  p = rec->prev_idx;
      66      911800 :     uint  n = rec->next_idx;
      67      911800 :     if( n!=UINT_MAX ) {
      68      911794 :       if( FD_UNLIKELY( (ulong)n>=rec_max ) ) FD_LOG_CRIT(( "progcache: corruption detected (evict unlink next_idx=%u rec_max=%lu)", n, rec_max ));
      69      911794 :       cache->rec.ele[ n ].prev_idx = p;
      70      911794 :     } else owner->rec_tail_idx = p;
      71      911800 :     if( p!=UINT_MAX ) {
      72      902853 :       if( FD_UNLIKELY( (ulong)p>=rec_max ) ) FD_LOG_CRIT(( "progcache: corruption detected (evict unlink prev_idx=%u rec_max=%lu)", p, rec_max ));
      73      902853 :       cache->rec.ele[ p ].next_idx = n;
      74      902853 :     } else owner->rec_head_idx = n;
      75      911800 :     atomic_store_explicit( &rec->txn_idx, UINT_MAX, memory_order_release );
      76      911800 :   }
      77             : 
      78             :   /* Read while still mapped under the chain: once MAPPED clears, an unclaimed
      79             :      zombie can be reinitialized by a sweep at any time. */
      80      914158 :   long rodata_sz = (long)rec->rodata_sz;
      81             : 
      82             :   /* Drop record */
      83      914158 :   int rm_err = fd_prog_recm_txn_remove( cache->rec.map, &pair, NULL, query, 0 );
      84      914158 :   if( FD_UNLIKELY( rm_err!=FD_MAP_SUCCESS ) )
      85           0 :     FD_LOG_CRIT(( "fd_prog_recm_txn_remove failed: %i-%s", rm_err, fd_map_strerror( rm_err ) ));
      86             :   /* Cleared inside the map txn that removed it, so it is serialized against any
      87             :      publish or lookup of this key. */
      88      914158 :   __atomic_fetch_and( &rec->state, (uchar)~FD_PROGCACHE_REC_MAPPED, __ATOMIC_RELAXED );
      89      914158 :   fd_racesan_hook( "prog_delete_rec:post_unmap" );
      90      914158 :   int test_err = fd_prog_recm_txn_test( map_txn );
      91      914158 :   if( FD_UNLIKELY( test_err!=FD_MAP_SUCCESS ) )
      92           0 :     FD_LOG_CRIT(( "fd_prog_recm_txn_test failed: %i-%s", test_err, fd_map_strerror( test_err ) ));
      93      914158 :   fd_prog_recm_txn_fini( map_txn );
      94             : 
      95             :   /* A claimed record is the caller's, write locked.  Otherwise it is now a
      96             :      zombie: unmapped, still LIVE, reused by a later sweep once it is detached
      97             :      from its fork and unheld. */
      98      914158 :   return rodata_sz;
      99      914158 : }
     100             : 
     101             : long
     102             : fd_prog_delete_rec( fd_progcache_join_t * cache,
     103        2262 :                     fd_progcache_rec_t *  rec ) {
     104        2262 :   if( !rec ) return -1L;
     105        2262 :   fd_progcache_rec_key_t pair = rec->pair;
     106        2262 :   return delete_rec_inner( cache, rec, &pair, 0, NULL );
     107        2262 : }
     108             : 
     109             : long
     110             : fd_prog_delete_rec_claim( fd_progcache_join_t *          cache,
     111             :                           fd_progcache_rec_t *           rec,
     112         105 :                           fd_progcache_rec_key_t const * pair ) {
     113         105 :   return delete_rec_inner( cache, rec, pair, 1, NULL );
     114         105 : }
     115             : 
     116             : long
     117             : fd_prog_delete_rec_claim_txn( fd_progcache_join_t *          cache,
     118             :                               fd_progcache_rec_t *           rec,
     119             :                               fd_progcache_rec_key_t const * pair,
     120      911825 :                               fd_progcache_txn_t *           owner ) {
     121      911825 :   return delete_rec_inner( cache, rec, pair, 1, owner );
     122      911825 : }
     123             : 
     124             : ulong
     125        4443 : fd_prog_reclaim_work( fd_progcache_join_t * join ) {
     126        4443 :   ulong cnt = 0UL;
     127      792180 :   for( ulong i=0UL; i<join->rec.max; i++ ) {
     128      787737 :     fd_progcache_rec_t * rec = join->rec.ele + i;
     129             : 
     130             :     /* Cheap pre-filter: only a zombie (LIVE, unmapped), detached and unheld,
     131             :        is a candidate. */
     132      787737 :     uchar st = __atomic_load_n( &rec->state, __ATOMIC_RELAXED );
     133      787737 :     if( FD_LIKELY( ( st & ( FD_PROGCACHE_REC_LIVE|FD_PROGCACHE_REC_MAPPED ) )!=FD_PROGCACHE_REC_LIVE ) ) continue;
     134        2265 :     if( FD_UNLIKELY( atomic_load_explicit( &rec->txn_idx,    memory_order_relaxed )!=UINT_MAX ) ) continue;
     135        2259 :     if( FD_UNLIKELY( atomic_load_explicit( &rec->lock.value, memory_order_relaxed )!=0        ) ) continue;
     136             : 
     137             :     /* The write lock freezes the record, so the re-check under it decides.  It
     138             :        is held only across the check and never while waiting, so nothing can
     139             :        deadlock against it. */
     140        2250 :     if( FD_UNLIKELY( !fd_rwlock_trywrite( &rec->lock ) ) ) continue;
     141        2250 :     uchar cur = __atomic_load_n( &rec->state, __ATOMIC_RELAXED );
     142        2250 :     if( FD_UNLIKELY( ( ( cur & ( FD_PROGCACHE_REC_LIVE|FD_PROGCACHE_REC_MAPPED ) )!=FD_PROGCACHE_REC_LIVE ) |
     143        2250 :                      ( atomic_load_explicit( &rec->txn_idx, memory_order_relaxed )!=UINT_MAX ) ) ) {
     144           0 :       fd_rwlock_unwrite( &rec->lock );
     145           0 :       continue;
     146           0 :     }
     147             :     /* Not unlocked: the write lock becomes the free-list state. */
     148        2250 :     fd_progcache_rec_release( join, rec );
     149        2250 :     cnt++;
     150        2250 :   }
     151        4443 :   return cnt;
     152        4443 : }