Line data    Source code

       1             : #include "fd_progcache_reclaim.h"
       2             : #include "fd_progcache_clock.h"
       3             : #include "fd_progcache_user.h"
       4             : #include "../../util/racesan/fd_racesan_target.h"
       5             : 
       6             : void
       7             : fd_prog_reclaim_enqueue( fd_progcache_join_t * join,
       8          51 :                          fd_progcache_rec_t *  rec ) {
       9          51 :   uint idx = (uint)( rec - join->rec.pool->ele );
      10          51 :   ulong rec_max = fd_prog_recp_ele_max( join->rec.pool );
      11          51 :   if( FD_UNLIKELY( (ulong)idx >= rec_max ) )
      12           0 :     FD_LOG_CRIT(( "progcache: corruption detected (reclaim_enqueue rec_idx=%u rec_max=%lu)", idx, rec_max ));
      13          51 :   rec->reclaim_next = join->rec.reclaim_head;
      14          51 :   join->rec.reclaim_head = idx;
      15          51 : }
      16             : 
      17             : static _Bool
      18             : rec_reclaim( fd_progcache_join_t * join,
      19          57 :              fd_progcache_rec_t *  rec ) {
      20             : 
      21             :   /* Remove the record from a transaction */
      22             : 
      23          57 :   ulong txn_max = fd_prog_txnp_max( join->txn.pool );
      24          57 :   uint txn_idx = atomic_load_explicit( &rec->txn_idx, memory_order_acquire );
      25          57 :   if( txn_idx!=UINT_MAX ) {
      26           3 :     if( FD_UNLIKELY( (ulong)txn_idx >= txn_max ) )
      27           0 :       FD_LOG_CRIT(( "progcache: corruption detected (rec_reclaim txn_idx=%u txn_max=%lu)", txn_idx, txn_max ));
      28           3 :     fd_progcache_txn_t * txn = &join->txn.pool[ txn_idx ];
      29           3 :     fd_rwlock_write( &txn->lock );
      30           3 :     fd_racesan_hook( "prog_reclaim:pre_cas" );
      31           3 :     if( atomic_compare_exchange_strong_explicit( &rec->txn_idx, &txn_idx, UINT_MAX, memory_order_acq_rel, memory_order_acquire ) ) {
      32             :       /* A transaction may not be deallocated before all records are
      33             :          unlinked. */
      34           3 :       fd_progcache_rec_unlink( join->rec.pool->ele, rec, txn, join->rec.pool->ele_max );
      35           3 :     } else {
      36             :       /* Strong CAS failure implies that another thread is already
      37             :          unlinking the record (the rooting logic) */
      38           0 :       FD_CRIT( atomic_load_explicit( &rec->txn_idx, memory_order_relaxed )==UINT_MAX, "concurrency violation" );
      39           0 :     }
      40           3 :     fd_rwlock_unwrite( &txn->lock );
      41           3 :   }
      42          57 :   fd_racesan_hook( "prog_reclaim:post_unlink" );
      43             : 
      44             :   /* Drain existing users
      45             : 
      46             :      Records are removed from recm (index) before the record is selected
      47             :      for reclamation.  Therefore, it is not necessary to acquire a lock.
      48             :      It is fine to wait for existing users to drain. */
      49             : 
      50          57 :   if( FD_UNLIKELY( FD_VOLATILE_CONST( rec->lock.value ) ) ) return 0;
      51             : 
      52             :   /* All users are gone, deallocate record */
      53             : 
      54          51 :   rec->reclaim_next = UINT_MAX;
      55          51 :   fd_progcache_val_free( rec, join );
      56          51 :   rec->exists = 0;
      57          51 :   fd_prog_clock_remove( join->clock.bits, (ulong)( rec - join->rec.pool->ele ) );
      58          51 :   fd_prog_recp_release( join->rec.pool, rec, 1 );
      59          51 :   return 1;
      60          57 : }
      61             : 
      62             : ulong
      63         198 : fd_prog_reclaim_work( fd_progcache_join_t * join ) {
      64         198 :   ulong  rec_max = fd_prog_recp_ele_max( join->rec.pool );
      65         198 :   ulong  cnt     = 0UL;
      66         198 :   uint * prev_p  = &join->rec.reclaim_head;
      67         198 :   uint   cur     = join->rec.reclaim_head;
      68         255 :   while( cur!=UINT_MAX ) {
      69          57 :     if( FD_UNLIKELY( (ulong)cur >= rec_max ) )
      70           0 :       FD_LOG_CRIT(( "progcache: corruption detected (reclaim_work rec_idx=%u rec_max=%lu)", cur, rec_max ));
      71          57 :     fd_progcache_rec_t * rec = &join->rec.pool->ele[ cur ];
      72          57 :     uint next = rec->reclaim_next;
      73          57 :     if( rec_reclaim( join, rec ) ) {
      74          51 :       *prev_p = next;
      75          51 :       cnt++;
      76          51 :     } else {
      77           6 :       prev_p = &rec->reclaim_next;
      78           6 :     }
      79          57 :     cur = next;
      80          57 :   }
      81         198 :   return cnt;
      82         198 : }
      83             : 
      84             : long
      85             : fd_prog_delete_rec( fd_progcache_join_t * cache,
      86          48 :                     fd_progcache_rec_t *  rec ) {
      87          48 :   if( !rec ) return -1L;
      88          48 :   fd_prog_recm_query_t query[1];
      89          48 :   int rm_err = fd_prog_recm_remove( cache->rec.map, &rec->pair, NULL, query, FD_MAP_FLAG_BLOCKING );
      90          48 :   if( rm_err==FD_MAP_ERR_KEY ) return -1L;
      91          45 :   if( FD_UNLIKELY( rm_err!=FD_MAP_SUCCESS ) ) FD_LOG_CRIT(( "fd_prog_recm_remove failed: %i-%s", rm_err, fd_map_strerror( rm_err ) ));
      92          45 :   if( FD_UNLIKELY( query->ele!=rec ) ) FD_LOG_CRIT(( "record collision (rec=%p found=%p)", (void *)rec, (void *)query->ele ));
      93          45 :   fd_prog_reclaim_enqueue( cache, rec );
      94          45 :   return (long)rec->data_max;
      95          45 : }
      96             : 
      97             : long
      98             : fd_prog_delete_rec_by_key( fd_progcache_join_t *          cache,
      99             :                            fd_funk_xid_key_pair_t const * key,
     100           9 :                            _Bool                          lock ) {
     101           9 :   fd_prog_recm_query_t query[1];
     102           9 :   int rm_err;
     103           9 :   if( lock ) rm_err = fd_prog_recm_remove    ( cache->rec.map, key, NULL, query, FD_MAP_FLAG_BLOCKING );
     104           9 :   else       rm_err = fd_prog_recm_txn_remove( cache->rec.map, key, NULL, query, 0                    );
     105           9 :   if( rm_err==FD_MAP_ERR_KEY ) return -1L;
     106           6 :   if( FD_UNLIKELY( rm_err!=FD_MAP_SUCCESS ) ) FD_LOG_CRIT(( "fd_prog_recm_remove failed: %i-%s", rm_err, fd_map_strerror( rm_err ) ));
     107           6 :   fd_prog_reclaim_enqueue( cache, query->ele );
     108           6 :   return (long)query->ele->data_max;
     109           6 : }