Line data    Source code

       1             : #include "fd_progcache_user.h"
       2             : #include "fd_progcache_clock.h"
       3             : #include "fd_progcache_reclaim.h"
       4             : #include "../../util/racesan/fd_racesan_target.h"
       5             : 
       6             : /* evict_inner is the eviction sweep on one size class, the shared body of
       7             :    fd_prog_evict and fd_prog_preevict.  metrics==NULL is the admin variant:
       8             :    identical scan and claim, no counters. */
       9             : 
      10             : static fd_progcache_rec_t *
      11             : evict_inner( fd_progcache_join_t *    join,
      12             :              fd_progcache_metrics_t * metrics,
      13     1347784 :              ulong                    class_idx ) {
      14     1347784 :   fd_progcache_shmem_t * shmem = join->shmem;
      15     1347784 :   fd_progcache_rec_t *   recs  = join->rec.ele;
      16             : 
      17     1347784 :   ulong base      = shmem->cache.rec_base [ class_idx ];
      18     1347784 :   ulong class_max = shmem->cache.class_max[ class_idx ];
      19     1347784 :   FD_TEST( class_max );
      20             : 
      21             :   /* Do at most 2x class_max (number of slots in class) iterations.
      22             :      If we can't evict, return NULL and we'll try to spill.  Every skip is a
      23             :      continue; the one way out with a slot is the handover at the bottom. */
      24     1347784 :   fd_progcache_rec_t * claim    = NULL;
      25     1347784 :   ulong                iter_rem = 2UL*class_max;
      26    48217953 :   while( iter_rem ) {
      27    47782077 :     iter_rem--;
      28             : 
      29             :     /* Get a candidate */
      30    47782077 :     ulong ticket = __atomic_fetch_add( &shmem->cache.clock_hand[ class_idx ].val, 1UL, __ATOMIC_RELAXED );
      31    47782077 :     fd_progcache_rec_t * rec = recs + base + ( ticket % class_max );
      32             : 
      33    47782077 :     uchar st = __atomic_load_n( &rec->state, __ATOMIC_RELAXED );
      34    47782077 :     fd_racesan_hook( "prog_clock_evict:post_load_bits" );
      35             : 
      36             :     /* Candidate is free or in flight: skip. */
      37    47782077 :     if( !( st & FD_PROGCACHE_REC_LIVE ) ) continue;
      38             : 
      39    45190979 :     if( FD_UNLIKELY( !( st & FD_PROGCACHE_REC_MAPPED ) ) ) {
      40             : 
      41             :       /* Candidate is a zombie, already unmapped, so we can try to
      42             :          write-lock it directly (skip if still read-locked / in use). */
      43             : 
      44           6 :       if( FD_UNLIKELY( atomic_load_explicit( &rec->txn_idx, memory_order_relaxed )!=UINT_MAX ) ) continue;
      45           6 :       if( FD_UNLIKELY( !fd_rwlock_trywrite( &rec->lock ) ) ) continue;
      46             :       /* The write lock freezes the record, so the re-check under it decides. */
      47           6 :       uchar cur = __atomic_load_n( &rec->state, __ATOMIC_RELAXED );
      48           6 :       if( FD_UNLIKELY( ( ( cur & ( FD_PROGCACHE_REC_LIVE|FD_PROGCACHE_REC_MAPPED ) )!=FD_PROGCACHE_REC_LIVE ) |
      49           6 :                        ( atomic_load_explicit( &rec->txn_idx, memory_order_relaxed )!=UINT_MAX ) ) ) {
      50           0 :         fd_rwlock_unwrite( &rec->lock );
      51           0 :         continue;
      52           0 :       }
      53             :       /* Write locked by the trywrite; fall through to fd_progcache_rec_reinit */
      54             : 
      55    45190973 :     } else {
      56             : 
      57             :       /* Candidate is a live cache record: run CLOCK */
      58    45190973 :       fd_progcache_rec_key_t pair = rec->pair;
      59             : 
      60             :       /* If there's any read-lock on the record, it'll fail later, so might as well
      61             :          skipping it now (this is just an optimization) */
      62    45190973 :       if( FD_UNLIKELY( atomic_load_explicit( &rec->lock.value, memory_order_relaxed )!=0 ) ) continue;
      63             : 
      64    34708876 :       if( st & FD_PROGCACHE_REC_VISITED ) {
      65             :         /* clear the VISITED bit, i.e. CLOCK second chance */
      66      912350 :         __atomic_fetch_and( &rec->state, (uchar)~FD_PROGCACHE_REC_VISITED, __ATOMIC_RELAXED );
      67      912350 :         continue;
      68      912350 :       }
      69             : 
      70             :       /* An attached record is on its fork's list: claim it under the fork lock,
      71             :          or skip it when eviction is rooted-only. */
      72    33796526 :       uint owner_idx = atomic_load_explicit( &rec->txn_idx, memory_order_relaxed );
      73    33796526 :       if( FD_UNLIKELY( owner_idx!=UINT_MAX ) ) {
      74    33796424 : #if FD_PROGCACHE_EVICT_UNROOTED
      75             :         /* Lock order matches fd_progcache_push: txn.rwlock -> txn->lock -> chain -> rec.lock. */
      76    33796424 :         if( FD_UNLIKELY( (ulong)owner_idx>=shmem->txn.max ) ) continue;
      77    33796424 :         fd_racesan_hook( "prog_clock_evict:pre_fork_lock" );
      78    33796424 :         fd_rwlock_read( &shmem->txn.rwlock );
      79             :         /* Recycling a txn slot needs the write lock, so a matching re-read here
      80             :            proves the slot still owns rec. */
      81    33796424 :         if( FD_UNLIKELY( atomic_load_explicit( &rec->txn_idx, memory_order_relaxed )!=owner_idx ) ) {
      82           0 :           fd_rwlock_unread( &shmem->txn.rwlock );
      83           0 :           continue;
      84           0 :         }
      85    33796424 :         fd_progcache_txn_t * owner = &join->txn.pool[ owner_idx ];
      86    33796424 :         if( FD_UNLIKELY( !fd_rwlock_trywrite( &owner->lock ) ) ) {
      87    32884599 :           fd_rwlock_unread( &shmem->txn.rwlock );
      88    32884599 :           continue;
      89    32884599 :         }
      90      911825 :         fd_racesan_hook( "prog_clock_evict:pre_delete" );
      91      911825 :         long res = fd_prog_delete_rec_claim_txn( join, rec, &pair, owner );
      92      911825 :         fd_racesan_hook( "prog_clock_evict:post_delete" );
      93      911825 :         fd_rwlock_unwrite( &owner->lock );
      94      911825 :         fd_rwlock_unread( &shmem->txn.rwlock );
      95      911825 :         if( FD_UNLIKELY( res<0L ) ) continue;
      96             : #else
      97             :         continue;
      98             : #endif
      99      911825 :       } else {
     100         102 :         fd_racesan_hook( "prog_clock_evict:pre_delete" );
     101         102 :         long res = fd_prog_delete_rec_claim( join, rec, &pair );
     102         102 :         fd_racesan_hook( "prog_clock_evict:post_delete" );
     103         102 :         if( FD_UNLIKELY( res<0L ) ) continue;
     104         102 :       }
     105             :       /* Write locked by the claim; fall through to fd_progcache_rec_reinit */
     106    33796526 :     }
     107             : 
     108      911908 :     if( FD_LIKELY( metrics ) ) {
     109      911893 :       metrics->evict_cnt++;
     110      911893 :       metrics->evict_tot_sz += rec->rodata_sz;
     111      911893 :       metrics->evict_per_class[ class_idx ]++;
     112      911893 :     }
     113             : 
     114             :     /* Unmapped and write locked, so nobody else can take the slot */
     115      911908 :     claim = fd_progcache_rec_reinit( join, rec );
     116      911908 :     break;
     117    45190979 :   }
     118             : 
     119     1347784 :   return claim;
     120     1347784 : }
     121             : 
     122             : fd_progcache_rec_t *
     123             : fd_prog_evict( fd_progcache_t * cache,
     124     1347769 :                ulong            sz ) {
     125     1347769 :   ulong class_idx = fd_progcache_cache_class( sz );
     126     1347769 :   FD_TEST( class_idx<FD_PROGCACHE_CACHE_CLASS_CNT );
     127     1347769 :   return evict_inner( cache->join, cache->metrics, class_idx );
     128     1347769 : }
     129             : 
     130             : ulong
     131             : fd_prog_preevict( fd_progcache_join_t *    join,
     132             :                   fd_progcache_metrics_t * metrics,
     133             :                   ulong                    class_idx,
     134          69 :                   ulong                    free_target ) {
     135          69 :   FD_TEST( class_idx<FD_PROGCACHE_CACHE_CLASS_CNT );
     136          69 :   if( FD_LIKELY( fd_progcache_class_free_cnt( join->shmem, class_idx )>=free_target ) ) return 0UL;
     137          15 :   fd_progcache_rec_t * rec = evict_inner( join, metrics, class_idx );
     138          15 :   if( FD_UNLIKELY( !rec ) ) return 0UL;
     139          15 :   fd_progcache_rec_abandon( join, rec );
     140          15 :   return 1UL;
     141          15 : }
     142             : 
     143             : void
     144             : fd_progcache_housekeeping( fd_progcache_join_t *    join,
     145          54 :                            fd_progcache_metrics_t * metrics ) {
     146          54 :   ulong ticket = __atomic_fetch_add( &join->shmem->cache.housekeep_hand.val, 1UL, __ATOMIC_RELAXED );
     147          54 :   fd_prog_preevict( join, metrics, ticket % FD_PROGCACHE_CACHE_CLASS_CNT, 2UL );
     148          54 : }