Line data    Source code

       1             : #include "fd_accdb_cache.h"
       2             : 
       3             : #include "../../util/bits/fd_bits.h"
       4             : #include "../../util/log/fd_log.h"
       5             : 
       6             : int
       7             : fd_accdb_cache_class_cnt( ulong   cache_footprint,
       8             :                           ulong   min_reserved,
       9        4419 :                           ulong * class_cnt ) {
      10             :   /* Estimated max account population per class on mainnet.  Based on a
      11             :      full mainnet snapshot (1.118B accounts, 363.7 GiB, slot 393863972)
      12             :      with ~20% headroom. */
      13             : 
      14        4419 :   static const ulong pop_max_raw[ FD_ACCDB_CACHE_CLASS_CNT ] = {
      15        4419 :      215000000UL,  /* class 0: ~16% of accounts   */
      16        4419 :     1041000000UL,  /* class 1: ~77.6% of accounts */
      17        4419 :       76000000UL,  /* class 2: ~5.6%              */
      18        4419 :        8000000UL,  /* class 3: ~0.6%              */
      19        4419 :        4000000UL,  /* class 4: ~0.3%              */
      20        4419 :         461000UL,  /* class 5: ~0.03%             */
      21        4419 :         244000UL,  /* class 6: ~0.02%             */
      22        4419 :           5000UL,  /* class 7: ~0.0003%           */
      23        4419 :   };
      24             : 
      25             :   /* Hard per-class ceiling: cidx packs only FD_ACCDB_CACHE_LINE_BITS
      26             :      bits of line index, so a class with more than
      27             :      FD_ACCDB_CACHE_LINE_MAX slots would let two distinct lines pack to
      28             :      the same cidx and silently alias on read.  Clamp pop_max[c] to that
      29             :      representable bound before any phase of the allocator consults it.  */
      30        4419 :   ulong pop_max[ FD_ACCDB_CACHE_CLASS_CNT ];
      31       39771 :   for( ulong c=0UL; c<FD_ACCDB_CACHE_CLASS_CNT; c++ ) {
      32       35352 :     pop_max[c] = fd_ulong_min( pop_max_raw[c], FD_ACCDB_CACHE_LINE_MAX );
      33       35352 :   }
      34             : 
      35             :   /* Access density weights: total_accesses / slot_size from empirical
      36             :      mainnet replay (1000-slot sample at slot 406546575), adjusted for
      37             :      64-byte header offset when mapping data_sz to stored_sz cache
      38             :      classes.  Higher weight means more cache hits per byte of cache
      39             :      spent.  Classes 6, 7 are floored to 1. */
      40             : 
      41        4419 :   static const ulong density[ FD_ACCDB_CACHE_CLASS_CNT ] = {
      42        4419 :        25UL,  /* class 0 */
      43        4419 :        20UL,  /* class 1 */
      44        4419 :        12UL,  /* class 2 */
      45        4419 :         7UL,  /* class 3 */
      46        4419 :         6UL,  /* class 4 */
      47        4419 :         3UL,  /* class 5 */
      48        4419 :         1UL,  /* class 6 */
      49        4419 :         1UL,  /* class 7 */
      50        4419 :   };
      51             : 
      52             :   /* Per-class working-set targets (slot counts).  Derived from p99 of
      53             :      the distinct-pubkey-per-class working set measured over a 32-slot
      54             :      sliding window on the same 1000-slot mainnet sample, with ~25-50%
      55             :      headroom so allocations cover the typical hot set without wasting
      56             :      budget on classes whose live working set is tiny.
      57             : 
      58             :      These are floors used by Phase 2: each class is topped up to
      59             :      ws_target[c] above the Phase 1 base reservation, before
      60             :      density-based distribution.  Phase 3 then distributes any remaining
      61             :      budget by density. */
      62             : 
      63        4419 :   static const ulong ws_target[ FD_ACCDB_CACHE_CLASS_CNT ] = {
      64        4419 :     16384UL,  /* class 0: p99 ~13.4K, ample headroom (small slots) */
      65        4419 :     13000UL,  /* class 1: p99 ~10.4K */
      66        4419 :      4096UL,  /* class 2: p99  ~3.2K */
      67        4419 :      3072UL,  /* class 3: p99  ~2.0K, was undersized at 1.3K */
      68        4419 :      1800UL,  /* class 4: p99  ~1.0K, needs headroom for pre-evict to keep up */
      69        4419 :       512UL,  /* class 5: p99    ~66, was wastefully sized at 1.3K */
      70        4419 :       704UL,  /* class 6: p99   ~212 */
      71        4419 :       544UL,  /* class 7: p99   ~179; staging covered by MIN_RESERVED */
      72        4419 :   };
      73             : 
      74        4419 :   ulong slot_sz_sum = ( fd_accdb_cache_slot_sz[ 0UL ] +
      75        4419 :                         fd_accdb_cache_slot_sz[ 1UL ] +
      76        4419 :                         fd_accdb_cache_slot_sz[ 2UL ] +
      77        4419 :                         fd_accdb_cache_slot_sz[ 3UL ] +
      78        4419 :                         fd_accdb_cache_slot_sz[ 4UL ] +
      79        4419 :                         fd_accdb_cache_slot_sz[ 5UL ] +
      80        4419 :                         fd_accdb_cache_slot_sz[ 6UL ] +
      81        4419 :                         fd_accdb_cache_slot_sz[ 7UL ] );
      82             : 
      83             :   /* min_reserved is a runtime parameter; guard the Phase-1 cost
      84             :      multiplication against overflow before it wraps and makes the
      85             :      "budget too small" check below misbehave (a wrapped small value
      86             :      would pass the check and yield bogus class counts). */
      87        4419 :   if( FD_UNLIKELY( min_reserved>ULONG_MAX/slot_sz_sum ) ) {
      88           0 :     FD_LOG_WARNING(( "cache_min_reserved %lu too large (overflows minimum cost)", min_reserved ));
      89           0 :     for( ulong c=0UL; c<FD_ACCDB_CACHE_CLASS_CNT; c++ ) class_cnt[ c ] = 0UL;
      90           0 :     return 0;
      91           0 :   }
      92             : 
      93        4419 :   ulong minimum_cost = min_reserved * slot_sz_sum;
      94             : 
      95        4419 :   if( FD_UNLIKELY( cache_footprint<minimum_cost ) ) {
      96             :     /* Budget too small to meet minimum requirement.  Return 0 to
      97             :        indicate failure. */
      98          15 :     FD_LOG_WARNING(( "cache_footprint must be at least %lu GiB to meet minimum requirements", (minimum_cost+(1UL<<30UL)-1)/(1UL<<30UL) ));
      99          15 :     FD_LOG_WARNING(( "%lu<%lu", cache_footprint, minimum_cost ));
     100         135 :     for( ulong c=0UL; c<FD_ACCDB_CACHE_CLASS_CNT; c++ ) class_cnt[ c ] = 0UL;
     101          15 :     return 0;
     102          15 :   }
     103             : 
     104             :   /* Phase 1: Reserve min_reserved of each class off the top.  This
     105             :      guarantees the worst-case batch (64 accounts per transaction,
     106             :      doubled to cover programdata, multiplied by max simultaneous
     107             :      transactions) can execute fully in memory.  Each referenced account
     108             :      reserves one slot in its own class plus one slot for its
     109             :      programdata account, which may land in any class.  Worst case all
     110             :      referenced accounts and all programdata accounts land in the same
     111             :      class. */
     112             : 
     113        4404 :   ulong remaining = cache_footprint;
     114       39636 :   for( ulong c=0UL; c<FD_ACCDB_CACHE_CLASS_CNT; c++ ) {
     115       35232 :     class_cnt[c] = min_reserved;
     116       35232 :     remaining   -= min_reserved * fd_accdb_cache_slot_sz[c];
     117       35232 :   }
     118             : 
     119             :   /* Phase 2: Reserve up to ws_target[c] slots per class as a floor.
     120             :      Phase 1 already gave each class min_reserved slots; here we top up
     121             :      to ws_target[c] (or as much as remaining budget allows).  This
     122             :      keeps tiny working sets (128K/1M/10M classes) from being
     123             :      over-allocated and frees budget for hotter classes (8K, 2K) in
     124             :      Phase 3. */
     125             : 
     126       39636 :   for( ulong c=0UL; c<FD_ACCDB_CACHE_CLASS_CNT; c++ ) {
     127       35232 :     if( ws_target[c]<=class_cnt[c] ) continue;
     128       35139 :     ulong want = ws_target[c] - class_cnt[c];
     129       35139 :     want = fd_ulong_min( want, pop_max[c]>class_cnt[c] ? pop_max[c]-class_cnt[c] : 0UL );
     130       35139 :     ulong cost = want * fd_accdb_cache_slot_sz[c];
     131       35139 :     if( FD_UNLIKELY( cost>remaining ) ) {
     132        5739 :       class_cnt[c] += remaining / fd_accdb_cache_slot_sz[c];
     133        5739 :       remaining     = 0UL;
     134       29400 :     } else {
     135       29400 :       class_cnt[c] += want;
     136       29400 :       remaining    -= cost;
     137       29400 :     }
     138       35139 :   }
     139             : 
     140             :   /* Phase 3: Iteratively allocate remaining budget proportional
     141             :      to access density.  When a class exceeds its population cap,
     142             :      freeze it and redistribute surplus to uncapped classes. */
     143             : 
     144        4404 :   int capped[ FD_ACCDB_CACHE_CLASS_CNT ];
     145       39636 :   for( ulong c=0UL; c<FD_ACCDB_CACHE_CLASS_CNT; c++ ) capped[c] = 0;
     146             : 
     147        4446 :   for( ulong iter=0UL; iter<FD_ACCDB_CACHE_CLASS_CNT && remaining; iter++ ) {
     148          75 :     ulong total_w = 0UL;
     149         675 :     for( ulong c=0UL; c<FD_ACCDB_CACHE_CLASS_CNT; c++ )
     150         600 :       if( !capped[c] ) total_w += density[c];
     151          75 :     if( FD_UNLIKELY( !total_w ) ) break;
     152             : 
     153          66 :     int any_capped = 0;
     154         594 :     for( ulong c=0UL; c<FD_ACCDB_CACHE_CLASS_CNT; c++ ) {
     155         528 :       if( capped[c] ) continue;
     156         387 :       ulong budget = remaining * density[c] / total_w;
     157         387 :       ulong extra  = budget / fd_accdb_cache_slot_sz[c];
     158         387 :       if( class_cnt[c]+extra >= pop_max[c] ) {
     159          81 :         ulong added  = pop_max[c] - class_cnt[c];
     160          81 :         class_cnt[c] = pop_max[c];
     161          81 :         remaining   -= added * fd_accdb_cache_slot_sz[c];
     162          81 :         capped[c]    = 1;
     163          81 :         any_capped   = 1;
     164          81 :       }
     165         387 :     }
     166             : 
     167          66 :     if( !any_capped ) {
     168             :       /* No caps hit.  Final proportional allocation. */
     169          24 :       total_w = 0UL;
     170         216 :       for( ulong c=0UL; c<FD_ACCDB_CACHE_CLASS_CNT; c++ )
     171         192 :         if( !capped[c] ) total_w += density[c];
     172          24 :       if( FD_UNLIKELY( !total_w ) ) break;
     173         216 :       for( ulong c=0UL; c<FD_ACCDB_CACHE_CLASS_CNT; c++ ) {
     174         192 :         if( capped[c] ) continue;
     175         183 :         ulong budget = remaining * density[c] / total_w;
     176         183 :         class_cnt[c] += budget / fd_accdb_cache_slot_sz[c];
     177         183 :       }
     178          24 :       break;
     179          24 :     }
     180          66 :   }
     181             : 
     182             :   /* Phase 4: If all classes hit their population caps, there may
     183             :      still be remaining budget.  The accounts database can grow at
     184             :      runtime, so distribute excess uncapped, proportional to
     185             :      density.  This ensures we always use the full cache budget
     186             :      the operator gave us, up to the per-class cidx ceiling
     187             :      (FD_ACCDB_CACHE_LINE_MAX, enforced via the capped[] array). */
     188             : 
     189        4404 :   remaining = cache_footprint;
     190       39636 :   for( ulong c=0UL; c<FD_ACCDB_CACHE_CLASS_CNT; c++ )
     191       35232 :     remaining -= class_cnt[c] * fd_accdb_cache_slot_sz[c];
     192             : 
     193       39636 :   for( ulong c=0UL; c<FD_ACCDB_CACHE_CLASS_CNT; c++ ) capped[c] = class_cnt[c]>=FD_ACCDB_CACHE_LINE_MAX;
     194             : 
     195        4410 :   for( ulong iter=0UL; iter<FD_ACCDB_CACHE_CLASS_CNT && remaining; iter++ ) {
     196        4245 :     ulong total_w = 0UL;
     197       38205 :     for( ulong c=0UL; c<FD_ACCDB_CACHE_CLASS_CNT; c++ ) {
     198       33960 :       if( !capped[c] ) total_w += density[c];
     199       33960 :     }
     200        4245 :     if( FD_UNLIKELY( !total_w ) ) break;
     201             : 
     202        4245 :     int any_capped = 0;
     203       38205 :     for( ulong c=0UL; c<FD_ACCDB_CACHE_CLASS_CNT; c++ ) {
     204       33960 :       if( capped[c] ) continue;
     205       33939 :       ulong budget = remaining * density[c] / total_w;
     206       33939 :       ulong extra  = budget / fd_accdb_cache_slot_sz[c];
     207       33939 :       if( class_cnt[c]+extra >= FD_ACCDB_CACHE_LINE_MAX ) {
     208           6 :         ulong added  = FD_ACCDB_CACHE_LINE_MAX - class_cnt[c];
     209           6 :         class_cnt[c] = FD_ACCDB_CACHE_LINE_MAX;
     210           6 :         remaining   -= added * fd_accdb_cache_slot_sz[c];
     211           6 :         capped[c]    = 1;
     212           6 :         any_capped   = 1;
     213           6 :       }
     214       33939 :     }
     215             : 
     216        4245 :     if( !any_capped ) {
     217        4239 :       total_w = 0UL;
     218       38151 :       for( ulong c=0UL; c<FD_ACCDB_CACHE_CLASS_CNT; c++ ) {
     219       33912 :         if( !capped[c] ) total_w += density[c];
     220       33912 :       }
     221        4239 :       if( FD_UNLIKELY( !total_w ) ) break;
     222       38151 :       for( ulong c=0UL; c<FD_ACCDB_CACHE_CLASS_CNT; c++ ) {
     223       33912 :         if( capped[c] ) continue;
     224       33897 :         ulong budget = remaining * density[c] / total_w;
     225       33897 :         ulong extra  = budget / fd_accdb_cache_slot_sz[c];
     226       33897 :         extra        = fd_ulong_min( extra, FD_ACCDB_CACHE_LINE_MAX - class_cnt[c] );
     227       33897 :         class_cnt[c] += extra;
     228       33897 :         remaining    -= extra * fd_accdb_cache_slot_sz[c];
     229       33897 :       }
     230        4239 :       break;
     231        4239 :     }
     232        4245 :   }
     233             : 
     234             :   /* Spend any per-class integer-division truncation slack left after
     235             :      Phase 4.  Walk uncapped classes in descending density order and
     236             :      add as many slots as fit, capped at FD_ACCDB_CACHE_LINE_MAX.  This
     237             :      converges in at most one pass per class because each iteration
     238             :      either fills the class to LINE_MAX or drains remaining below
     239             :      slot_sz[c]. */
     240       38316 :   for( ulong c=0UL; c<FD_ACCDB_CACHE_CLASS_CNT && remaining; c++ ) {
     241       33912 :     if( class_cnt[c]>=FD_ACCDB_CACHE_LINE_MAX ) continue;
     242       33897 :     ulong room  = FD_ACCDB_CACHE_LINE_MAX - class_cnt[c];
     243       33897 :     ulong extra = fd_ulong_min( room, remaining / fd_accdb_cache_slot_sz[c] );
     244       33897 :     class_cnt[c] += extra;
     245       33897 :     remaining    -= extra * fd_accdb_cache_slot_sz[c];
     246       33897 :   }
     247             : 
     248             :   /* Past FD_ACCDB_CACHE_LINE_MAX*slot_sz[c] (~6 TiB) the index space is
     249             :      fully saturated and any further budget is unspendable on cache
     250             :      lines.  Warn the operator so they can size down. */
     251        4404 :   ulong unspent = cache_footprint;
     252       39636 :   for( ulong c=0UL; c<FD_ACCDB_CACHE_CLASS_CNT; c++ )
     253       35232 :     unspent -= class_cnt[c] * fd_accdb_cache_slot_sz[c];
     254        4404 :   if( FD_UNLIKELY( unspent >= fd_accdb_cache_slot_sz[ 0UL ] ) ) {
     255           6 :     FD_LOG_WARNING(( "cache_footprint exceeds per-class index space; %lu GiB will be unused (raise FD_ACCDB_CACHE_LINE_BITS to use more)",
     256           6 :                      (unspent+(1UL<<30UL)-1UL)/(1UL<<30UL) ));
     257           6 :   }
     258             : 
     259        4404 :   return 1;
     260        4419 : }
     261             : 
     262             : ulong
     263      515082 : fd_accdb_cache_class( ulong data_sz ) {
     264      515082 :   if( FD_LIKELY( data_sz<=128UL ) ) return 0UL;
     265      203523 :   else if( FD_LIKELY( data_sz<=512UL ) ) return 1UL;
     266      187248 :   else if( FD_LIKELY( data_sz<=2048UL ) ) return 2UL;
     267      186948 :   else if( FD_LIKELY( data_sz<=8192UL ) ) return 3UL;
     268      136329 :   else if( FD_LIKELY( data_sz<=32768UL ) ) return 4UL;
     269       35823 :   else if( FD_LIKELY( data_sz<=131072UL ) ) return 5UL;
     270       35205 :   else if( FD_LIKELY( data_sz<=1048576UL ) ) return 6UL;
     271        1626 :   return 7UL;
     272      515082 : }