Line data    Source code

       1             : #ifndef HEADER_fd_src_flamenco_accdb_fd_accdb_private_h
       2             : #define HEADER_fd_src_flamenco_accdb_fd_accdb_private_h
       3             : 
       4             : #include "fd_accdb_shmem.h"
       5             : #include "fd_accdb_cache.h"
       6             : 
       7             : /* Maximum accounts a single acquire may request.
       8             :    FD_ACCDB_MAX_TX_ACCOUNT_LOCKS mirrors the mainnet per-transaction
       9             :    account-lock limit (Agave get_transaction_account_lock_limit == 64;
      10             :    the increase_tx_account_lock_limit feature -> 128 is assumed never to
      11             :    activate).  FD_ACCDB_MAX_TXN_PER_ACQUIRE mirrors
      12             :    FD_PACK_MAX_TXN_PER_BUNDLE, a bundle coalesces up to that many
      13             :    transactions into one acquire. */
      14      272376 : #define FD_ACCDB_MAX_TX_ACCOUNT_LOCKS (64UL)
      15           0 : #define FD_ACCDB_MAX_TXN_PER_ACQUIRE  (5UL)
      16           0 : #define FD_ACCDB_MAX_ACQUIRE_CNT      (FD_ACCDB_MAX_TXN_PER_ACQUIRE*FD_ACCDB_MAX_TX_ACCOUNT_LOCKS)
      17             : 
      18             : static inline void
      19          42 : spin_lock_acquire( int * lock ) {
      20          42 : # if FD_HAS_THREADS
      21          42 :   for(;;) {
      22          42 :     if( FD_LIKELY( !FD_ATOMIC_CAS( lock, 0, 1 ) ) ) break;
      23           0 :     FD_SPIN_PAUSE();
      24           0 :   }
      25             : # else
      26             :   *lock = 1;
      27             : # endif
      28          42 :   FD_COMPILER_MFENCE();
      29          42 : }
      30             : 
      31             : static inline void
      32          42 : spin_lock_release( int * lock ) {
      33          42 :   FD_COMPILER_MFENCE();
      34          42 : # if FD_HAS_THREADS
      35          42 :   FD_VOLATILE( *lock ) = 0;
      36             : # else
      37             :   *lock = 0;
      38             : # endif
      39          42 : }
      40             : 
      41             : #ifndef FD_ACCDB_NO_FORK_ID
      42             : struct fd_accdb_fork_id { ushort val; };
      43             : typedef struct fd_accdb_fork_id fd_accdb_fork_id_t;
      44             : #endif
      45             : 
      46             : struct __attribute__((packed)) fd_accdb_disk_meta {
      47             :   uchar pubkey[ 32UL ];
      48             :   uint  size;
      49             :   uchar owner[ 32UL ];
      50             : };
      51             : 
      52             : typedef struct fd_accdb_disk_meta fd_accdb_disk_meta_t;
      53             : 
      54             : struct fd_accdb_txn {
      55             :   union {
      56             :     struct { uint next; } pool;
      57             :     struct { uint next; } fork;
      58             :   };
      59             : 
      60             :   uint acc_map_idx;
      61             :   uint acc_pool_idx;
      62             : };
      63             : 
      64             : typedef struct fd_accdb_txn fd_accdb_txn_t;
      65             : 
      66             : #define POOL_NAME       txn_pool
      67      421245 : #define POOL_ELE_T      fd_accdb_txn_t
      68        2376 : #define POOL_NEXT       pool.next
      69             : #define POOL_IDX_T      uint
      70      970035 : #define POOL_IDX_WIDTH  32
      71             : #define POOL_IMPL_STYLE 0
      72             : #define POOL_LAZY       1
      73             : 
      74             : #include "../../util/tmpl/fd_pool_para.c"
      75             : 
      76             : #define SET_NAME       descends_set
      77             : #define SET_IMPL_STYLE 1
      78             : #include "../../util/tmpl/fd_set_dynamic.c"
      79             : 
      80             : struct fd_accdb_fork_shmem {
      81             :   uint generation;
      82             : 
      83             :   fd_accdb_fork_id_t parent_id;
      84             :   fd_accdb_fork_id_t child_id;
      85             :   fd_accdb_fork_id_t sibling_id;
      86             : 
      87             :   struct {
      88             :     ulong next;
      89             :   } pool;
      90             : 
      91             :   uint txn_head;
      92             : };
      93             : 
      94             : typedef struct fd_accdb_fork_shmem fd_accdb_fork_shmem_t;
      95             : 
      96             : #define POOL_NAME       fork_pool
      97       92958 : #define POOL_ELE_T      fd_accdb_fork_shmem_t
      98       73320 : #define POOL_NEXT       pool.next
      99             : #define POOL_IDX_T      ulong
     100             : #define POOL_IMPL_STYLE 0
     101             : 
     102             : #include "../../util/tmpl/fd_pool_para.c"
     103             : 
     104             : struct fd_accdb_partition {
     105             :   ulong marked_compaction;
     106             :   ulong write_offset;
     107             :   ulong compaction_offset;
     108             : 
     109             :   ulong bytes_freed;
     110             : 
     111             :   uchar layer; /* compaction tier this partition belongs to. */
     112             : 
     113             :   ulong read_ops;
     114             :   ulong bytes_read;
     115             :   ulong write_ops;
     116             :   ulong bytes_written;
     117             : 
     118             :   /* Tickcount (fd_tickcount) of the partition's lifecycle events.  Set
     119             :      only at partition creation and again when the partition closes
     120             :      (i.e. the layer's write head rotates off it). */
     121             :   long created_ticks;
     122             :   long filled_ticks;
     123             : 
     124             :   /* Compaction lifecycle flags.  queued is set when the partition is
     125             :      pushed onto the compaction_dlist and cleared when it is popped.
     126             :      compacting_now is set by the compaction tile around the actual
     127             :      compaction work for this partition. */
     128             :   uchar queued;
     129             :   uchar compacting_now;
     130             : 
     131             :   /* Per-compaction-pass telemetry accumulators for the
     132             :      accdb_compaction_completed event.  Reset when a partition's
     133             :      compaction begins (compacting_now set in background_compact) and
     134             :      accumulated as records are scanned/relocated. */
     135             :   long  compaction_start_wallclock;    /* timestamp when this pass began */
     136             :   ulong compaction_accounts_relocated; /* live records moved this pass */
     137             :   ulong compaction_bytes_relocated;    /* bytes moved this pass */
     138             :   ulong compaction_dead_records;       /* records skipped (no live index entry) this pass */
     139             : 
     140             :   /* Epoch at which this partition was enqueued for compaction.  Set by
     141             :      fd_accdb_shmem_bytes_freed when the partition crosses the
     142             :      freed-bytes threshold.  The compaction tile will not begin reading
     143             :      from this partition until all joiners that were in an
     144             :      epoch-protected critical section at enqueue time have exited,
     145             :      ensuring any in-flight pwritev2 to this partition has completed. */
     146             :   ulong compaction_ready_epoch;
     147             : 
     148             :   /* Epoch at which this partition was enqueued for deferred freeing.
     149             :      Set by compaction when the partition finishes compaction, and
     150             :      checked by the reclamation scan to determine when it is safe to
     151             :      release the partition back to the pool. */
     152             :   ulong epoch_tag;
     153             : 
     154             :   ulong pool_next;
     155             : 
     156             :   ulong dlist_prev;
     157             :   ulong dlist_next;
     158             : };
     159             : 
     160             : typedef struct fd_accdb_partition fd_accdb_partition_t;
     161             : 
     162             : #define POOL_NAME       partition_pool
     163             : #define POOL_T          fd_accdb_partition_t
     164          42 : #define POOL_NEXT       pool_next
     165             : #define POOL_IDX_T      ulong
     166             : #define POOL_IMPL_STYLE 1
     167             : 
     168             : #include "../../util/tmpl/fd_pool.c"
     169             : 
     170             : #define DLIST_NAME       compaction_dlist
     171             : #define DLIST_ELE_T      fd_accdb_partition_t
     172          24 : #define DLIST_PREV       dlist_prev
     173          24 : #define DLIST_NEXT       dlist_next
     174             : #define DLIST_IMPL_STYLE 1
     175             : 
     176             : #include "../../util/tmpl/fd_dlist.c"
     177             : 
     178             : /* deferred_free_dlist reuses the same prev/next fields as
     179             :    compaction_dlist.  A partition is in at most one of the two lists at
     180             :    any time: it is popped from compaction_dlist before being pushed onto
     181             :    deferred_free_dlist. */
     182             : 
     183             : #define DLIST_NAME       deferred_free_dlist
     184             : #define DLIST_ELE_T      fd_accdb_partition_t
     185           0 : #define DLIST_PREV       dlist_prev
     186           0 : #define DLIST_NEXT       dlist_next
     187             : #define DLIST_IMPL_STYLE 1
     188             : 
     189             : #include "../../util/tmpl/fd_dlist.c"
     190             : 
     191             : struct fd_accdb_cache_key {
     192             :   uchar pubkey[ 32UL ];
     193             :   uint generation;
     194             : };
     195             : 
     196             : typedef struct fd_accdb_cache_key fd_accdb_cache_key_t;
     197             : 
     198             : struct __attribute__((aligned(64))) fd_accdb_accmeta {
     199             :   fd_accdb_cache_key_t key;
     200             : 
     201             :   struct {
     202             :     uint next;
     203             :   } map;
     204             : 
     205             :   union {
     206             :     struct {
     207             :       uint next;
     208             :     } pool;
     209             :     uint cache_idx;
     210             :   };
     211             : 
     212             :   uint   executable_size;
     213             : 
     214             :   ulong  lamports;
     215             : 
     216             :   /* Pack offset and fork_id together into a single ulong to pack the
     217             :      struct into a single 64 byte cache line.  This is a performance win
     218             :      of 2-3%. */
     219             :   ulong  offset_fork;
     220             : };
     221             : 
     222             : typedef struct fd_accdb_accmeta fd_accdb_accmeta_t;
     223             : 
     224             : FD_STATIC_ASSERT( alignof(fd_accdb_accmeta_t)==64, layout );
     225             : FD_STATIC_ASSERT( sizeof (fd_accdb_accmeta_t)==64, layout );
     226             : 
     227      452193 : #define FD_ACCDB_OFF_BITS  48UL
     228      332550 : #define FD_ACCDB_OFF_MASK  ((1UL<<FD_ACCDB_OFF_BITS)-1UL)       /* 0x0000_FFFF_FFFF_FFFF */
     229      213921 : #define FD_ACCDB_OFF_INVAL FD_ACCDB_OFF_MASK                    /* sentinel: offset bits all-ones */
     230             : 
     231             : /* The `size` field in fd_accdb_disk_meta_t (named executable_size in
     232             :    fd_accdb_accmeta_t) packs five things into 32 bits:
     233             : 
     234             :      bit  31     executable flag                       (FD_ACCDB_SIZE_EXEC_BIT)
     235             :      bit  30     cache_valid flag, in-memory only      (FD_ACCDB_SIZE_CACHE_VALID_BIT)
     236             :      bit  29     cache_claim flag, in-memory only      (FD_ACCDB_SIZE_CACHE_CLAIM_BIT)
     237             :      bit  28     pd_write flag,    in-memory only      (FD_ACCDB_SIZE_PD_WRITE_BIT)
     238             :      bits 27..0  data length in bytes                  (FD_ACCDB_SIZE_MASK)
     239             : 
     240             :    The data length is therefore 28 bits, max 256 MiB, still well above
     241             :    FD_RUNTIME_ACC_SZ_MAX of 10 MiB (enforced by the static assert below).
     242             : 
     243             :    The three upper flag bits exist only in the in-memory index, never on
     244             :    disk:
     245             :      - cache_valid (bit 30): when set, cache_idx holds a valid
     246             :        (class, idx) pair; when clear, cache_idx must not be dereferenced
     247             :        (it may hold a snapshot slot number or garbage).
     248             :      - cache_claim (bit 29): a short-lived eviction/install lock taken
     249             :        via CAS while a writer mutates the cache_idx <-> cache-line
     250             :        binding, so concurrent readers and the evictor do not race.
     251             :      - pd_write (bit 28): set on a committed version whose write changed
     252             :        BPF upgradeable-loader deploy status this slot (Deploy/Upgrade/
     253             :        Extend/Close).  Meaningful only while the accmeta's key.generation
     254             :        equals a live fork's generation (i.e. the version was committed on
     255             :        that fork this slot); across snapshot/root boundaries the bit is
     256             :        dead by construction because the generation no longer matches any
     257             :        live fork.  Carried explicitly by the two commit sites in
     258             :        fd_accdb_release and nowhere else.
     259             : 
     260             :    The on-disk representation (written via SIZE_PACK / SIZE_DATA) carries
     261             :    no in-memory flag: persisted bytes are unchanged, and compaction's
     262             :    copy_file_range preserves the record headers verbatim without
     263             :    rewriting them. */
     264             : 
     265      136137 : #define FD_ACCDB_SIZE_EXEC_BIT        (1U<<31)
     266        1869 : #define FD_ACCDB_SIZE_CACHE_VALID_BIT (1U<<30)
     267        5010 : #define FD_ACCDB_SIZE_CACHE_CLAIM_BIT (1U<<29)
     268        2886 : #define FD_ACCDB_SIZE_PD_WRITE_BIT    (1U<<28)
     269      281745 : #define FD_ACCDB_SIZE_MASK            ((1U<<28)-1U)
     270      106095 : #define FD_ACCDB_SIZE_PACK(sz,exec)   ((uint)(sz) | ((exec) ? FD_ACCDB_SIZE_EXEC_BIT : 0U))
     271      281745 : #define FD_ACCDB_SIZE_DATA(packed)    ((packed) & FD_ACCDB_SIZE_MASK)
     272       88929 : #define FD_ACCDB_SIZE_EXEC(packed)    (!!((packed) & FD_ACCDB_SIZE_EXEC_BIT))
     273        1845 : #define FD_ACCDB_SIZE_CACHE_VALID(p)  (!!((p) & FD_ACCDB_SIZE_CACHE_VALID_BIT))
     274          24 : #define FD_ACCDB_SIZE_CACHE_CLAIM(p)  (!!((p) & FD_ACCDB_SIZE_CACHE_CLAIM_BIT))
     275         120 : #define FD_ACCDB_SIZE_PD_WRITE(p)     (!!((p) & FD_ACCDB_SIZE_PD_WRITE_BIT))
     276             : 
     277             : FD_STATIC_ASSERT( (10UL<<20) < (1UL<<28), pd_write_bit_collides_with_len );
     278             : 
     279             : static inline ulong
     280        1491 : fd_accdb_acc_offset( fd_accdb_accmeta_t const * acc ) {
     281        1491 :   return acc->offset_fork & FD_ACCDB_OFF_MASK;
     282        1491 : }
     283             : 
     284             : static inline ushort
     285       16374 : fd_accdb_acc_fork_id( fd_accdb_accmeta_t const * acc ) {
     286       16374 :   return (ushort)( acc->offset_fork >> FD_ACCDB_OFF_BITS );
     287       16374 : }
     288             : 
     289             : static inline ulong
     290             : fd_accdb_acc_pack_offset_fork( ulong  offset,
     291      103269 :                                ushort fork_id ) {
     292      103269 :   return ( (ulong)fork_id << FD_ACCDB_OFF_BITS ) | ( offset & FD_ACCDB_OFF_MASK );
     293      103269 : }
     294             : 
     295             : /* fd_accdb_acc_xchg_offset atomically replaces the 48-bit offset
     296             :    portion of acc->offset_fork with new_offset while preserving the
     297             :    16-bit fork_id, and returns the previous 48-bit offset.  Uses a
     298             :    CAS loop so that concurrent compaction CAS and release-overwrite
     299             :    exchanges serialize correctly. */
     300             : 
     301             : static inline ulong
     302             : fd_accdb_acc_xchg_offset( fd_accdb_accmeta_t * acc,
     303        4623 :                            ulong               new_offset ) {
     304        4623 :   for(;;) {
     305        4623 :     ulong old_packed = FD_VOLATILE_CONST( acc->offset_fork );
     306        4623 :     ulong new_packed = ( old_packed & ~FD_ACCDB_OFF_MASK ) | ( new_offset & FD_ACCDB_OFF_MASK );
     307        4623 :     if( FD_LIKELY( FD_ATOMIC_CAS( &acc->offset_fork, old_packed, new_packed )==old_packed ) )
     308        4623 :       return old_packed & FD_ACCDB_OFF_MASK;
     309           0 :     FD_SPIN_PAUSE();
     310           0 :   }
     311        4623 : }
     312             : 
     313             : /* Packing helpers for the embedded acc cache index.  3 bits class
     314             :    in bits 31-29, 29 bits line index.  INVAL is the sentinel for
     315             :    "no cached location known".  FD_ACCDB_CACHE_LINE_MAX (defined in
     316             :    fd_accdb_cache.h) is the exclusive upper bound on representable
     317             :    line indices; per-class slot counts must not exceed it, or cidx
     318             :    values would alias. */
     319             : 
     320      197127 : #define FD_ACCDB_ACC_CIDX_IDX_MASK  ((uint)(FD_ACCDB_CACHE_LINE_MAX-1UL))
     321        3960 : #define FD_ACCDB_ACC_CIDX_INVAL     UINT_MAX
     322      106413 : #define FD_ACCDB_ACC_CIDX_PACK(c,i) ((uint)( ((uint)(c)<<FD_ACCDB_CACHE_LINE_BITS) | ((uint)(i) & FD_ACCDB_ACC_CIDX_IDX_MASK) ))
     323       92514 : #define FD_ACCDB_ACC_CIDX_CLASS(ci) ((ulong)((uint)(ci) >> FD_ACCDB_CACHE_LINE_BITS))
     324       90714 : #define FD_ACCDB_ACC_CIDX_IDX(ci)   ((ulong)((uint)(ci) & FD_ACCDB_ACC_CIDX_IDX_MASK))
     325             : 
     326             : #define POOL_NAME       acc_pool
     327      418272 : #define POOL_ELE_T      fd_accdb_accmeta_t
     328        1815 : #define POOL_NEXT       pool.next
     329             : #define POOL_IDX_T      uint
     330      970638 : #define POOL_IDX_WIDTH  32
     331             : #define POOL_IMPL_STYLE 0
     332             : #define POOL_LAZY       1
     333             : 
     334             : #include "../../util/tmpl/fd_pool_para.c"
     335             : 
     336             : struct fd_accdb_cache_line {
     337             :   fd_accdb_cache_key_t key;
     338             : 
     339             :   uint acc_idx;
     340             :   uint cache_idx;
     341             : 
     342             :   uint  refcnt;
     343             :   uchar persisted;
     344             :   uchar referenced;
     345             : 
     346             :   uint next;
     347             : 
     348             :   uchar owner[ 32UL ];
     349             : };
     350             : 
     351             : typedef struct fd_accdb_cache_line fd_accdb_cache_line_t;
     352             : 
     353             : typedef struct __attribute__((aligned(64))) { ulong val; } accdb_offset_t;
     354             : 
     355             : /* Partition offsets are packed into accdb_offset_t as:
     356             :      bits 63..51: partition pool index
     357             :      bits 50..0 : byte offset within the partition */
     358             : 
     359       11862 : #define FD_ACCDB_PARTITION_OFF_BITS 51UL
     360             : 
     361             : static FD_FN_CONST inline accdb_offset_t
     362             : accdb_offset( ulong partition_idx,
     363       11556 :               ulong partition_offset ) {
     364       11556 :   return (accdb_offset_t){ .val = (partition_idx<<FD_ACCDB_PARTITION_OFF_BITS) | partition_offset };
     365       11556 : }
     366             : 
     367             : static FD_FN_PURE inline ulong
     368         132 : packed_partition_idx( accdb_offset_t const * offset ) {
     369         132 :   return offset->val>>FD_ACCDB_PARTITION_OFF_BITS;
     370         132 : }
     371             : 
     372             : static FD_FN_PURE inline ulong
     373         174 : packed_partition_offset( accdb_offset_t const * offset ) {
     374         174 :   return offset->val & ((1UL<<FD_ACCDB_PARTITION_OFF_BITS)-1UL);
     375         174 : }
     376             : 
     377             : static FD_FN_PURE inline ulong
     378             : packed_partition_file_offset( accdb_offset_t const * offset,
     379          45 :                               ulong                  partition_sz ) {
     380          45 :    return (packed_partition_idx( offset )*partition_sz + packed_partition_offset( offset ));
     381          45 : }
     382             : 
     383             : /* Accounts are written to a tiered partition layout.  Layer 0 is the
     384             :    hot write head used by acquire/release (execution).  Layers 1..N-1
     385             :    are successively colder compaction tiers: partitions at layer K are
     386             :    compacted into layer K+1. */
     387             : 
     388     4158721 : #define FD_ACCDB_COMPACTION_LAYER_CNT (3UL)
     389             : 
     390             : /* Maximum number of concurrent joiners (tiles) that can publish an
     391             :    epoch in the accdb.  Each joiner claims a slot in the shared epoch
     392             :    array during fd_accdb_new.  Must be less than or equal to 256 so
     393             :    refcnt in cache lines can safely track the number of threads
     394             :    referencing each cache line without overflow.  With a uint refcnt
     395             :    field, 256 joiners is well within range. */
     396      986880 : #define FD_ACCDB_MAX_JOINERS (256UL)
     397             : 
     398             : /* EVICT_SENTINEL: stored in refcnt to indicate a cache line is being
     399             :    claimed by an eviction scan.  Any thread seeing this value must treat
     400             :    the line as unavailable. */
     401             : #define FD_ACCDB_EVICT_SENTINEL UINT_MAX
     402             : 
     403             : struct fd_accdb_shmem_private {
     404             :   int partition_lock  __attribute__((aligned(64)));
     405             : 
     406             :   /* Set non-zero by the snapin tile while a snapshot is being loaded.
     407             :      Suppresses compaction enqueue so the compaction tile does not race
     408             :      with bulk snapshot writes; fd_accdb_snapshot_load_end performs a
     409             :      one-shot sweep to enqueue any partitions that crossed the
     410             :      fragmentation threshold during the load. */
     411             :   int snapshot_loading;
     412             : 
     413             :   /* Set at construction (fd_accdb_shmem_new) when this validator
     414             :      supports bundles.  A bundle coalesces up to
     415             :      FD_ACCDB_MAX_TXN_PER_ACQUIRE transactions into one acquire, so when
     416             :      set, fd_accdb_acquire_inner permits pubkeys_cnt up to
     417             :      FD_ACCDB_MAX_ACQUIRE_CNT instead of the single-transaction limit
     418             :      FD_ACCDB_MAX_TX_ACCOUNT_LOCKS. */
     419             :   int bundle_enabled;
     420             : 
     421             :   /* Per-class CLOCK sweep position.  Atomically incremented by
     422             :      eviction scans (modulo cache_class_max[c]).  Each element is on
     423             :      its own cacheline to avoid false sharing between classes. */
     424             :   struct __attribute__((aligned(64))) { ulong val; } clock_hand[ FD_ACCDB_CACHE_CLASS_CNT ];
     425             : 
     426             :   /* Per-class CAS free list (Treiber stack) for fully-freed cache
     427             :      lines.  ver_top packs a 32-bit ABA version counter in bits
     428             :      63..32 and a uint pool index in bits 31..0.  UINT_MAX in the
     429             :      low 32 bits means empty. */
     430             :   struct __attribute__((aligned(64))) { ulong ver_top; } cache_free[ FD_ACCDB_CACHE_CLASS_CNT ];
     431             : 
     432             :   /* Per-class approximate depth of the CAS free list.  Atomically
     433             :      incremented on push, decremented on pop.  Used by the
     434             :      background pre-eviction loop to decide when to refill. */
     435             :   struct __attribute__((aligned(64))) { ulong val; } cache_free_cnt[ FD_ACCDB_CACHE_CLASS_CNT ];
     436             : 
     437             :   fd_accdb_fork_id_t root_fork_id;
     438             : 
     439             :   ulong seed;
     440             : 
     441             :   /* generation is a monotonically increasing counter assigned to each
     442             :      fork on creation.  When a fork is rooted, its pool slot (fork_id)
     443             :      is freed and may be recycled by a new fork, making fork_id in
     444             :      on-disk metadata useless for identifying entries from that freed
     445             :      fork.  But generation persists in disk metadata and is never
     446             :      recycled.
     447             : 
     448             :      Any rooted fork is by definition an ancestor of all live forks, so
     449             :      entries with generation <= root_fork->generation are
     450             :      unconditionally visible without consulting descends_set.  For
     451             :      entries with generation > root_fork->generation, the fork_id is
     452             :      still valid and descends_set is used to check ancestry.
     453             : 
     454             :      KEY INVARIANT: descends_set is ONLY consulted when generation >
     455             :      root_generation, which means the fork_id has NOT been rooted yet
     456             :      and its pool slot is still live.  This is what makes it safe for
     457             :      fork_slot_defer to eagerly clear descends_set bits for retired
     458             :      forks: rooted fork bits are dead (bypassed by the generation fast
     459             :      path), and purged fork bits were already 0 in all live forks'
     460             :      descends_sets (a purged fork is never an ancestor of a live fork).
     461             :      */
     462             :   uint generation;
     463             : 
     464             :   /* Lazy initial-allocation counter per size class.  Atomically
     465             :      incremented by acquire_cache_line (with undo on overflow). Each
     466             :      element is on its own cacheline to avoid false sharing between
     467             :      classes. */
     468             :   struct __attribute__((aligned(64))) { ulong val; } cache_class_init[ FD_ACCDB_CACHE_CLASS_CNT ];
     469             : 
     470             :   ulong cache_class_max[ FD_ACCDB_CACHE_CLASS_CNT ];
     471             : 
     472             :   /* Byte offsets from shmem base to the per-class cache regions. Each
     473             :      region is cache_class_max[c] * fd_accdb_cache_slot_sz[c] bytes,
     474             :      with each slot holding an fd_accdb_cache_line_t header followed by
     475             :      up to (fd_accdb_cache_slot_sz[c] - META_SZ) bytes of account data.
     476             :      */
     477             :   ulong cache_region_off[ FD_ACCDB_CACHE_CLASS_CNT ];
     478             : 
     479             :   /* Background pre-eviction watermarks (computed once in shmem_new).
     480             :      cache_free_target[c]: desired free-list depth for class c.
     481             :      cache_free_low_water[c]: trigger threshold ((target*3)/4). */
     482             :   ulong cache_free_target   [ FD_ACCDB_CACHE_CLASS_CNT ];
     483             :   ulong cache_free_low_water[ FD_ACCDB_CACHE_CLASS_CNT ];
     484             : 
     485             :   /* cache_class_used[i].val holds the number of reserved cache
     486             :      slots in size class i.  Acquire atomically increments; if the
     487             :      result exceeds cache_class_max[i] the reservation overflowed
     488             :      and the thread subtracts back and retries.  Release atomically
     489             :      decrements.  Each element is on its own cacheline to avoid
     490             :      false sharing between classes.  Invariant:
     491             :        used[i].val + available[i] == cache_class_max[i]
     492             :      at all times. */
     493             :   struct __attribute__((aligned(64))) { ulong val; } cache_class_used[ FD_ACCDB_CACHE_CLASS_CNT ];
     494             : 
     495             :   /* Per-layer write heads.  whead[0] is the hot (execution) write
     496             :      head, updated with atomic fetch-and-add by acquire/release
     497             :      threads.  whead[1..N-1] are compaction write heads, each
     498             :      single-writer (compaction tile only). */
     499             :   accdb_offset_t whead[ FD_ACCDB_COMPACTION_LAYER_CNT ];
     500             :   int            has_partition[ FD_ACCDB_COMPACTION_LAYER_CNT ];
     501             : 
     502             :   ulong partition_cnt;
     503             :   ulong partition_sz;
     504             :   ulong partition_max;
     505             : 
     506             :   ulong chain_cnt;
     507             :   ulong max_live_slots;
     508             :   ulong max_accounts;
     509             :   ulong max_account_writes_per_slot;
     510             : 
     511             :   /* Hard upper bound on concurrent joiners, set at construction.
     512             :      Used to determine whether cache_class_used tracking can be
     513             :      skipped for a given class (when max[c] >= MIN_RESERVED *
     514             :      joiner_cnt, every reservation succeeds trivially). */
     515             :   ulong joiner_cnt_max;
     516             : 
     517             :   /* Per-joiner worst-case cache reservation, set at construction.
     518             :      Used by fd_accdb_reset to replicate the cache_class_used sentinel
     519             :      logic from fd_accdb_shmem_new. */
     520             :   ulong cache_min_reserved;
     521             : 
     522             :   ulong partition_pool_off;
     523             : 
     524             :   /* compaction_dlist_off[k] is the byte offset (from shmem base) of
     525             :      the dlist sentinel for layer k.  Partitions at layer k that
     526             :      reach the freed-bytes threshold are enqueued here for compaction
     527             :      into layer k+1, or into layer k itself for the deepest layer. */
     528             :   ulong compaction_dlist_off[ FD_ACCDB_COMPACTION_LAYER_CNT ];
     529             : 
     530             :   /* Epoch-based safe reclamation for compacted partitions.
     531             : 
     532             :      epoch is a monotonically increasing counter incremented by the
     533             :      compaction tile each time a partition finishes compaction.  The
     534             :      completed partition is tagged with the current epoch and pushed
     535             :      onto a deferred-free list instead of being released immediately.
     536             : 
     537             :      joiner_epochs[i] holds the epoch observed by joiner i at the start
     538             :      of its epoch-protected critical section, or ULONG_MAX when idle.
     539             :      The compaction tile scans this array to find the minimum observed
     540             :      epoch; any deferred partition tagged with an epoch strictly less
     541             :      than that minimum is safe to release, because every epoch-protected
     542             :      operation that could have snapshotted an offset into that partition
     543             :      has since exited its critical section.
     544             : 
     545             :      joiner_cnt is claimed via atomic fetch-and-add in fd_accdb_new
     546             :      and never decremented. */
     547             :   ulong epoch __attribute__((aligned(64)));
     548             : 
     549             :   /* Each joiner epoch is padded to a full cache line to prevent
     550             :      false sharing between joiners writing to adjacent slots. */
     551             :   struct __attribute__((aligned(64))) { ulong val; } joiner_epochs[ FD_ACCDB_MAX_JOINERS ];
     552             :   ulong joiner_cnt __attribute__((aligned(64)));
     553             :   ulong deferred_free_dlist_off;
     554             : 
     555             :   fd_accdb_shmem_metrics_t shmetrics[1];
     556             : 
     557             :   /* Command slot for T1 -> T2 offloading of advance_root / purge.
     558             :      Padded to its own cache line to avoid false sharing with the
     559             :      hot epoch / joiner_epochs fields above. */
     560             : 
     561     5270989 : #define FD_ACCDB_CMD_IDLE            (0U)
     562        1200 : #define FD_ACCDB_CMD_ADVANCE_ROOT    (1U)
     563          30 : #define FD_ACCDB_CMD_PURGE           (2U)
     564           6 : #define FD_ACCDB_CMD_CLEAR_DEFERRED  (3U)
     565           6 : #define FD_ACCDB_CMD_DRAIN_DEFERRED  (4U)
     566             : 
     567             :   uint   cmd_op       __attribute__((aligned(64))); /* FD_ACCDB_CMD_* */
     568             :   ushort cmd_fork_id;                               /* argument       */
     569             : 
     570             :   /* T2-only side buffer for deferred acc unlinks.  Holds the indices of
     571             :      accs that have been CAS-unlinked from their map chains in the
     572             :      current advance_root or purge call but cannot yet have pool.next
     573             :      written: a concurrent cold_load_acc may stomp it via the cache_idx
     574             :      union alias.  After wait_for_epoch_drain the list is materialized
     575             :      into pool.next links and released via acc_pool_release_chain.
     576             : 
     577             :      T2 is the sole writer (advance_root and purge both run on T2 via
     578             :      the cmd offload above), so cnt is plain.  Capacity is
     579             :      txn_max = max_live_slots * max_account_writes_per_slot, the same
     580             :      bound used to size txn_pool.  Stored as a byte offset from shmem
     581             :      base. */
     582             : 
     583             :   ulong deferred_acc_buf_off;
     584             :   ulong deferred_acc_buf_cnt;
     585             :   ulong deferred_acc_buf_max;
     586             :   ulong deferred_acc_epoch;
     587             : 
     588             :   acc_pool_shmem_t  acc_pool [1];
     589             :   fork_pool_shmem_t fork_pool[1];
     590             :   txn_pool_shmem_t  txn_pool [1];
     591             : 
     592             :   ulong magic; /* ==FD_ACCDB_SHMEM_MAGIC */
     593             : };
     594             : 
     595             : #endif /* HEADER_fd_src_flamenco_accdb_fd_accdb_private_h */