Line data    Source code

       1             : #ifndef HEADER_fd_src_flamenco_progcache_fd_progcache_rec_h
       2             : #define HEADER_fd_src_flamenco_progcache_fd_progcache_rec_h
       3             : 
       4             : #include "fd_progcache_base.h"
       5             : #include "../../ballet/sbpf/fd_sbpf_loader.h"
       6             : #include "../fd_flamenco_base.h"
       7             : #include "../fd_rwlock.h"
       8             : 
       9             : #include <stdatomic.h>
      10             : 
      11             : struct fd_progcache_rec_key {
      12             :   fd_progcache_fork_id_t xid;
      13             :   fd_pubkey_t            prog;
      14             : };
      15             : 
      16             : typedef struct fd_progcache_rec_key fd_progcache_rec_key_t;
      17             : 
      18             : static inline int
      19             : fd_progcache_rec_key_eq( fd_progcache_rec_key_t const * k0,
      20     2661849 :                          fd_progcache_rec_key_t const * k1 ) {
      21     2661849 :   return ( (k0->xid == k1->xid) & fd_pubkey_eq( &k0->prog, &k1->prog ) );
      22     2661849 : }
      23             : 
      24             : /* fd_progcache_rec_t is a program cache entry.  Entries are either
      25             :    executable or non-executable (e.g. programs that failed verification).
      26             :    An executable entry's program data (rodata/ROM segment, control flow
      27             :    metadata, ...) lives in its size class's arena slot, addressed by
      28             :    data_gaddr; a non-executable entry has data_gaddr==0. */
      29             : 
      30             : /* The first 64 bytes hold everything a lookup touches -- key, revision, chain
      31             :    link, lock and CLOCK state -- so walking a chain and locking the winner costs
      32             :    one cache line.  The rest is only read once the record is in hand. */
      33             : 
      34             : struct __attribute__((aligned(64))) fd_progcache_rec {
      35             :   fd_progcache_rec_key_t pair;  /* Transaction id and record key pair */
      36             : 
      37             :   ulong feature_slot;
      38             :   ulong deploy_slot;
      39             : 
      40             :   uint        map_next;  /* Internal use by map */
      41             :   fd_rwlock_t lock;
      42             : 
      43             :   /* CLOCK/liveness bits, atomically accessed (see fd_progcache_clock.h).  Its own
      44             :      byte: an atomic RMW cannot target a bitfield, and widening one to the bitfield
      45             :      below would race with the plain writes to those fields. */
      46             :   uchar       state;
      47             : 
      48             :   atomic_uint txn_idx;
      49             :   uint        next_idx;  /* Record map index of next record in its transaction */
      50             :   uint        prev_idx;  /* Record map index of previous record in its transaction */
      51             :   uint        data_max;  /* size of allocation */
      52             : 
      53             :   ulong data_gaddr;  /* wksp-base relative pointer to data */
      54             : 
      55             :   uint entry_pc;
      56             :   uint text_cnt;
      57             :   uint text_off;
      58             :   uint text_sz;
      59             : 
      60             :   uint rodata_sz;
      61             : 
      62             :   uint calldests_off;  /* offset to sbpf_calldests map */
      63             :   uint rodata_off;     /* offset to rodata segment */
      64             : 
      65             :   ushort      sbpf_version : 8; /* SBPF version, SIMD-0161 */
      66             :   ushort      exists       : 1; /* 0 once released to the free list or as a closed spill frame */
      67             :   ushort      size_class   : 3; /* the class the record's slot belongs to, set at acquire */
      68             : 
      69             :   uint free_next; /* next record in the class's free list; a stale popper may read it concurrently */
      70             : };
      71             : 
      72             : FD_STATIC_ASSERT( sizeof(fd_progcache_rec_t)==128, layout );
      73             : 
      74             : FD_PROTOTYPES_BEGIN
      75             : 
      76             : /* Accessors */
      77             : 
      78             : static inline uchar const *
      79             : fd_progcache_rec_rodata( fd_progcache_rec_t const * rec,
      80        2469 :                          fd_wksp_t *                wksp ) {
      81        2469 :   return fd_wksp_laddr_fast( wksp, rec->data_gaddr + rec->rodata_off );
      82        2469 : }
      83             : 
      84             : static inline fd_sbpf_calldests_t const *
      85             : fd_progcache_rec_calldests( fd_progcache_rec_t const * rec,
      86        2472 :                             fd_wksp_t *                wksp ) {
      87        2472 :   if( rec->calldests_off==UINT_MAX ) return NULL;
      88        2469 :   return fd_sbpf_calldests_join( fd_wksp_laddr_fast( wksp, rec->data_gaddr + rec->calldests_off ) );
      89        2472 : }
      90             : 
      91             : /* Record + value slot management.  Records are partitioned by size class
      92             :    (see fd_progcache.h): acquiring a record from a class IS acquiring its
      93             :    value slot. */
      94             : 
      95             : /* fd_progcache_val_{align,footprint} give the alignment and size the program
      96             :    data of an executable cache entry needs, which is what picks its size class.
      97             :    elf_info must describe a successfully peeked ELF.  A non-executable entry
      98             :    leaves its slot unused; fd_progcache_rec_nx marks the record with
      99             :    data_gaddr==0. */
     100             : 
     101             : FD_FN_CONST static inline ulong
     102     5528160 : fd_progcache_val_align( void ) {
     103     5528160 :   return fd_sbpf_calldests_align();
     104     5528160 : }
     105             : 
     106             : FD_FN_PURE ulong
     107             : fd_progcache_val_footprint( fd_sbpf_elf_info_t const * elf_info );
     108             : 
     109             : /* fd_progcache_rec_acquire pops a free record from the class fitting
     110             :    val_footprint and initializes it as an in-flight record: read-locked by
     111             :    the caller, not in the map, its arena slot attached.  Returns NULL if the class
     112             :    is full, leaving the caller to evict within the class (fd_prog_evict hands
     113             :    back a record of that class) or spill.  No cross-class borrowing. */
     114             : 
     115             : fd_progcache_rec_t *
     116             : fd_progcache_rec_acquire( fd_progcache_join_t * join,
     117             :                           ulong                 val_footprint );
     118             : 
     119             : /* fd_progcache_rec_reinit turns a record the caller holds write-locked and
     120             :    out of the map into an in-flight record, exactly as acquire does, without
     121             :    passing through the class free list.  The eviction sweep uses it to hand a
     122             :    slot it just claimed straight to the requester, so the slot cannot be taken
     123             :    in between. */
     124             : 
     125             : fd_progcache_rec_t *
     126             : fd_progcache_rec_reinit( fd_progcache_join_t * join,
     127             :                          fd_progcache_rec_t *  rec );
     128             : 
     129             : /* fd_progcache_rec_release returns a record to its class free list and
     130             :    releases its value storage.  The caller must hold the record's WRITE
     131             :    lock (guaranteeing no other user) and the record must not be in the
     132             :    map.  The record stays write-locked on the free list, so a stale
     133             :    speculative reader can never lock a free record. */
     134             : 
     135             : void
     136             : fd_progcache_rec_release( fd_progcache_join_t * join,
     137             :                           fd_progcache_rec_t *  rec );
     138             : 
     139             : /* fd_progcache_rec_abandon releases an in-flight record that was never
     140             :    published to the map: a peer owns the key, at this revision or another.  The
     141             :    caller holds the read lock from fd_progcache_rec_acquire; abandon
     142             :    upgrades it to a write lock (draining transient speculative readers of
     143             :    a previous incarnation of this record) and releases. */
     144             : 
     145             : void
     146             : fd_progcache_rec_abandon( fd_progcache_join_t * join,
     147             :                           fd_progcache_rec_t *  rec );
     148             : 
     149             : fd_progcache_rec_t *
     150             : fd_progcache_rec_load( fd_progcache_rec_t *            rec,
     151             :                        fd_wksp_t *                     wksp,
     152             :                        fd_sbpf_elf_info_t const *      elf_info,
     153             :                        fd_sbpf_loader_config_t const * config,
     154             :                        ulong                           load_slot,
     155             :                        fd_features_t const *           features,
     156             :                        void const *                    progdata,
     157             :                        ulong                           progdata_sz,
     158             :                        void *                          scratch,
     159             :                        ulong                           scratch_sz );
     160             : 
     161             : fd_progcache_rec_t *
     162             : fd_progcache_rec_nx( fd_progcache_rec_t * rec );
     163             : 
     164             : FD_PROTOTYPES_END
     165             : 
     166             : #endif /* HEADER_fd_src_flamenco_progcache_fd_progcache_rec_h */