Line data    Source code

       1             : #ifndef HEADER_fd_src_discof_txsend_fd_txsend_tile_h
       2             : #define HEADER_fd_src_discof_txsend_fd_txsend_tile_h
       3             : 
       4             : /* Sender tile signs and sends transactions to the current leader.
       5             :    Currently only supports transactions which require one signature.
       6             :    Designed with voting as primary use case. Signing those votes will
       7             :    eventually move to a separate consensus tile.*/
       8             : #define _GNU_SOURCE
       9             : 
      10             : #include "../../util/net/fd_net_headers.h"
      11             : #include "../../disco/stem/fd_stem.h"
      12             : #include "../../disco/fd_disco.h"
      13             : #include "../../disco/net/fd_net_tile.h"
      14             : #include "../../disco/keyguard/fd_keyguard_client.h"
      15             : #include "../../flamenco/leaders/fd_multi_epoch_leaders.h"
      16             : #include "../../waltz/quic/fd_quic.h"
      17             : #include "../../util/clock/fd_clock.h"
      18             : 
      19             : /* Send votes to next FD_SEND_TARGET_LEADER_CNT leaders (slot x, x+4, x+8, ...) */
      20           0 : #define FD_TXSEND_TARGET_LEADER_CNT (3UL)
      21             : 
      22             : /* Connect FD_CONNECT_AHEAD_LEADER_CNT leaders ahead (slot x, x+4, x+8, ...) */
      23           0 : #define FD_TXSEND_CONNECT_AHEAD_LEADER_CNT  (7UL)
      24             : 
      25             : /* Agave currently rate limits connections per minute per IP */
      26             : #define FD_AGAVE_MAX_CONNS_PER_MINUTE (8UL)
      27             : 
      28             : /* so each of our connections must survive at least 60/8 = 7.5 seconds
      29             :    Let's conservatively go to 10 */
      30           0 : #define FD_TXSEND_QUIC_MIN_CONN_LIFETIME_SECONDS (10L)
      31             : 
      32             : /* Wait FD_TXSEND_QUIC_VOTE_MIN_CONN_COOLDOWN_SECONDS many seconds before
      33             :    re-establishing a conn to a quic_vote port. Why?
      34             :    After timing out, the agave server puts the conn in a draining state, during
      35             :    which time it remains in their connection map. So our new attempt gets
      36             :    rejected, as the quic_vote port limits to 1 conn per client. Without cooldown,
      37             :    we keep trying rapidly, and can quickly hit their 8 conn/min limit.
      38             :    That prevents us from connecting for far longer than just cooling down
      39             :    (which should be free because we connect ahead). This number is based
      40             :    on empirical observation, but has much room for improvement. */
      41           0 : #define FD_TXSEND_QUIC_VOTE_MIN_CONN_COOLDOWN_SECONDS (2L)
      42             : 
      43             : /* the 1M lets this be integer math */
      44             : FD_STATIC_ASSERT((60*1000000)/FD_TXSEND_QUIC_MIN_CONN_LIFETIME_SECONDS <= 1000000*FD_AGAVE_MAX_CONNS_PER_MINUTE, "QUIC conn lifetime too low for rate limit");
      45             : 
      46           0 : #define FD_TXSEND_QUIC_IDLE_TIMEOUT_NS (30e9L) /* 30 s - minimize keep_alive work */
      47           0 : #define FD_TXSEND_QUIC_ACK_DELAY_NS    (25e6L) /* 25 ms */
      48             : 
      49             : /* quic ports first, so we can re-use idx to select conn ptr
      50             :    Don't rearrange, lots of stuff depends on this order. */
      51           0 : #define FD_TXSEND_PORT_QUIC_VOTE_IDX  (0UL)
      52           0 : #define FD_TXSEND_PORT_QUIC_TPU_IDX   (1UL)
      53             : #define FD_TXSEND_PORT_UDP_VOTE_IDX   (2UL)
      54             : #define FD_TXSEND_PORT_UDP_TPU_IDX    (3UL)
      55           0 : #define FD_TXSEND_PORT_QUIC_CNT       (2UL)
      56           0 : #define FD_TXSEND_PORT_CNT            (4UL)
      57             : 
      58             : struct fd_txsend_link_in {
      59             :   fd_wksp_t *  mem;
      60             :   ulong        chunk0;
      61             :   ulong        wmark;
      62             :   ulong        kind;
      63             :   void      *  dcache;
      64             : };
      65             : typedef struct fd_txsend_link_in fd_txsend_link_in_t;
      66             : 
      67             : struct fd_txsend_link_out {
      68             :   ulong            idx;
      69             :   fd_frag_meta_t * mcache;
      70             :   ulong *          sync;
      71             :   ulong            depth;
      72             : 
      73             :   fd_wksp_t * mem;
      74             :   ulong       chunk0;
      75             :   ulong       wmark;
      76             :   ulong       chunk;
      77             : };
      78             : typedef struct fd_txsend_link_out fd_txsend_link_out_t;
      79             : 
      80             : struct fd_txsend_conn_entry {
      81             :   fd_pubkey_t      pubkey;
      82             :   uint             hash;
      83             : 
      84             :   fd_quic_conn_t * conn[ FD_TXSEND_PORT_UDP_VOTE_IDX ]; /* quic ports first in enum */
      85             :   long             last_quic_vote_close;
      86             : 
      87             :   uint             ip4s [ FD_TXSEND_PORT_CNT ]; /* net order */
      88             :   ushort           ports[ FD_TXSEND_PORT_CNT ]; /* host order */
      89             : };
      90             : typedef struct fd_txsend_conn_entry fd_txsend_conn_entry_t;
      91             : 
      92             : 
      93             : struct fd_txsend_tile_ctx {
      94             : 
      95             :   /* link things */
      96             :   #define FD_TXSEND_MAX_IN_LINK_CNT 32UL
      97             :   fd_stem_context_t *  stem;
      98             :   fd_txsend_link_in_t    in_links[ FD_TXSEND_MAX_IN_LINK_CNT ];
      99             :   fd_net_rx_bounds_t   net_in_bounds[ FD_TXSEND_MAX_IN_LINK_CNT ];
     100             :   fd_txsend_link_out_t   gossip_verify_out[ 1 ];
     101             :   fd_txsend_link_out_t   net_out          [ 1 ];
     102             : 
     103             :   fd_keyguard_client_t keyguard_client  [ 1 ];
     104             : 
     105             :   /* buffers btwn during_frag and after_frag :( */
     106             :   union {
     107             :     /* IN_KIND_GOSSIP */
     108             :     struct {
     109             :       fd_shred_dest_wire_t       contact_buf[ MAX_STAKED_LEADERS ];
     110             :       ulong                      contact_cnt;
     111             :     };
     112             : 
     113             :     /* IN_KIND_NET */
     114             :     uchar quic_buf[ FD_NET_MTU ];
     115             :   };
     116             : 
     117             :   /* networking things */
     118             :   uint               src_ip_addr;
     119             :   ushort             src_port;
     120             :   fd_ip4_udp_hdrs_t  packet_hdr[1]; /* template, but will be modified directly */
     121             :   ushort             net_id;
     122             : 
     123             :   /* identity pubkey used for tls */
     124             :   fd_pubkey_t identity_key[ 1 ];
     125             :   /* Leader schedule tracking */
     126             :   fd_multi_epoch_leaders_t * mleaders;
     127             : 
     128             :   /* QUIC handles */
     129             :   fd_quic_t * quic;
     130             :   fd_aio_t    quic_tx_aio[1];
     131             : 
     132             :   /* Connection map for outgoing QUIC connections and contact info */
     133             :   fd_txsend_conn_entry_t * conn_map;
     134             : 
     135             :   /* timekeeping */
     136             :   long             now;            /* current time in ns!     */
     137             :   fd_clock_t       clock[1];       /* memory for fd_clock_t   */
     138             :   long             recal_next;     /* next recalibration time (ns) */
     139             : 
     140             :   struct {
     141             :     /* Contact info */
     142             :     ulong unstaked_ci_rcvd;
     143             :     ulong new_contact_info[FD_TXSEND_PORT_CNT][FD_METRICS_ENUM_NEW_CONTACT_OUTCOME_CNT];
     144             :     ulong ci_removed;
     145             : 
     146             :     /* Outcome of trying to send data */
     147             :     ulong send_result_cnt[FD_TXSEND_PORT_CNT][FD_METRICS_ENUM_TXN_SEND_RESULT_CNT];
     148             : 
     149             :     /* QUIC-specific metrics */
     150             :     ulong quic_hs_complete   [FD_METRICS_ENUM_TXSEND_QUIC_PORTS_CNT];
     151             :     ulong quic_conn_final    [FD_METRICS_ENUM_TXSEND_QUIC_PORTS_CNT];
     152             :     ulong ensure_conn_result [FD_METRICS_ENUM_TXSEND_QUIC_PORTS_CNT]
     153             :                                 [FD_METRICS_ENUM_TXSEND_ENSURE_CONN_RESULT_CNT];
     154             : 
     155             :     /* Time spent waiting for tls_cv signatures */
     156             :     fd_histf_t sign_duration[ 1 ];
     157             :   } metrics;
     158             : 
     159             :   uchar __attribute__((aligned(FD_MULTI_EPOCH_LEADERS_ALIGN))) mleaders_mem[ FD_MULTI_EPOCH_LEADERS_FOOTPRINT ];
     160             :   uchar __attribute__((aligned(FD_CLOCK_ALIGN))) clock_mem[ FD_CLOCK_FOOTPRINT ];
     161             : };
     162             : 
     163             : typedef struct fd_txsend_tile_ctx fd_txsend_tile_ctx_t;
     164             : 
     165             : #endif /* HEADER_fd_src_discof_txsend_fd_txsend_tile_h */
     166             :