Line data    Source code

       1             : #ifndef HEADER_fd_src_discof_send_fd_send_tile_h
       2             : #define HEADER_fd_src_discof_send_fd_send_tile_h
       3             : 
       4             : /* Sender tile signs and sends transactions to the current leader.
       5             :    Currently only supports transactions which require one signature.
       6             :    Designed with voting as primary use case. Signing those votes will
       7             :    eventually move to a separate consensus tile.*/
       8             : #define _GNU_SOURCE
       9             : 
      10             : #include "../../util/net/fd_net_headers.h"
      11             : #include "../../disco/stem/fd_stem.h"
      12             : #include "../../disco/fd_disco.h"
      13             : #include "../../disco/net/fd_net_tile.h"
      14             : #include "../../disco/keyguard/fd_keyguard_client.h"
      15             : #include "../../flamenco/leaders/fd_multi_epoch_leaders.h"
      16             : #include "../../waltz/quic/fd_quic.h"
      17             : #include "../../util/clock/fd_clock.h"
      18             : 
      19             : #define IN_KIND_SIGN   (0UL)
      20           0 : #define IN_KIND_GOSSIP (1UL)
      21           0 : #define IN_KIND_STAKE  (2UL)
      22           0 : #define IN_KIND_TOWER  (3UL)
      23           0 : #define IN_KIND_NET    (4UL)
      24             : 
      25             : /* Send votes to next FD_SEND_TARGET_LEADER_CNT leaders (slot x, x+4, x+8, ...) */
      26           0 : #define FD_SEND_TARGET_LEADER_CNT (3UL)
      27             : 
      28             : /* Connect FD_CONNECT_AHEAD_LEADER_CNT leaders ahead (slot x, x+4, x+8, ...) */
      29           0 : #define FD_SEND_CONNECT_AHEAD_LEADER_CNT  (7UL)
      30             : 
      31             : /* Agave currently rate limits connections per minute per IP */
      32             : #define FD_AGAVE_MAX_CONNS_PER_MINUTE (8UL)
      33             : /* so each of our connections must survive at least 60/8 = 7.5 seconds
      34             :    Let's conservatively go to 10 */
      35           0 : #define FD_SEND_QUIC_MIN_CONN_LIFETIME_SECONDS (10L)
      36             : 
      37             : /* Wait FD_SEND_QUIC_VOTE_MIN_CONN_COOLDOWN_SECONDS many seconds before
      38             :    re-establishing a conn to a quic_vote port. Why?
      39             :    After timing out, the agave server puts the conn in a draining state, during
      40             :    which time it remains in their connection map. So our new attempt gets
      41             :    rejected, as the quic_vote port limits to 1 conn per client. Without cooldown,
      42             :    we keep trying rapidly, and can quickly hit their 8 conn/min limit.
      43             :    That prevents us from connecting for far longer than just cooling down
      44             :    (which should be free because we connect ahead). This number is based
      45             :    on empirical observation, but has much room for improvement. */
      46           0 : #define FD_SEND_QUIC_VOTE_MIN_CONN_COOLDOWN_SECONDS (2L)
      47             : 
      48             : /* the 1M lets this be integer math */
      49             : FD_STATIC_ASSERT((60*1000000)/FD_SEND_QUIC_MIN_CONN_LIFETIME_SECONDS <= 1000000*FD_AGAVE_MAX_CONNS_PER_MINUTE, "QUIC conn lifetime too low for rate limit");
      50             : 
      51           0 : #define FD_SEND_QUIC_IDLE_TIMEOUT_NS (30e9L) /* 30 s - minimize keep_alive work */
      52           0 : #define FD_SEND_QUIC_ACK_DELAY_NS    (25e6L) /* 25 ms */
      53             : 
      54             : /* quic ports first, so we can re-use idx to select conn ptr
      55             :    Don't rearrange, lots of stuff depends on this order. */
      56           0 : #define FD_SEND_PORT_QUIC_VOTE_IDX  (0UL)
      57           0 : #define FD_SEND_PORT_QUIC_TPU_IDX   (1UL)
      58             : #define FD_SEND_PORT_UDP_VOTE_IDX   (2UL)
      59             : #define FD_SEND_PORT_UDP_TPU_IDX    (3UL)
      60           0 : #define FD_SEND_PORT_QUIC_CNT       (2UL)
      61           0 : #define FD_SEND_PORT_CNT            (4UL)
      62             : 
      63             : struct fd_send_link_in {
      64             :   fd_wksp_t *  mem;
      65             :   ulong        chunk0;
      66             :   ulong        wmark;
      67             :   ulong        kind;
      68             :   void      *  dcache;
      69             : };
      70             : typedef struct fd_send_link_in fd_send_link_in_t;
      71             : 
      72             : struct fd_send_link_out {
      73             :   ulong            idx;
      74             :   fd_frag_meta_t * mcache;
      75             :   ulong *          sync;
      76             :   ulong            depth;
      77             : 
      78             :   fd_wksp_t * mem;
      79             :   ulong       chunk0;
      80             :   ulong       wmark;
      81             :   ulong       chunk;
      82             : };
      83             : typedef struct fd_send_link_out fd_send_link_out_t;
      84             : 
      85             : struct fd_send_conn_entry {
      86             :   fd_pubkey_t      pubkey;
      87             :   uint             hash;
      88             : 
      89             :   fd_quic_conn_t * conn[ FD_SEND_PORT_UDP_VOTE_IDX ]; /* quic ports first in enum */
      90             :   long             last_quic_vote_close;
      91             : 
      92             :   uint             ip4s [ FD_SEND_PORT_CNT ]; /* net order */
      93             :   ushort           ports[ FD_SEND_PORT_CNT ]; /* host order */
      94             : };
      95             : typedef struct fd_send_conn_entry fd_send_conn_entry_t;
      96             : 
      97             : 
      98             : struct fd_send_tile_ctx {
      99             : 
     100             :   /* link things */
     101             :   #define FD_SEND_MAX_IN_LINK_CNT 32UL
     102             :   fd_stem_context_t *  stem;
     103             :   fd_send_link_in_t    in_links[ FD_SEND_MAX_IN_LINK_CNT ];
     104             :   fd_net_rx_bounds_t   net_in_bounds[ FD_SEND_MAX_IN_LINK_CNT ];
     105             :   fd_send_link_out_t   gossip_verify_out[ 1 ];
     106             :   fd_send_link_out_t   net_out          [ 1 ];
     107             : 
     108             :   fd_keyguard_client_t keyguard_client  [ 1 ];
     109             : 
     110             :   /* buffers btwn during_frag and after_frag :( */
     111             :   union {
     112             :     /* IN_KIND_GOSSIP */
     113             :     struct {
     114             :       fd_shred_dest_wire_t       contact_buf[ MAX_STAKED_LEADERS ];
     115             :       ulong                      contact_cnt;
     116             :     };
     117             : 
     118             :     /* IN_KIND_NET */
     119             :     uchar quic_buf[ FD_NET_MTU ];
     120             :   };
     121             : 
     122             :   /* networking things */
     123             :   uint               src_ip_addr;
     124             :   ushort             src_port;
     125             :   fd_ip4_udp_hdrs_t  packet_hdr[1]; /* template, but will be modified directly */
     126             :   ushort             net_id;
     127             : 
     128             :   /* tls pubkey */
     129             :   fd_pubkey_t identity_key  [ 1 ];    /* also tls pubkey - only really used by quic */
     130             : 
     131             :   /* Leader schedule tracking */
     132             :   fd_multi_epoch_leaders_t * mleaders;
     133             : 
     134             :   /* QUIC handles */
     135             :   fd_quic_t * quic;
     136             :   fd_aio_t    quic_tx_aio[1];
     137             : 
     138             :   /* Connection map for outgoing QUIC connections and contact info */
     139             :   fd_send_conn_entry_t * conn_map;
     140             : 
     141             :   /* timekeeping */
     142             :   long             now;            /* current time in ns!     */
     143             :   fd_clock_t       clock[1];       /* memory for fd_clock_t   */
     144             :   long             recal_next;     /* next recalibration time (ns) */
     145             : 
     146             :   struct {
     147             :     ulong leader_not_found;
     148             : 
     149             :     /* Contact info */
     150             :     ulong unstaked_ci_rcvd;
     151             :     ulong new_contact_info[FD_SEND_PORT_CNT][FD_METRICS_ENUM_NEW_CONTACT_OUTCOME_CNT];
     152             :     ulong ci_removed;
     153             : 
     154             :     /* Outcome of trying to send data */
     155             :     ulong send_result_cnt[FD_SEND_PORT_CNT][FD_METRICS_ENUM_TXN_SEND_RESULT_CNT];
     156             : 
     157             :     /* QUIC-specific metrics */
     158             :     ulong quic_hs_complete   [FD_METRICS_ENUM_SEND_QUIC_PORTS_CNT];
     159             :     ulong quic_conn_final    [FD_METRICS_ENUM_SEND_QUIC_PORTS_CNT];
     160             :     ulong ensure_conn_result [FD_METRICS_ENUM_SEND_QUIC_PORTS_CNT]
     161             :                                 [FD_METRICS_ENUM_SEND_ENSURE_CONN_RESULT_CNT];
     162             : 
     163             :     /* Time spent waiting for tls_cv signatures */
     164             :     fd_histf_t sign_duration[ 1 ];
     165             :   } metrics;
     166             : 
     167             :   uchar __attribute__((aligned(FD_MULTI_EPOCH_LEADERS_ALIGN))) mleaders_mem[ FD_MULTI_EPOCH_LEADERS_FOOTPRINT ];
     168             :   uchar __attribute__((aligned(FD_CLOCK_ALIGN))) clock_mem[ FD_CLOCK_FOOTPRINT ];
     169             : };
     170             : 
     171             : typedef struct fd_send_tile_ctx fd_send_tile_ctx_t;
     172             : 
     173             : #endif /* HEADER_fd_src_discof_send_fd_send_tile_h */
     174             :