Line data Source code
1 : /* Repair tile runs the repair protocol for a Firedancer node. */
2 : #define _GNU_SOURCE
3 :
4 : #include "../../disco/topo/fd_topo.h"
5 : #include <sys/socket.h>
6 :
7 : #include "generated/fd_rpcserv_tile_seccomp.h"
8 :
9 : #include "../rpcserver/fd_rpc_service.h"
10 :
11 : #include "../../flamenco/leaders/fd_multi_epoch_leaders.h"
12 : #include "../../util/pod/fd_pod_format.h"
13 : #include "../../disco/keyguard/fd_keyload.h"
14 : #include "../../disco/keyguard/fd_keyswitch.h"
15 :
16 : #include <errno.h>
17 : #include <fcntl.h>
18 : #include <unistd.h>
19 :
20 0 : #define REPLAY_NOTIF_IDX 0
21 0 : #define STAKE_IN_IDX 1
22 :
23 : struct fd_rpcserv_tile_ctx {
24 : fd_rpcserver_args_t args;
25 :
26 : fd_rpc_ctx_t * ctx;
27 : fd_store_t * store;
28 :
29 : fd_pubkey_t identity_key;
30 : fd_keyswitch_t * keyswitch;
31 :
32 : fd_wksp_t * replay_notif_in_mem;
33 : ulong replay_notif_in_chunk0;
34 : ulong replay_notif_in_wmark;
35 : fd_replay_notif_msg_t replay_notif_in_state;
36 :
37 : fd_wksp_t * stake_in_mem;
38 : ulong stake_in_chunk0;
39 : ulong stake_in_wmark;
40 :
41 : uchar __attribute__((aligned(FD_MULTI_EPOCH_LEADERS_ALIGN))) mleaders_mem[ FD_MULTI_EPOCH_LEADERS_FOOTPRINT ];
42 : };
43 : typedef struct fd_rpcserv_tile_ctx fd_rpcserv_tile_ctx_t;
44 :
45 0 : #define FD_RPC_SCRATCH_MAX (1LU<<30)
46 :
47 : const fd_http_server_params_t RPCSERV_HTTP_PARAMS = {
48 : .max_connection_cnt = 10,
49 : .max_ws_connection_cnt = 10,
50 : .max_request_len = 1<<16,
51 : .max_ws_recv_frame_len = 1<<16,
52 : .max_ws_send_frame_cnt = 10,
53 : .outgoing_buffer_sz = 100<<20,
54 : };
55 :
56 : FD_FN_CONST static inline ulong
57 0 : scratch_align( void ) {
58 0 : return 128UL;
59 0 : }
60 :
61 : FD_FN_PURE static inline ulong
62 0 : scratch_footprint( fd_topo_tile_t const * tile FD_PARAM_UNUSED) {
63 0 : ulong l = FD_LAYOUT_INIT;
64 0 : l = FD_LAYOUT_APPEND( l, alignof(fd_rpcserv_tile_ctx_t), sizeof(fd_rpcserv_tile_ctx_t) );
65 0 : l = FD_LAYOUT_APPEND( l, fd_spad_align(), fd_spad_footprint( FD_RPC_SCRATCH_MAX ) );
66 0 : return FD_LAYOUT_FINI( l, scratch_align() );
67 0 : }
68 :
69 : FD_FN_PURE static inline ulong
70 0 : loose_footprint( fd_topo_tile_t const * tile FD_PARAM_UNUSED ) {
71 0 : return 1UL * FD_SHMEM_GIGANTIC_PAGE_SZ;
72 0 : }
73 :
74 : static inline void
75 0 : during_housekeeping( fd_rpcserv_tile_ctx_t * ctx ) {
76 0 : if( FD_UNLIKELY( fd_keyswitch_state_query( ctx->keyswitch )==FD_KEYSWITCH_STATE_SWITCH_PENDING ) ) {
77 0 : memcpy( &ctx->identity_key, ctx->keyswitch->bytes, sizeof(fd_pubkey_t) );
78 0 : fd_keyswitch_state( ctx->keyswitch, FD_KEYSWITCH_STATE_COMPLETED );
79 0 : }
80 0 : }
81 :
82 : static inline void
83 : before_credit( fd_rpcserv_tile_ctx_t * ctx,
84 : fd_stem_context_t * stem,
85 0 : int * charge_busy ) {
86 0 : (void)stem;
87 0 : *charge_busy = fd_rpc_ws_poll( ctx->ctx );
88 0 : }
89 :
90 : static void
91 : during_frag( fd_rpcserv_tile_ctx_t * ctx,
92 : ulong in_idx,
93 : ulong seq FD_PARAM_UNUSED,
94 : ulong sig FD_PARAM_UNUSED,
95 : ulong chunk,
96 : ulong sz,
97 0 : ulong ctl FD_PARAM_UNUSED ) {
98 :
99 0 : if( FD_UNLIKELY( in_idx==REPLAY_NOTIF_IDX ) ) {
100 0 : if( FD_UNLIKELY( chunk<ctx->replay_notif_in_chunk0 || chunk>ctx->replay_notif_in_wmark ) ) {
101 0 : FD_LOG_ERR(( "chunk %lu %lu corrupt, not in range [%lu,%lu]", chunk, sz,
102 0 : ctx->replay_notif_in_chunk0, ctx->replay_notif_in_wmark ));
103 0 : }
104 0 : fd_rpc_replay_during_frag( ctx->ctx, &ctx->replay_notif_in_state, fd_chunk_to_laddr_const( ctx->replay_notif_in_mem, chunk ), (int)sz );
105 :
106 0 : } else if( FD_UNLIKELY( in_idx==STAKE_IN_IDX ) ) {
107 0 : if( FD_UNLIKELY( chunk<ctx->stake_in_chunk0 || chunk>ctx->stake_in_wmark ) ) {
108 0 : FD_LOG_ERR(( "chunk %lu %lu corrupt, not in range [%lu,%lu]", chunk, sz,
109 0 : ctx->stake_in_chunk0, ctx->stake_in_wmark ));
110 0 : }
111 0 : fd_rpc_stake_during_frag( ctx->ctx, ctx->args.leaders, fd_chunk_to_laddr_const( ctx->stake_in_mem, chunk ), (int)sz );
112 :
113 0 : } else {
114 0 : FD_LOG_ERR(("Unknown in_idx %lu for rpc", in_idx));
115 0 : }
116 0 : }
117 :
118 : static void
119 : after_frag( fd_rpcserv_tile_ctx_t * ctx,
120 : ulong in_idx,
121 : ulong seq,
122 : ulong sig,
123 : ulong sz,
124 : ulong tsorig,
125 : ulong tspub,
126 0 : fd_stem_context_t * stem ) {
127 0 : (void)seq;
128 0 : (void)sig;
129 0 : (void)sz;
130 0 : (void)tsorig;
131 0 : (void)tspub;
132 0 : (void)stem;
133 :
134 0 : if( FD_LIKELY( in_idx==REPLAY_NOTIF_IDX ) ) {
135 0 : fd_rpc_replay_after_frag( ctx->ctx, &ctx->replay_notif_in_state );
136 0 : } else if( FD_UNLIKELY( in_idx==STAKE_IN_IDX ) ) {
137 0 : fd_rpc_stake_after_frag( ctx->ctx, ctx->args.leaders );
138 0 : } else {
139 0 : FD_LOG_ERR(("Unknown in_idx %lu for rpc", in_idx));
140 0 : }
141 0 : }
142 :
143 : static void
144 : privileged_init( fd_topo_t * topo,
145 0 : fd_topo_tile_t * tile ) {
146 0 : void * scratch = fd_topo_obj_laddr( topo, tile->tile_obj_id );
147 :
148 0 : FD_SCRATCH_ALLOC_INIT( l, scratch );
149 0 : fd_rpcserv_tile_ctx_t * ctx = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_rpcserv_tile_ctx_t), sizeof(fd_rpcserv_tile_ctx_t) );
150 0 : void * spad_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_spad_align(), fd_spad_footprint( FD_RPC_SCRATCH_MAX ) );
151 0 : FD_SCRATCH_ALLOC_FINI( l, scratch_align() );
152 :
153 0 : if( FD_UNLIKELY( !strcmp( tile->rpcserv.identity_key_path, "" ) ) )
154 0 : FD_LOG_ERR( ( "identity_key_path not set" ) );
155 0 : memcpy( &ctx->identity_key, fd_keyload_load( tile->rpcserv.identity_key_path, /* pubkey only: */ 1 ), sizeof(fd_pubkey_t) );
156 :
157 0 : fd_rpcserver_args_t * args = &ctx->args;
158 0 : fd_memset( args, 0, sizeof(fd_rpcserver_args_t) );
159 :
160 0 : args->offline = 0;
161 0 : args->params = RPCSERV_HTTP_PARAMS;
162 :
163 0 : args->port = tile->rpcserv.rpc_port;
164 :
165 0 : args->tpu_addr.sin_family = AF_INET;
166 0 : args->tpu_addr.sin_addr.s_addr = tile->rpcserv.tpu_ip_addr;
167 0 : args->tpu_addr.sin_port = htons( (ushort)tile->rpcserv.tpu_port );
168 :
169 0 : args->leaders = fd_multi_epoch_leaders_join( fd_multi_epoch_leaders_new( ctx->mleaders_mem) );
170 :
171 0 : uchar * spad_mem_cur = spad_mem;
172 0 : args->spad = fd_spad_join( fd_spad_new( spad_mem_cur, FD_RPC_SCRATCH_MAX ) );
173 :
174 : /* Blockstore setup */
175 0 : ulong store_obj_id = fd_pod_queryf_ulong( topo->props, ULONG_MAX, "store" );
176 0 : FD_TEST( store_obj_id!=ULONG_MAX );
177 0 : args->store = fd_store_join( ctx->store );
178 0 : FD_TEST( args->store!=NULL );
179 :
180 0 : args->block_index_max = tile->rpcserv.block_index_max;
181 0 : args->txn_index_max = tile->rpcserv.txn_index_max;
182 0 : args->acct_index_max = tile->rpcserv.acct_index_max;
183 0 : strncpy( args->history_file, tile->rpcserv.history_file, sizeof(args->history_file) );
184 0 : args->identity_key = &ctx->identity_key;
185 :
186 0 : fd_spad_push( args->spad ); /* We close this out when we stop the server */
187 0 : fd_rpc_create_ctx( args, &ctx->ctx );
188 :
189 :
190 : /* Wait until after replay tile boots before starting service */
191 0 : }
192 :
193 : static void
194 : unprivileged_init( fd_topo_t * topo,
195 0 : fd_topo_tile_t * tile ) {
196 0 : void * scratch = fd_topo_obj_laddr( topo, tile->tile_obj_id );
197 :
198 0 : if( FD_UNLIKELY( tile->in_cnt != 2 ||
199 0 : strcmp( topo->links[ tile->in_link_id[ REPLAY_NOTIF_IDX ] ].name, "replay_notif") ||
200 0 : strcmp( topo->links[ tile->in_link_id[ STAKE_IN_IDX ] ].name, "stake_out" ) ) ) {
201 0 : FD_LOG_ERR(( "repair tile has none or unexpected input links %lu %s %s",
202 0 : tile->in_cnt, topo->links[ tile->in_link_id[ 0 ] ].name, topo->links[ tile->in_link_id[ 1 ] ].name ));
203 0 : }
204 :
205 0 : if( FD_UNLIKELY( tile->out_cnt != 0 ) ) {
206 0 : FD_LOG_ERR(( "repair tile has none or unexpected output links %lu %s %s",
207 0 : tile->out_cnt, topo->links[ tile->out_link_id[ 0 ] ].name, topo->links[ tile->out_link_id[ 1 ] ].name ));
208 0 : }
209 :
210 : /* Scratch mem setup */
211 :
212 0 : FD_SCRATCH_ALLOC_INIT( l, scratch );
213 0 : fd_rpcserv_tile_ctx_t * ctx = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_rpcserv_tile_ctx_t), sizeof(fd_rpcserv_tile_ctx_t) );
214 0 : FD_SCRATCH_ALLOC_APPEND( l, fd_spad_align(), fd_ulong_align_up( FD_RPC_SCRATCH_MAX, fd_spad_align() ) );
215 0 : ulong scratch_top = FD_SCRATCH_ALLOC_FINI( l, scratch_align() );
216 0 : if( FD_UNLIKELY( scratch_top > (ulong)scratch + scratch_footprint( tile ) ) )
217 0 : FD_LOG_ERR(( "scratch overflow %lu %lu %lu", scratch_top - (ulong)scratch - scratch_footprint( tile ), scratch_top, (ulong)scratch + scratch_footprint( tile ) ));
218 :
219 0 : fd_topo_link_t * replay_notif_in_link = &topo->links[ tile->in_link_id[ REPLAY_NOTIF_IDX ] ];
220 0 : ctx->replay_notif_in_mem = topo->workspaces[ topo->objs[ replay_notif_in_link->dcache_obj_id ].wksp_id ].wksp;
221 0 : ctx->replay_notif_in_chunk0 = fd_dcache_compact_chunk0( ctx->replay_notif_in_mem, replay_notif_in_link->dcache );
222 0 : ctx->replay_notif_in_wmark = fd_dcache_compact_wmark ( ctx->replay_notif_in_mem, replay_notif_in_link->dcache, replay_notif_in_link->mtu );
223 :
224 0 : fd_topo_link_t * stake_in_link = &topo->links[ tile->in_link_id[ STAKE_IN_IDX ] ];
225 0 : ctx->stake_in_mem = topo->workspaces[ topo->objs[ stake_in_link->dcache_obj_id ].wksp_id ].wksp;
226 0 : ctx->stake_in_chunk0 = fd_dcache_compact_chunk0( ctx->stake_in_mem, stake_in_link->dcache );
227 0 : ctx->stake_in_wmark = fd_dcache_compact_wmark ( ctx->stake_in_mem, stake_in_link->dcache, stake_in_link->mtu );
228 :
229 0 : ctx->keyswitch = fd_keyswitch_join( fd_topo_obj_laddr( topo, tile->keyswitch_obj_id ) );
230 0 : FD_TEST( ctx->keyswitch );
231 :
232 0 : fd_rpcserver_args_t * args = &ctx->args;
233 0 : if( FD_UNLIKELY( !fd_funk_join( args->funk, fd_topo_obj_laddr( topo, tile->rpcserv.funk_obj_id ) ) ) ) {
234 0 : FD_LOG_ERR(( "Failed to join database cache" ));
235 0 : }
236 0 : fd_rpc_start_service( args, ctx->ctx );
237 0 : }
238 :
239 : static ulong
240 : populate_allowed_seccomp( fd_topo_t const * topo,
241 : fd_topo_tile_t const * tile,
242 : ulong out_cnt,
243 0 : struct sock_filter * out ) {
244 0 : void * scratch = fd_topo_obj_laddr( topo, tile->tile_obj_id );
245 0 : FD_SCRATCH_ALLOC_INIT( l, scratch );
246 0 : fd_rpcserv_tile_ctx_t * ctx = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_rpcserv_tile_ctx_t), sizeof(fd_rpcserv_tile_ctx_t) );
247 :
248 0 : populate_sock_filter_policy_fd_rpcserv_tile( out_cnt, out, (uint)fd_log_private_logfile_fd(), (uint)fd_rpc_ws_fd( ctx->ctx ) );
249 0 : return sock_filter_policy_fd_rpcserv_tile_instr_cnt;
250 0 : }
251 :
252 : static ulong
253 : populate_allowed_fds( fd_topo_t const * topo,
254 : fd_topo_tile_t const * tile,
255 : ulong out_fds_cnt,
256 0 : int * out_fds ) {
257 0 : void * scratch = fd_topo_obj_laddr( topo, tile->tile_obj_id );
258 0 : FD_SCRATCH_ALLOC_INIT( l, scratch );
259 0 : fd_rpcserv_tile_ctx_t * ctx = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_rpcserv_tile_ctx_t), sizeof(fd_rpcserv_tile_ctx_t) );
260 :
261 0 : if( FD_UNLIKELY( out_fds_cnt<3UL ) ) FD_LOG_ERR(( "out_fds_cnt %lu", out_fds_cnt ));
262 :
263 0 : ulong out_cnt = 0;
264 0 : out_fds[ out_cnt++ ] = 2; /* stderr */
265 0 : if( FD_LIKELY( -1!=fd_log_private_logfile_fd() ) )
266 0 : out_fds[ out_cnt++ ] = fd_log_private_logfile_fd(); /* logfile */
267 0 : out_fds[ out_cnt++ ] = fd_rpc_ws_fd( ctx->ctx ); /* listen socket */
268 0 : return out_cnt;
269 0 : }
270 :
271 : /* TODO: This is probably not correct. */
272 0 : #define STEM_BURST (1UL)
273 :
274 0 : #define STEM_CALLBACK_CONTEXT_TYPE fd_rpcserv_tile_ctx_t
275 0 : #define STEM_CALLBACK_CONTEXT_ALIGN alignof(fd_rpcserv_tile_ctx_t)
276 :
277 0 : #define STEM_CALLBACK_BEFORE_CREDIT before_credit
278 0 : #define STEM_CALLBACK_DURING_HOUSEKEEPING during_housekeeping
279 0 : #define STEM_CALLBACK_DURING_FRAG during_frag
280 0 : #define STEM_CALLBACK_AFTER_FRAG after_frag
281 :
282 : #include "../../disco/stem/fd_stem.c"
283 :
284 : fd_topo_run_tile_t fd_tile_rpcserv = {
285 : .name = "rpcsrv",
286 : .loose_footprint = loose_footprint,
287 : .populate_allowed_seccomp = populate_allowed_seccomp,
288 : .populate_allowed_fds = populate_allowed_fds,
289 : .scratch_align = scratch_align,
290 : .scratch_footprint = scratch_footprint,
291 : .unprivileged_init = unprivileged_init,
292 : .privileged_init = privileged_init,
293 : .run = stem_run,
294 : };
|
