Line data Source code
1 : /* Repair tile runs the repair protocol for a Firedancer node. */
2 : #define _GNU_SOURCE
3 :
4 : #include "../../disco/topo/fd_topo.h"
5 : #include <sys/socket.h>
6 : #include "generated/fd_rpcserv_tile_seccomp.h"
7 :
8 : #include "../rpcserver/fd_rpc_service.h"
9 :
10 : #include "../../disco/tiles.h"
11 : #include "../../flamenco/runtime/fd_blockstore.h"
12 : #include "../../flamenco/fd_flamenco.h"
13 : #include "../../util/fd_util.h"
14 : #include "../../disco/fd_disco.h"
15 : #include "../../disco/shred/fd_stake_ci.h"
16 : #include "../../disco/topo/fd_pod_format.h"
17 : #include "../../funk/fd_funk_filemap.h"
18 : #include "../../disco/keyguard/fd_keyload.h"
19 :
20 : #include <errno.h>
21 : #include <fcntl.h>
22 : #include <unistd.h>
23 :
24 0 : #define REPLAY_NOTIF_IDX 0
25 0 : #define STAKE_CI_IN_IDX 1
26 :
27 : struct fd_rpcserv_tile_ctx {
28 : fd_rpcserver_args_t args;
29 : char funk_file[ PATH_MAX ];
30 :
31 : int activated;
32 :
33 : fd_rpc_ctx_t * ctx;
34 :
35 : fd_pubkey_t identity_key[1]; /* Just the public key */
36 :
37 : fd_wksp_t * replay_notif_in_mem;
38 : ulong replay_notif_in_chunk0;
39 : ulong replay_notif_in_wmark;
40 : fd_replay_notif_msg_t replay_notif_in_state;
41 :
42 : fd_wksp_t * stake_ci_in_mem;
43 : ulong stake_ci_in_chunk0;
44 : ulong stake_ci_in_wmark;
45 :
46 : int blockstore_fd;
47 : };
48 : typedef struct fd_rpcserv_tile_ctx fd_rpcserv_tile_ctx_t;
49 :
50 0 : #define FD_RPC_SCRATCH_MAX (1LU<<28)
51 0 : #define FD_RPC_SCRATCH_DEPTH 64
52 :
53 : const fd_http_server_params_t RPCSERV_HTTP_PARAMS = {
54 : .max_connection_cnt = 10,
55 : .max_ws_connection_cnt = 10,
56 : .max_request_len = 1<<16,
57 : .max_ws_recv_frame_len = 1<<16,
58 : .max_ws_send_frame_cnt = 10,
59 : .outgoing_buffer_sz = 100<<20,
60 : };
61 :
62 : FD_FN_CONST static inline ulong
63 0 : scratch_align( void ) {
64 0 : return 128UL;
65 0 : }
66 :
67 : FD_FN_PURE static inline ulong
68 0 : scratch_footprint( fd_topo_tile_t const * tile FD_PARAM_UNUSED) {
69 0 : ulong l = FD_LAYOUT_INIT;
70 0 : l = FD_LAYOUT_APPEND( l, alignof(fd_rpcserv_tile_ctx_t), sizeof(fd_rpcserv_tile_ctx_t) );
71 0 : l = FD_LAYOUT_APPEND( l, fd_alloc_align(), fd_alloc_footprint() );
72 0 : l = FD_LAYOUT_APPEND( l, fd_stake_ci_align(), fd_stake_ci_footprint() );
73 0 : l = FD_LAYOUT_APPEND( l, fd_scratch_smem_align(), fd_scratch_smem_footprint( FD_RPC_SCRATCH_MAX ) );
74 0 : l = FD_LAYOUT_APPEND( l, fd_scratch_fmem_align(), fd_scratch_fmem_footprint( FD_RPC_SCRATCH_DEPTH ) );
75 0 : return FD_LAYOUT_FINI( l, scratch_align() );
76 0 : }
77 :
78 : FD_FN_PURE static inline ulong
79 0 : loose_footprint( fd_topo_tile_t const * tile FD_PARAM_UNUSED ) {
80 0 : return 1UL * FD_SHMEM_GIGANTIC_PAGE_SZ;
81 0 : }
82 :
83 : static inline void
84 : before_credit( fd_rpcserv_tile_ctx_t * ctx,
85 : fd_stem_context_t * stem,
86 0 : int * charge_busy ) {
87 0 : (void)stem;
88 :
89 0 : if( FD_UNLIKELY( !ctx->activated ) ) {
90 0 : *charge_busy = 0;
91 0 : } else {
92 0 : *charge_busy = fd_rpc_ws_poll( ctx->ctx );
93 0 : }
94 0 : }
95 :
96 : static void
97 : during_frag( fd_rpcserv_tile_ctx_t * ctx,
98 : ulong in_idx,
99 : ulong seq FD_PARAM_UNUSED,
100 : ulong sig FD_PARAM_UNUSED,
101 : ulong chunk,
102 : ulong sz,
103 0 : ulong ctl FD_PARAM_UNUSED ) {
104 :
105 0 : if( FD_UNLIKELY( in_idx==REPLAY_NOTIF_IDX ) ) {
106 0 : if( FD_UNLIKELY( chunk<ctx->replay_notif_in_chunk0 || chunk>ctx->replay_notif_in_wmark ) ) {
107 0 : FD_LOG_ERR(( "chunk %lu %lu corrupt, not in range [%lu,%lu]", chunk, sz,
108 0 : ctx->replay_notif_in_chunk0, ctx->replay_notif_in_wmark ));
109 0 : }
110 0 : fd_rpc_replay_during_frag( ctx->ctx, &ctx->replay_notif_in_state, fd_chunk_to_laddr_const( ctx->replay_notif_in_mem, chunk ), (int)sz );
111 :
112 0 : } else if( FD_UNLIKELY( in_idx==STAKE_CI_IN_IDX ) ) {
113 0 : if( FD_UNLIKELY( chunk<ctx->stake_ci_in_chunk0 || chunk>ctx->stake_ci_in_wmark ) ) {
114 0 : FD_LOG_ERR(( "chunk %lu %lu corrupt, not in range [%lu,%lu]", chunk, sz,
115 0 : ctx->stake_ci_in_chunk0, ctx->stake_ci_in_wmark ));
116 0 : }
117 0 : fd_rpc_stake_during_frag( ctx->ctx, ctx->args.stake_ci, fd_chunk_to_laddr_const( ctx->stake_ci_in_mem, chunk ), (int)sz );
118 :
119 0 : } else {
120 0 : FD_LOG_ERR(("Unknown in_idx %lu for rpc", in_idx));
121 0 : }
122 0 : }
123 :
124 : static void
125 : after_frag( fd_rpcserv_tile_ctx_t * ctx,
126 : ulong in_idx,
127 : ulong seq,
128 : ulong sig,
129 : ulong sz,
130 : ulong tsorig,
131 : ulong tspub,
132 0 : fd_stem_context_t * stem ) {
133 0 : (void)seq;
134 0 : (void)sig;
135 0 : (void)sz;
136 0 : (void)tsorig;
137 0 : (void)tspub;
138 0 : (void)stem;
139 :
140 0 : if( FD_LIKELY( in_idx==REPLAY_NOTIF_IDX ) ) {
141 0 : if( FD_UNLIKELY( !ctx->activated ) ) {
142 0 : fd_rpcserver_args_t * args = &ctx->args;
143 0 : args->funk = fd_funk_open_file(
144 0 : ctx->funk_file, 1, 0, 0, 0, 0, FD_FUNK_READ_WRITE, NULL );
145 0 : if( FD_UNLIKELY( args->funk == NULL ) ) {
146 0 : FD_LOG_ERR(( "failed to join a funky" ));
147 0 : }
148 :
149 0 : ctx->activated = 1;
150 0 : fd_rpc_start_service( args, ctx->ctx );
151 0 : }
152 :
153 0 : fd_rpc_replay_after_frag( ctx->ctx, &ctx->replay_notif_in_state );
154 :
155 0 : } else if( FD_UNLIKELY( in_idx==STAKE_CI_IN_IDX ) ) {
156 0 : fd_rpc_stake_after_frag( ctx->ctx, ctx->args.stake_ci );
157 :
158 0 : } else {
159 0 : FD_LOG_ERR(("Unknown in_idx %lu for rpc", in_idx));
160 0 : }
161 0 : }
162 :
163 : static void
164 : privileged_init( fd_topo_t * topo,
165 0 : fd_topo_tile_t * tile ) {
166 0 : void * scratch = fd_topo_obj_laddr( topo, tile->tile_obj_id );
167 :
168 0 : FD_SCRATCH_ALLOC_INIT( l, scratch );
169 0 : fd_rpcserv_tile_ctx_t * ctx = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_rpcserv_tile_ctx_t), sizeof(fd_rpcserv_tile_ctx_t) );
170 0 : void * alloc_shmem = FD_SCRATCH_ALLOC_APPEND( l, fd_alloc_align(), fd_alloc_footprint() );
171 0 : void * stake_ci_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_stake_ci_align(), fd_stake_ci_footprint() );
172 0 : FD_SCRATCH_ALLOC_FINI( l, alignof(fd_rpcserv_tile_ctx_t) );
173 :
174 0 : if( FD_UNLIKELY( !strcmp( tile->rpcserv.identity_key_path, "" ) ) )
175 0 : FD_LOG_ERR( ( "identity_key_path not set" ) );
176 0 : ctx->identity_key[0] = *(fd_pubkey_t const *) fd_type_pun_const( fd_keyload_load( tile->rpcserv.identity_key_path, /* pubkey only: */ 1 ) );
177 :
178 0 : fd_rpcserver_args_t * args = &ctx->args;
179 0 : fd_memset( args, 0, sizeof(fd_rpcserver_args_t) );
180 :
181 0 : args->offline = 0;
182 0 : args->params = RPCSERV_HTTP_PARAMS;
183 :
184 0 : args->port = tile->rpcserv.rpc_port;
185 :
186 0 : args->tpu_addr.sin_family = AF_INET;
187 0 : args->tpu_addr.sin_addr.s_addr = tile->rpcserv.tpu_ip_addr;
188 0 : args->tpu_addr.sin_port = htons( (ushort)tile->rpcserv.tpu_port );
189 :
190 0 : args->stake_ci = fd_stake_ci_join( fd_stake_ci_new( stake_ci_mem, ctx->identity_key ) );
191 :
192 0 : strncpy( ctx->funk_file, tile->replay.funk_file, sizeof(ctx->funk_file) );
193 : /* Open funk after replay tile is booted */
194 :
195 : /* Blockstore setup */
196 0 : ulong blockstore_obj_id = fd_pod_queryf_ulong( topo->props, ULONG_MAX, "blockstore" );
197 0 : FD_TEST( blockstore_obj_id!=ULONG_MAX );
198 0 : args->blockstore = fd_blockstore_join( &args->blockstore_ljoin, fd_topo_obj_laddr( topo, blockstore_obj_id ) );
199 0 : FD_TEST( args->blockstore!=NULL );
200 0 : ctx->blockstore_fd = open( tile->replay.blockstore_file, O_RDONLY );
201 0 : if ( FD_UNLIKELY(ctx->blockstore_fd == -1) ){
202 0 : FD_LOG_WARNING(("%s: %s", tile->replay.blockstore_file, strerror( errno )));
203 0 : }
204 :
205 0 : args->blockstore_fd = ctx->blockstore_fd;
206 :
207 0 : void * alloc_shalloc = fd_alloc_new( alloc_shmem, 3UL );
208 0 : if( FD_UNLIKELY( !alloc_shalloc ) ) {
209 0 : FD_LOG_ERR( ( "fd_alloc_new failed" ) ); }
210 0 : fd_alloc_t * alloc = fd_alloc_join( alloc_shalloc, 3UL );
211 0 : if( FD_UNLIKELY( !alloc ) ) {
212 0 : FD_LOG_ERR( ( "fd_alloc_join failed" ) );
213 0 : }
214 0 : args->valloc = fd_alloc_virtual( alloc );
215 :
216 0 : fd_rpc_create_ctx( args, &ctx->ctx );
217 :
218 :
219 : /* Wait until after replay tile boots before starting service */
220 0 : }
221 :
222 : static void
223 : unprivileged_init( fd_topo_t * topo,
224 0 : fd_topo_tile_t * tile ) {
225 0 : void * scratch = fd_topo_obj_laddr( topo, tile->tile_obj_id );
226 :
227 0 : if( FD_UNLIKELY( tile->in_cnt != 2 ||
228 0 : strcmp( topo->links[ tile->in_link_id[ REPLAY_NOTIF_IDX ] ].name, "replay_notif") ||
229 0 : strcmp( topo->links[ tile->in_link_id[ STAKE_CI_IN_IDX ] ].name, "stake_out" ) ) ) {
230 0 : FD_LOG_ERR(( "repair tile has none or unexpected input links %lu %s %s",
231 0 : tile->in_cnt, topo->links[ tile->in_link_id[ 0 ] ].name, topo->links[ tile->in_link_id[ 1 ] ].name ));
232 0 : }
233 :
234 0 : if( FD_UNLIKELY( tile->out_cnt != 0 ) ) {
235 0 : FD_LOG_ERR(( "repair tile has none or unexpected output links %lu %s %s",
236 0 : tile->out_cnt, topo->links[ tile->out_link_id[ 0 ] ].name, topo->links[ tile->out_link_id[ 1 ] ].name ));
237 0 : }
238 :
239 : /* Scratch mem setup */
240 :
241 0 : FD_SCRATCH_ALLOC_INIT( l, scratch );
242 0 : fd_rpcserv_tile_ctx_t * ctx = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_rpcserv_tile_ctx_t), sizeof(fd_rpcserv_tile_ctx_t) );
243 0 : (void)FD_SCRATCH_ALLOC_APPEND( l, fd_alloc_align(), fd_alloc_footprint() );
244 0 : (void)FD_SCRATCH_ALLOC_APPEND( l, fd_stake_ci_align(), fd_stake_ci_footprint() );
245 0 : void * smem = FD_SCRATCH_ALLOC_APPEND( l, fd_scratch_smem_align(), fd_scratch_smem_footprint( FD_RPC_SCRATCH_MAX ) );
246 0 : void * fmem = FD_SCRATCH_ALLOC_APPEND( l, fd_scratch_fmem_align(), fd_scratch_fmem_footprint( FD_RPC_SCRATCH_DEPTH ) );
247 0 : ulong scratch_top = FD_SCRATCH_ALLOC_FINI( l, scratch_align() );
248 0 : if( FD_UNLIKELY( scratch_top > (ulong)scratch + scratch_footprint( tile ) ) )
249 0 : FD_LOG_ERR(( "scratch overflow %lu %lu %lu", scratch_top - (ulong)scratch - scratch_footprint( tile ), scratch_top, (ulong)scratch + scratch_footprint( tile ) ));
250 :
251 0 : FD_TEST( ( !!smem ) & ( !!fmem ) );
252 0 : fd_scratch_attach( smem, fmem, FD_RPC_SCRATCH_MAX, FD_RPC_SCRATCH_DEPTH );
253 :
254 0 : ctx->activated = 0;
255 :
256 0 : fd_topo_link_t * replay_notif_in_link = &topo->links[ tile->in_link_id[ REPLAY_NOTIF_IDX ] ];
257 0 : ctx->replay_notif_in_mem = topo->workspaces[ topo->objs[ replay_notif_in_link->dcache_obj_id ].wksp_id ].wksp;
258 0 : ctx->replay_notif_in_chunk0 = fd_dcache_compact_chunk0( ctx->replay_notif_in_mem, replay_notif_in_link->dcache );
259 0 : ctx->replay_notif_in_wmark = fd_dcache_compact_wmark ( ctx->replay_notif_in_mem, replay_notif_in_link->dcache, replay_notif_in_link->mtu );
260 :
261 0 : fd_topo_link_t * stake_ci_in_link = &topo->links[ tile->in_link_id[ STAKE_CI_IN_IDX ] ];
262 0 : ctx->stake_ci_in_mem = topo->workspaces[ topo->objs[ stake_ci_in_link->dcache_obj_id ].wksp_id ].wksp;
263 0 : ctx->stake_ci_in_chunk0 = fd_dcache_compact_chunk0( ctx->stake_ci_in_mem, stake_ci_in_link->dcache );
264 0 : ctx->stake_ci_in_wmark = fd_dcache_compact_wmark ( ctx->stake_ci_in_mem, stake_ci_in_link->dcache, stake_ci_in_link->mtu );
265 0 : }
266 :
267 : static ulong
268 : populate_allowed_seccomp( fd_topo_t const * topo,
269 : fd_topo_tile_t const * tile,
270 : ulong out_cnt,
271 0 : struct sock_filter * out ) {
272 0 : void * scratch = fd_topo_obj_laddr( topo, tile->tile_obj_id );
273 0 : FD_SCRATCH_ALLOC_INIT( l, scratch );
274 0 : fd_rpcserv_tile_ctx_t * ctx = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_rpcserv_tile_ctx_t), sizeof(fd_rpcserv_tile_ctx_t) );
275 0 : FD_SCRATCH_ALLOC_FINI( l, alignof(fd_rpcserv_tile_ctx_t) );
276 :
277 0 : populate_sock_filter_policy_fd_rpcserv_tile( out_cnt, out, (uint)fd_log_private_logfile_fd(), (uint)fd_rpc_ws_fd( ctx->ctx ), (uint)ctx->blockstore_fd );
278 0 : return sock_filter_policy_fd_rpcserv_tile_instr_cnt;
279 0 : }
280 :
281 : static ulong
282 : populate_allowed_fds( fd_topo_t const * topo,
283 : fd_topo_tile_t const * tile,
284 : ulong out_fds_cnt,
285 0 : int * out_fds ) {
286 0 : void * scratch = fd_topo_obj_laddr( topo, tile->tile_obj_id );
287 0 : FD_SCRATCH_ALLOC_INIT( l, scratch );
288 0 : fd_rpcserv_tile_ctx_t * ctx = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_rpcserv_tile_ctx_t), sizeof(fd_rpcserv_tile_ctx_t) );
289 0 : FD_SCRATCH_ALLOC_FINI( l, alignof(fd_rpcserv_tile_ctx_t) );
290 :
291 0 : if( FD_UNLIKELY( out_fds_cnt<3UL ) ) FD_LOG_ERR(( "out_fds_cnt %lu", out_fds_cnt ));
292 :
293 0 : ulong out_cnt = 0;
294 0 : out_fds[ out_cnt++ ] = 2; /* stderr */
295 0 : if( FD_LIKELY( -1!=fd_log_private_logfile_fd() ) )
296 0 : out_fds[ out_cnt++ ] = fd_log_private_logfile_fd(); /* logfile */
297 0 : out_fds[ out_cnt++ ] = fd_rpc_ws_fd( ctx->ctx ); /* listen socket */
298 0 : out_fds[ out_cnt++ ] = ctx->blockstore_fd;
299 0 : return out_cnt;
300 0 : }
301 :
302 : /* TODO: This is probably not correct. */
303 0 : #define STEM_BURST (1UL)
304 :
305 0 : #define STEM_CALLBACK_CONTEXT_TYPE fd_rpcserv_tile_ctx_t
306 0 : #define STEM_CALLBACK_CONTEXT_ALIGN alignof(fd_rpcserv_tile_ctx_t)
307 :
308 0 : #define STEM_CALLBACK_BEFORE_CREDIT before_credit
309 0 : #define STEM_CALLBACK_DURING_FRAG during_frag
310 0 : #define STEM_CALLBACK_AFTER_FRAG after_frag
311 :
312 : #include "../../disco/stem/fd_stem.c"
313 :
314 : fd_topo_run_tile_t fd_tile_rpcserv = {
315 : .name = "rpcsrv",
316 : .loose_footprint = loose_footprint,
317 : .populate_allowed_seccomp = populate_allowed_seccomp,
318 : .populate_allowed_fds = populate_allowed_fds,
319 : .scratch_align = scratch_align,
320 : .scratch_footprint = scratch_footprint,
321 : .unprivileged_init = unprivileged_init,
322 : .privileged_init = privileged_init,
323 : .run = stem_run,
324 : };
|
