Line data Source code
1 : /* Repair tile runs the repair protocol for a Firedancer node. */
2 : #define _GNU_SOURCE
3 :
4 : #include "../../disco/topo/fd_topo.h"
5 : #include <sys/socket.h>
6 : #include "generated/fd_rpcserv_tile_seccomp.h"
7 :
8 : #include "../rpcserver/fd_rpc_service.h"
9 :
10 : #include "../../disco/tiles.h"
11 : #include "../../flamenco/runtime/fd_blockstore.h"
12 : #include "../../flamenco/fd_flamenco.h"
13 : #include "../../util/fd_util.h"
14 : #include "../../disco/fd_disco.h"
15 : #include "../../disco/shred/fd_stake_ci.h"
16 : #include "../../disco/topo/fd_pod_format.h"
17 : #include "../../funk/fd_funk_filemap.h"
18 : #include "../../disco/keyguard/fd_keyload.h"
19 :
20 : #include <errno.h>
21 : #include <fcntl.h>
22 : #include <unistd.h>
23 :
24 0 : #define REPLAY_NOTIF_IDX 0
25 0 : #define STAKE_CI_IN_IDX 1
26 :
27 : struct fd_rpcserv_tile_ctx {
28 : fd_rpcserver_args_t args;
29 : char funk_file[ PATH_MAX ];
30 :
31 : int activated;
32 :
33 : fd_rpc_ctx_t * ctx;
34 :
35 : fd_pubkey_t identity_key[1]; /* Just the public key */
36 :
37 : fd_wksp_t * replay_notif_in_mem;
38 : ulong replay_notif_in_chunk0;
39 : ulong replay_notif_in_wmark;
40 : fd_replay_notif_msg_t replay_notif_in_state;
41 :
42 : fd_wksp_t * stake_ci_in_mem;
43 : ulong stake_ci_in_chunk0;
44 : ulong stake_ci_in_wmark;
45 :
46 : int blockstore_fd;
47 : };
48 : typedef struct fd_rpcserv_tile_ctx fd_rpcserv_tile_ctx_t;
49 :
50 0 : #define FD_RPC_SCRATCH_MAX (1LU<<30)
51 :
52 : const fd_http_server_params_t RPCSERV_HTTP_PARAMS = {
53 : .max_connection_cnt = 10,
54 : .max_ws_connection_cnt = 10,
55 : .max_request_len = 1<<16,
56 : .max_ws_recv_frame_len = 1<<16,
57 : .max_ws_send_frame_cnt = 10,
58 : .outgoing_buffer_sz = 100<<20,
59 : };
60 :
61 : FD_FN_CONST static inline ulong
62 0 : scratch_align( void ) {
63 0 : return 128UL;
64 0 : }
65 :
66 : FD_FN_PURE static inline ulong
67 0 : scratch_footprint( fd_topo_tile_t const * tile FD_PARAM_UNUSED) {
68 0 : ulong l = FD_LAYOUT_INIT;
69 0 : l = FD_LAYOUT_APPEND( l, alignof(fd_rpcserv_tile_ctx_t), sizeof(fd_rpcserv_tile_ctx_t) );
70 0 : l = FD_LAYOUT_APPEND( l, fd_stake_ci_align(), fd_stake_ci_footprint() );
71 0 : l = FD_LAYOUT_APPEND( l, fd_spad_align(), fd_spad_footprint( FD_RPC_SCRATCH_MAX ) );
72 0 : return FD_LAYOUT_FINI( l, scratch_align() );
73 0 : }
74 :
75 : FD_FN_PURE static inline ulong
76 0 : loose_footprint( fd_topo_tile_t const * tile FD_PARAM_UNUSED ) {
77 0 : return 1UL * FD_SHMEM_GIGANTIC_PAGE_SZ;
78 0 : }
79 :
80 : static inline void
81 : before_credit( fd_rpcserv_tile_ctx_t * ctx,
82 : fd_stem_context_t * stem,
83 0 : int * charge_busy ) {
84 0 : (void)stem;
85 :
86 0 : if( FD_UNLIKELY( !ctx->activated ) ) {
87 0 : *charge_busy = 0;
88 0 : } else {
89 0 : *charge_busy = fd_rpc_ws_poll( ctx->ctx );
90 0 : }
91 0 : }
92 :
93 : static void
94 : during_frag( fd_rpcserv_tile_ctx_t * ctx,
95 : ulong in_idx,
96 : ulong seq FD_PARAM_UNUSED,
97 : ulong sig FD_PARAM_UNUSED,
98 : ulong chunk,
99 : ulong sz,
100 0 : ulong ctl FD_PARAM_UNUSED ) {
101 :
102 0 : if( FD_UNLIKELY( in_idx==REPLAY_NOTIF_IDX ) ) {
103 0 : if( FD_UNLIKELY( chunk<ctx->replay_notif_in_chunk0 || chunk>ctx->replay_notif_in_wmark ) ) {
104 0 : FD_LOG_ERR(( "chunk %lu %lu corrupt, not in range [%lu,%lu]", chunk, sz,
105 0 : ctx->replay_notif_in_chunk0, ctx->replay_notif_in_wmark ));
106 0 : }
107 0 : fd_rpc_replay_during_frag( ctx->ctx, &ctx->replay_notif_in_state, fd_chunk_to_laddr_const( ctx->replay_notif_in_mem, chunk ), (int)sz );
108 :
109 0 : } else if( FD_UNLIKELY( in_idx==STAKE_CI_IN_IDX ) ) {
110 0 : if( FD_UNLIKELY( chunk<ctx->stake_ci_in_chunk0 || chunk>ctx->stake_ci_in_wmark ) ) {
111 0 : FD_LOG_ERR(( "chunk %lu %lu corrupt, not in range [%lu,%lu]", chunk, sz,
112 0 : ctx->stake_ci_in_chunk0, ctx->stake_ci_in_wmark ));
113 0 : }
114 0 : fd_rpc_stake_during_frag( ctx->ctx, ctx->args.stake_ci, fd_chunk_to_laddr_const( ctx->stake_ci_in_mem, chunk ), (int)sz );
115 :
116 0 : } else {
117 0 : FD_LOG_ERR(("Unknown in_idx %lu for rpc", in_idx));
118 0 : }
119 0 : }
120 :
121 : static void
122 : after_frag( fd_rpcserv_tile_ctx_t * ctx,
123 : ulong in_idx,
124 : ulong seq,
125 : ulong sig,
126 : ulong sz,
127 : ulong tsorig,
128 : ulong tspub,
129 0 : fd_stem_context_t * stem ) {
130 0 : (void)seq;
131 0 : (void)sig;
132 0 : (void)sz;
133 0 : (void)tsorig;
134 0 : (void)tspub;
135 0 : (void)stem;
136 :
137 0 : if( FD_LIKELY( in_idx==REPLAY_NOTIF_IDX ) ) {
138 0 : if( FD_UNLIKELY( !ctx->activated ) ) {
139 0 : fd_rpcserver_args_t * args = &ctx->args;
140 0 : fd_funk_t * funk = fd_funk_open_file(
141 0 : args->funk, ctx->funk_file, 1, 0, 0, 0, 0, FD_FUNK_READ_WRITE, NULL );
142 0 : if( FD_UNLIKELY( !funk ) ) {
143 0 : FD_LOG_ERR(( "failed to join a funky" ));
144 0 : }
145 :
146 0 : ctx->activated = 1;
147 0 : fd_rpc_start_service( args, ctx->ctx );
148 0 : }
149 :
150 0 : fd_rpc_replay_after_frag( ctx->ctx, &ctx->replay_notif_in_state );
151 :
152 0 : } else if( FD_UNLIKELY( in_idx==STAKE_CI_IN_IDX ) ) {
153 0 : fd_rpc_stake_after_frag( ctx->ctx, ctx->args.stake_ci );
154 :
155 0 : } else {
156 0 : FD_LOG_ERR(("Unknown in_idx %lu for rpc", in_idx));
157 0 : }
158 0 : }
159 :
160 : static void
161 : privileged_init( fd_topo_t * topo,
162 0 : fd_topo_tile_t * tile ) {
163 0 : void * scratch = fd_topo_obj_laddr( topo, tile->tile_obj_id );
164 :
165 0 : FD_SCRATCH_ALLOC_INIT( l, scratch );
166 0 : fd_rpcserv_tile_ctx_t * ctx = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_rpcserv_tile_ctx_t), sizeof(fd_rpcserv_tile_ctx_t) );
167 0 : void * stake_ci_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_stake_ci_align(), fd_stake_ci_footprint() );
168 0 : void * spad_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_spad_align(), fd_spad_footprint( FD_RPC_SCRATCH_MAX ) );
169 0 : FD_SCRATCH_ALLOC_FINI( l, scratch_align() );
170 :
171 0 : if( FD_UNLIKELY( !strcmp( tile->rpcserv.identity_key_path, "" ) ) )
172 0 : FD_LOG_ERR( ( "identity_key_path not set" ) );
173 0 : ctx->identity_key[0] = *(fd_pubkey_t const *) fd_type_pun_const( fd_keyload_load( tile->rpcserv.identity_key_path, /* pubkey only: */ 1 ) );
174 :
175 0 : fd_rpcserver_args_t * args = &ctx->args;
176 0 : fd_memset( args, 0, sizeof(fd_rpcserver_args_t) );
177 :
178 0 : args->offline = 0;
179 0 : args->params = RPCSERV_HTTP_PARAMS;
180 :
181 0 : args->port = tile->rpcserv.rpc_port;
182 :
183 0 : args->tpu_addr.sin_family = AF_INET;
184 0 : args->tpu_addr.sin_addr.s_addr = tile->rpcserv.tpu_ip_addr;
185 0 : args->tpu_addr.sin_port = htons( (ushort)tile->rpcserv.tpu_port );
186 :
187 0 : args->stake_ci = fd_stake_ci_join( fd_stake_ci_new( stake_ci_mem, ctx->identity_key ) );
188 :
189 0 : uchar * spad_mem_cur = spad_mem;
190 0 : args->spad = fd_spad_join( fd_spad_new( spad_mem_cur, FD_RPC_SCRATCH_MAX ) );
191 :
192 0 : strncpy( ctx->funk_file, tile->replay.funk_file, sizeof(ctx->funk_file) );
193 : /* Open funk after replay tile is booted */
194 :
195 : /* Blockstore setup */
196 0 : ulong blockstore_obj_id = fd_pod_queryf_ulong( topo->props, ULONG_MAX, "blockstore" );
197 0 : FD_TEST( blockstore_obj_id!=ULONG_MAX );
198 0 : args->blockstore = fd_blockstore_join( &args->blockstore_ljoin, fd_topo_obj_laddr( topo, blockstore_obj_id ) );
199 0 : FD_TEST( args->blockstore!=NULL );
200 0 : ctx->blockstore_fd = open( tile->replay.blockstore_file, O_RDONLY );
201 0 : if ( FD_UNLIKELY(ctx->blockstore_fd == -1) ){
202 0 : FD_LOG_WARNING(("%s: %s", tile->replay.blockstore_file, strerror( errno )));
203 0 : }
204 :
205 0 : args->blockstore_fd = ctx->blockstore_fd;
206 :
207 0 : fd_spad_push( args->spad ); /* We close this out when we stop the server */
208 0 : fd_rpc_create_ctx( args, &ctx->ctx );
209 :
210 :
211 : /* Wait until after replay tile boots before starting service */
212 0 : }
213 :
214 : static void
215 : unprivileged_init( fd_topo_t * topo,
216 0 : fd_topo_tile_t * tile ) {
217 0 : void * scratch = fd_topo_obj_laddr( topo, tile->tile_obj_id );
218 :
219 0 : if( FD_UNLIKELY( tile->in_cnt != 2 ||
220 0 : strcmp( topo->links[ tile->in_link_id[ REPLAY_NOTIF_IDX ] ].name, "replay_notif") ||
221 0 : strcmp( topo->links[ tile->in_link_id[ STAKE_CI_IN_IDX ] ].name, "stake_out" ) ) ) {
222 0 : FD_LOG_ERR(( "repair tile has none or unexpected input links %lu %s %s",
223 0 : tile->in_cnt, topo->links[ tile->in_link_id[ 0 ] ].name, topo->links[ tile->in_link_id[ 1 ] ].name ));
224 0 : }
225 :
226 0 : if( FD_UNLIKELY( tile->out_cnt != 0 ) ) {
227 0 : FD_LOG_ERR(( "repair tile has none or unexpected output links %lu %s %s",
228 0 : tile->out_cnt, topo->links[ tile->out_link_id[ 0 ] ].name, topo->links[ tile->out_link_id[ 1 ] ].name ));
229 0 : }
230 :
231 : /* Scratch mem setup */
232 :
233 0 : FD_SCRATCH_ALLOC_INIT( l, scratch );
234 0 : fd_rpcserv_tile_ctx_t * ctx = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_rpcserv_tile_ctx_t), sizeof(fd_rpcserv_tile_ctx_t) );
235 0 : FD_SCRATCH_ALLOC_APPEND( l, fd_stake_ci_align(), fd_stake_ci_footprint() );
236 0 : FD_SCRATCH_ALLOC_APPEND( l, fd_spad_align(), fd_ulong_align_up( FD_RPC_SCRATCH_MAX, fd_spad_align() ) );
237 0 : ulong scratch_top = FD_SCRATCH_ALLOC_FINI( l, scratch_align() );
238 0 : if( FD_UNLIKELY( scratch_top > (ulong)scratch + scratch_footprint( tile ) ) )
239 0 : FD_LOG_ERR(( "scratch overflow %lu %lu %lu", scratch_top - (ulong)scratch - scratch_footprint( tile ), scratch_top, (ulong)scratch + scratch_footprint( tile ) ));
240 :
241 0 : ctx->activated = 0;
242 :
243 0 : fd_topo_link_t * replay_notif_in_link = &topo->links[ tile->in_link_id[ REPLAY_NOTIF_IDX ] ];
244 0 : ctx->replay_notif_in_mem = topo->workspaces[ topo->objs[ replay_notif_in_link->dcache_obj_id ].wksp_id ].wksp;
245 0 : ctx->replay_notif_in_chunk0 = fd_dcache_compact_chunk0( ctx->replay_notif_in_mem, replay_notif_in_link->dcache );
246 0 : ctx->replay_notif_in_wmark = fd_dcache_compact_wmark ( ctx->replay_notif_in_mem, replay_notif_in_link->dcache, replay_notif_in_link->mtu );
247 :
248 0 : fd_topo_link_t * stake_ci_in_link = &topo->links[ tile->in_link_id[ STAKE_CI_IN_IDX ] ];
249 0 : ctx->stake_ci_in_mem = topo->workspaces[ topo->objs[ stake_ci_in_link->dcache_obj_id ].wksp_id ].wksp;
250 0 : ctx->stake_ci_in_chunk0 = fd_dcache_compact_chunk0( ctx->stake_ci_in_mem, stake_ci_in_link->dcache );
251 0 : ctx->stake_ci_in_wmark = fd_dcache_compact_wmark ( ctx->stake_ci_in_mem, stake_ci_in_link->dcache, stake_ci_in_link->mtu );
252 0 : }
253 :
254 : static ulong
255 : populate_allowed_seccomp( fd_topo_t const * topo,
256 : fd_topo_tile_t const * tile,
257 : ulong out_cnt,
258 0 : struct sock_filter * out ) {
259 0 : void * scratch = fd_topo_obj_laddr( topo, tile->tile_obj_id );
260 0 : FD_SCRATCH_ALLOC_INIT( l, scratch );
261 0 : fd_rpcserv_tile_ctx_t * ctx = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_rpcserv_tile_ctx_t), sizeof(fd_rpcserv_tile_ctx_t) );
262 :
263 0 : populate_sock_filter_policy_fd_rpcserv_tile( out_cnt, out, (uint)fd_log_private_logfile_fd(), (uint)fd_rpc_ws_fd( ctx->ctx ), (uint)ctx->blockstore_fd );
264 0 : return sock_filter_policy_fd_rpcserv_tile_instr_cnt;
265 0 : }
266 :
267 : static ulong
268 : populate_allowed_fds( fd_topo_t const * topo,
269 : fd_topo_tile_t const * tile,
270 : ulong out_fds_cnt,
271 0 : int * out_fds ) {
272 0 : void * scratch = fd_topo_obj_laddr( topo, tile->tile_obj_id );
273 0 : FD_SCRATCH_ALLOC_INIT( l, scratch );
274 0 : fd_rpcserv_tile_ctx_t * ctx = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_rpcserv_tile_ctx_t), sizeof(fd_rpcserv_tile_ctx_t) );
275 :
276 0 : if( FD_UNLIKELY( out_fds_cnt<3UL ) ) FD_LOG_ERR(( "out_fds_cnt %lu", out_fds_cnt ));
277 :
278 0 : ulong out_cnt = 0;
279 0 : out_fds[ out_cnt++ ] = 2; /* stderr */
280 0 : if( FD_LIKELY( -1!=fd_log_private_logfile_fd() ) )
281 0 : out_fds[ out_cnt++ ] = fd_log_private_logfile_fd(); /* logfile */
282 0 : out_fds[ out_cnt++ ] = fd_rpc_ws_fd( ctx->ctx ); /* listen socket */
283 0 : out_fds[ out_cnt++ ] = ctx->blockstore_fd;
284 0 : return out_cnt;
285 0 : }
286 :
287 : /* TODO: This is probably not correct. */
288 0 : #define STEM_BURST (1UL)
289 :
290 0 : #define STEM_CALLBACK_CONTEXT_TYPE fd_rpcserv_tile_ctx_t
291 0 : #define STEM_CALLBACK_CONTEXT_ALIGN alignof(fd_rpcserv_tile_ctx_t)
292 :
293 0 : #define STEM_CALLBACK_BEFORE_CREDIT before_credit
294 0 : #define STEM_CALLBACK_DURING_FRAG during_frag
295 0 : #define STEM_CALLBACK_AFTER_FRAG after_frag
296 :
297 : #include "../../disco/stem/fd_stem.c"
298 :
299 : fd_topo_run_tile_t fd_tile_rpcserv = {
300 : .name = "rpcsrv",
301 : .loose_footprint = loose_footprint,
302 : .populate_allowed_seccomp = populate_allowed_seccomp,
303 : .populate_allowed_fds = populate_allowed_fds,
304 : .scratch_align = scratch_align,
305 : .scratch_footprint = scratch_footprint,
306 : .unprivileged_init = unprivileged_init,
307 : .privileged_init = privileged_init,
308 : .run = stem_run,
309 : };
|
