Line data Source code
1 : /* The failover controller: the state machine the failov tile drives
2 : from its stem callbacks. ctx->action holds the state, one of
3 : FD_FAILOVER_ACTION_*, and set_action is its only writer.
4 :
5 : Two machines share one identity. The holder gives it up through the
6 : demotion flow and the taker adopts it through the handoff request and
7 : promotion flows.
8 :
9 : Holder, giving up the identity Taker, adopting it
10 :
11 : IDLE IDLE
12 : | accept_handoff | run_handoff_cmd
13 : DEMOTE_SWITCH HANDOFF_WAIT_PEER
14 : | the admin tile installs the junk | poll_peer dials and sends
15 : | key, step_demote_switch | HANDOFF_REQUEST, on_demoted_msg
16 : DEMOTE_DRAIN | takes the final state
17 : | the votor's last votes arrive, PROMOTE_WAIT_REPLAY
18 : | step_demote_drain | replay reaches the anchor,
19 : DEMOTE_WAIT_ACK | step_promote_wait_replay
20 : | DEMOTED sent, the peer answers, PROMOTE_WAIT_ADOPT
21 : | on_demoted_answer_msg | the votor adopts the history,
22 : IDLE | step_promote_wait_adopt
23 : PROMOTE_SWITCH
24 : | the admin tile installs the
25 : | identity, step_promote_switch
26 : HANDOFF_WAIT_RESULT
27 : | the peer records our answer,
28 : | on_handoff_result_msg
29 : IDLE
30 :
31 : This is not a translation unit. Keep it free of stem callbacks and
32 : the topology or seccomp, which stay in the tile. */
33 :
34 : /* The state */
35 :
36 : /* FSM state list */
37 : static char const * const action_names[ FD_FAILOVER_ACTION_CNT ] = {
38 : [ FD_FAILOVER_ACTION_IDLE ] = "idle",
39 : [ FD_FAILOVER_ACTION_DEMOTE_SWITCH ] = "demote_switch",
40 : [ FD_FAILOVER_ACTION_DEMOTE_DRAIN ] = "demote_drain",
41 : [ FD_FAILOVER_ACTION_DEMOTE_WAIT_ACK ] = "demote_wait_ack",
42 : [ FD_FAILOVER_ACTION_HANDOFF_WAIT_PEER ] = "handoff_wait_peer",
43 : [ FD_FAILOVER_ACTION_PROMOTE_WAIT_REPLAY ] = "promote_wait_replay",
44 : [ FD_FAILOVER_ACTION_PROMOTE_WAIT_ADOPT ] = "promote_wait_adopt",
45 : [ FD_FAILOVER_ACTION_PROMOTE_SWITCH ] = "promote_switch",
46 : [ FD_FAILOVER_ACTION_HANDOFF_WAIT_RESULT ] = "handoff_wait_result",
47 : };
48 :
49 : static char const *
50 693 : get_action_name( ulong action ) {
51 693 : return action<FD_FAILOVER_ACTION_CNT ? action_names[ action ] : "unknown";
52 693 : }
53 :
54 : /* Phase helpers */
55 : static inline int
56 141 : is_requesting( fd_failover_tile_ctx_t const * ctx ) {
57 141 : return ctx->action==FD_FAILOVER_ACTION_HANDOFF_WAIT_PEER || ctx->action==FD_FAILOVER_ACTION_HANDOFF_WAIT_RESULT;
58 141 : }
59 :
60 : static inline int
61 30 : is_promoting( fd_failover_tile_ctx_t const * ctx ) {
62 30 : return ctx->action==FD_FAILOVER_ACTION_PROMOTE_WAIT_REPLAY || ctx->action==FD_FAILOVER_ACTION_PROMOTE_WAIT_ADOPT ||
63 30 : ctx->action==FD_FAILOVER_ACTION_PROMOTE_SWITCH;
64 30 : }
65 :
66 : static inline int
67 30 : is_switching_out( fd_failover_tile_ctx_t const * ctx ) {
68 30 : return ctx->action==FD_FAILOVER_ACTION_DEMOTE_SWITCH || ctx->action==FD_FAILOVER_ACTION_DEMOTE_DRAIN;
69 30 : }
70 :
71 : static inline int
72 495 : is_switch_pending( fd_failover_tile_ctx_t const * ctx ) {
73 495 : return ctx->id_switch.pending_key!=FD_FAILOVER_SWITCH_KEY_CNT;
74 495 : }
75 :
76 : /* Change the controller state: the only thing that should ever write to ctx->action! */
77 : static void
78 : set_action( fd_failover_tile_ctx_t * ctx,
79 324 : ulong action ) {
80 324 : FD_LOG_INFO(( "action %s -> %s", get_action_name( ctx->action ), get_action_name( action ) ));
81 324 : if( FD_LIKELY( action!=ctx->action ) ) {
82 309 : ctx->deadline_slot = ctx->replay_slot==FD_FAILOVER_SLOT_NULL ? FD_FAILOVER_SLOT_NULL : fd_ulong_sat_add( ctx->replay_slot, FD_FAILOVER_DEADLINE_SLOTS );
83 309 : ctx->deadline_at = fd_long_sat_add( fd_failover_clock(), FD_FAILOVER_DEADLINE_NANOS );
84 309 : }
85 324 : ctx->action = action;
86 324 : }
87 :
88 : /* The current state passed its slot deadline or its clock deadline. */
89 : static int
90 57 : is_overdue( fd_failover_tile_ctx_t const * ctx ) {
91 57 : int slots = ctx->deadline_slot!=FD_FAILOVER_SLOT_NULL && ctx->replay_slot!=FD_FAILOVER_SLOT_NULL && ctx->replay_slot>ctx->deadline_slot;
92 57 : return slots || fd_failover_clock()>=ctx->deadline_at;
93 57 : }
94 :
95 : /* Returns the controller to IDLE with nothing in flight. */
96 : static void
97 21 : reset_controller( fd_failover_tile_ctx_t * ctx ) {
98 21 : fd_memset( &ctx->request, 0, sizeof(ctx->request) );
99 21 : fd_memset( &ctx->handoff, 0, sizeof(ctx->handoff) );
100 21 : fd_memset( &ctx->promotion, 0, sizeof(ctx->promotion) );
101 21 : fd_memset( &ctx->reply, 0, sizeof(ctx->reply) );
102 21 : fd_memset( &ctx->tx, 0, sizeof(ctx->tx) );
103 21 : ctx->cmd_addr = 0U;
104 21 : ctx->cmd_port = (ushort)0;
105 21 : ctx->adopt_result_fresh = 0;
106 21 : ctx->session.close_after_send = 0;
107 21 : ctx->peer_history.valid = 0;
108 21 : ctx->current_history.valid = 0;
109 21 : ctx->last_vote_slot = FD_FAILOVER_SLOT_NULL;
110 21 : ctx->stuck = 0;
111 :
112 : /* reset to idle */
113 21 : set_action( ctx, FD_FAILOVER_ACTION_IDLE );
114 21 : }
115 :
116 : /* Shared helpers: the session, control queue, and log */
117 :
118 : static inline ushort
119 183 : get_dial_port( fd_failover_tile_ctx_t const * ctx ) {
120 183 : return ctx->cmd_port ? ctx->cmd_port : FD_FAILOVER_DEFAULT_PORT;
121 183 : }
122 :
123 : /* The peer on the current session is the bound one. */
124 : static inline int
125 : is_bound_peer( fd_failover_tile_ctx_t const * ctx,
126 84 : fd_failover_peer_t const * bound ) {
127 84 : return is_same_peer( bound, fd_failover_channel_peer_hello( ctx->channel )->boot_id );
128 84 : }
129 :
130 : static void
131 : finish_cmd( fd_failover_tile_ctx_t * ctx,
132 72 : ulong result ) {
133 72 : if( FD_LIKELY( !ctx->cmd_nonce || ctx->cmd_done ) ) return;
134 24 : if( FD_LIKELY( result==FD_ADMINCTL_RESULT_SUCCESS ) ) FD_LOG_NOTICE (( "the failover is done, this machine runs the identity" ));
135 24 : else FD_LOG_WARNING(( "the failover failed (result %#lx)", result ));
136 24 : ctx->cmd_result = result;
137 24 : ctx->cmd_done = 1;
138 24 : }
139 :
140 : /* Updates our session state when the channel's session changes. */
141 : static void
142 432 : sync_session( fd_failover_tile_ctx_t * ctx ) {
143 432 : ulong state = fd_failover_channel_state( ctx->channel );
144 432 : ulong generation = fd_failover_channel_generation( ctx->channel );
145 432 : if( FD_LIKELY( state==ctx->session.state && generation==ctx->session.generation ) ) return;
146 :
147 : /* new generation */
148 204 : ctx->session.state = state;
149 204 : ctx->session.generation = generation;
150 204 : ctx->request.sent = 0;
151 204 : ctx->session.close_after_send = 0;
152 : /* invalidate the previous session's FD_FAILOVER_MSG_HANDOFF_RESULT */
153 204 : if( ctx->tx.valid && ( ctx->tx.type==(ushort)FD_FAILOVER_MSG_HANDOFF_RESULT ||
154 36 : ( ctx->tx.type==(ushort)FD_FAILOVER_MSG_DEMOTED && ctx->action!=FD_FAILOVER_ACTION_DEMOTE_WAIT_ACK ) ) ) {
155 9 : ctx->tx.valid = 0;
156 9 : }
157 204 : if( FD_LIKELY( state!=FD_FAILOVER_SESSION_PAIRED ) ) return;
158 129 : fd_failover_hello_t const * peer = fd_failover_channel_peer_hello( ctx->channel );
159 129 : ctx->session.peer_boot_id = peer->boot_id;
160 129 : if( ctx->handoff.delivery==FD_FAILOVER_DELIVERY_SENT ) ctx->handoff.delivery = FD_FAILOVER_DELIVERY_PENDING;
161 129 : if( FD_UNLIKELY( peer->holds ) ) {
162 30 : if( is_promoting( ctx ) ) ctx->promotion.holder_seen = 1;
163 30 : if( ctx->peer_history.valid ) FD_LOG_NOTICE(( "peer boot %016lx runs the identity, discarding the saved final state, keeping the coverage floors", peer->boot_id ));
164 30 : ctx->peer_history.valid = 0;
165 30 : }
166 129 : }
167 :
168 : /* Drops the session and stops dialing. */
169 : static void
170 : hang_up( fd_failover_tile_ctx_t * ctx,
171 111 : long now ) {
172 111 : fd_failover_channel_init_dialer( ctx->channel, 0U, 0 );
173 111 : fd_failover_channel_hangup( ctx->channel, now );
174 111 : sync_session( ctx );
175 111 : }
176 :
177 : /* One-message controller outbox (queue_control is the primitive) */
178 :
179 : static int
180 : queue_control( fd_failover_tile_ctx_t * ctx,
181 : ushort type,
182 : fd_failover_peer_t const * to,
183 : void const * payload,
184 84 : ulong payload_sz ) {
185 84 : if( FD_UNLIKELY( ctx->tx.valid || !payload_sz || payload_sz>sizeof(ctx->tx.payload) ) ) return -1;
186 84 : ctx->tx.type = type;
187 84 : ctx->tx.sz = (ushort)payload_sz;
188 84 : ctx->tx.valid = 1;
189 84 : ctx->tx.to = *to;
190 84 : fd_memcpy( ctx->tx.payload, payload, payload_sz );
191 84 : return 0;
192 84 : }
193 :
194 : /* The holder's hand-over message (consensus-critical payload). */
195 : static int
196 39 : queue_demoted( fd_failover_tile_ctx_t * ctx ) {
197 : /* => DEMOTED */
198 39 : return queue_control( ctx, (ushort)FD_FAILOVER_MSG_DEMOTED, &ctx->handoff.target, ctx->handoff.payload, ctx->handoff.payload_sz );
199 39 : }
200 :
201 : /* Tells the requester how its handoff ended and closes the session once
202 : that has drained. */
203 : static void
204 : queue_handoff_result( fd_failover_tile_ctx_t * ctx,
205 : fd_failover_peer_t const * to,
206 : ulong id,
207 39 : ulong result ) {
208 : /* => HANDOFF_RESULT */
209 39 : fd_failover_peer_t peer;
210 39 : if( !to ) {
211 36 : peer.boot_id = fd_failover_channel_peer_hello( ctx->channel )->boot_id;
212 36 : to = &peer;
213 36 : }
214 39 : fd_failover_handoff_result_t response = { .handoff_id=id, .result=result };
215 39 : if( FD_LIKELY( !queue_control( ctx, FD_FAILOVER_MSG_HANDOFF_RESULT, to, &response, sizeof(response) ) ) ) {
216 39 : ctx->session.close_after_send = 1;
217 39 : }
218 39 : }
219 :
220 : /* Queues our last response to a DEMOTED. */
221 : static void
222 6 : queue_reply( fd_failover_tile_ctx_t * ctx ) {
223 : /* => PROMOTED_ACK or PROMOTE_REJECTED */
224 6 : uchar payload[ sizeof(fd_failover_promote_rejected_t) ];
225 6 : ulong payload_sz = ctx->reply.type==(ushort)FD_FAILOVER_MSG_PROMOTE_ACK
226 6 : ? fd_failover_promote_ack_encode( payload, ctx->reply.handoff_id )
227 6 : : fd_failover_promote_rejected_encode( payload, ctx->reply.handoff_id, ctx->reply.reason );
228 6 : ctx->reply.state = queue_control( ctx, ctx->reply.type, &ctx->reply.peer, payload, payload_sz )
229 6 : ? FD_FAILOVER_REPLY_PENDING
230 6 : : FD_FAILOVER_REPLY_KEPT;
231 6 : }
232 :
233 : /* The taker answers the DEMOTED */
234 : static void
235 : finish_reply( fd_failover_tile_ctx_t * ctx,
236 : fd_failover_peer_t const * peer,
237 : ulong handoff_id,
238 : ushort type,
239 6 : uchar reason ) {
240 6 : ctx->reply.peer = *peer;
241 6 : ctx->reply.handoff_id = handoff_id;
242 6 : ctx->reply.type = type;
243 6 : ctx->reply.reason = reason;
244 6 : queue_reply( ctx );
245 6 : }
246 :
247 : static char const *
248 51 : get_handoff_name( ulong result ) {
249 51 : static char const * const names[ FD_FAILOVER_HANDOFF_CNT ] = {
250 51 : "none", "pending", "taken", "declined", "restarted", "cancelled", "not holder"
251 51 : };
252 51 : return result<FD_FAILOVER_HANDOFF_CNT ? names[ result ] : "unknown";
253 51 : }
254 :
255 : static char const *
256 0 : get_msg_name( ushort type ) {
257 0 : switch( type ) {
258 0 : case (ushort)FD_FAILOVER_MSG_DEMOTED: return "DEMOTED";
259 0 : case (ushort)FD_FAILOVER_MSG_PROMOTE_ACK: return "PROMOTE_ACK";
260 0 : case (ushort)FD_FAILOVER_MSG_PROMOTE_REJECTED: return "PROMOTE_REJECTED";
261 0 : case (ushort)FD_FAILOVER_MSG_HANDOFF_RESULT: return "HANDOFF_RESULT";
262 0 : default: return "unknown";
263 0 : }
264 0 : }
265 :
266 : /* Sends the queued control message once the session can take it. */
267 : static void
268 : flush_pending( fd_failover_tile_ctx_t * ctx,
269 0 : long now ) {
270 0 : if( FD_LIKELY( !ctx->tx.valid ) ) return;
271 0 : if( FD_UNLIKELY( fd_failover_channel_state( ctx->channel )!=FD_FAILOVER_SESSION_PAIRED || fd_failover_channel_tx_pending( ctx->channel ) ) ) return;
272 0 : if( FD_UNLIKELY( !is_bound_peer( ctx, &ctx->tx.to ) ) ) return;
273 0 : if( FD_UNLIKELY( fd_failover_channel_send( ctx->channel, now, ctx->tx.type, ctx->tx.payload, ctx->tx.sz ) ) ) return;
274 0 : if( ctx->tx.type==(ushort)FD_FAILOVER_MSG_DEMOTED && ctx->handoff.delivery==FD_FAILOVER_DELIVERY_PENDING ) ctx->handoff.delivery = FD_FAILOVER_DELIVERY_SENT;
275 0 : if( ctx->tx.type==(ushort)FD_FAILOVER_MSG_HANDOFF_RESULT ) {
276 0 : fd_failover_handoff_result_t result;
277 0 : fd_memcpy( &result, ctx->tx.payload, sizeof(result) );
278 0 : FD_LOG_NOTICE(( "handoff %lu: sent HANDOFF_RESULT to peer boot %016lx (result %#lx)", result.handoff_id, ctx->tx.to.boot_id, result.result ));
279 0 : } else {
280 0 : ulong id = ctx->tx.type==(ushort)FD_FAILOVER_MSG_DEMOTED ? ctx->handoff.id : ctx->reply.handoff_id;
281 0 : FD_LOG_NOTICE(( "handoff %lu: sent %s to peer boot %016lx", id, get_msg_name( ctx->tx.type ), ctx->tx.to.boot_id ));
282 0 : }
283 0 : ctx->tx.valid = 0;
284 0 : }
285 :
286 : static char const *
287 12 : get_reject_name( uint reason ) {
288 12 : static char const * const names[ FD_FAILOVER_REJECT_CNT ] = {
289 12 : "none", "busy", "holds identity", "replay behind", "adoption mismatch", "adoption failed", "switch failed"
290 12 : };
291 12 : return reason<FD_FAILOVER_REJECT_CNT ? names[ reason ] : "unknown";
292 12 : }
293 :
294 : static char const *
295 12 : get_adopt_err_name( ulong result ) {
296 12 : switch( result ) {
297 0 : case FD_VOTOR_ADOPT_ERR_DECODE: return "decode error";
298 0 : case FD_VOTOR_ADOPT_ERR_INVALID: return "invalid";
299 0 : case FD_VOTOR_ADOPT_ERR_UNREPLAYED_ROOT: return "finality anchor not replayed";
300 6 : case FD_VOTOR_ADOPT_ERR_UNREPLAYED: return "consensus not initialized";
301 0 : case FD_VOTOR_ADOPT_ERR_STALE: return "older than the votes this machine sent as this identity";
302 3 : case FD_VOTOR_ADOPT_ERR_CAPACITY: return "insufficient slot state capacity";
303 3 : default: return "unknown";
304 12 : }
305 12 : }
306 :
307 : static char const *
308 138 : get_source_name( ulong source ) {
309 138 : return source==FD_FAILOVER_SOURCE_PEER ? "the vote history the peer gave us" : "an empty history";
310 138 : }
311 :
312 : /* Identity switching: the admin tile installs the junk key or the identity */
313 :
314 : /* Sets the installed identity this machine currently signs with. */
315 : static void
316 : set_installed( fd_failover_tile_ctx_t * ctx,
317 57 : uchar const * pubkey ) {
318 57 : fd_memcpy( ctx->installed, pubkey, 32UL );
319 57 : ctx->hello.holds = (uchar)holds_identity( ctx );
320 57 : fd_failover_channel_set_holds( ctx->channel, ctx->hello.holds );
321 57 : }
322 :
323 : /* Asks the admin tile to switch to the given key.
324 : Returns the id, or ULONG_MAX if nothing was sent. */
325 : static ulong
326 : request_switch( fd_failover_tile_ctx_t * ctx,
327 : fd_stem_context_t * stem,
328 45 : ulong key ) {
329 45 : if( FD_UNLIKELY( ctx->admin_out_idx==ULONG_MAX || is_switch_pending( ctx ) ) ) return ULONG_MAX;
330 45 : if( FD_UNLIKELY( !++ctx->id_switch.request_id ) ) ctx->id_switch.request_id++;
331 45 : fd_failover_bus_msg_t * out = fd_chunk_to_laddr( ctx->admin_out_mem, ctx->admin_out_chunk );
332 45 : fd_memset( out, 0, sizeof(*out) );
333 45 : out->nonce = ctx->id_switch.request_id;
334 45 : fd_failover_switch_req_t req = { .epoch=ctx->id_switch.epoch };
335 45 : fd_memcpy( req.identity, key==FD_FAILOVER_SWITCH_KEY_STAKED ? ctx->hello.staked_pubkey : ctx->spare_pubkey, 32UL );
336 45 : fd_memcpy( out->payload, &req, sizeof(req) );
337 45 : ulong tspub = fd_frag_meta_ts_comp( fd_tickcount() );
338 45 : fd_stem_publish( stem, ctx->admin_out_idx, FD_FAILOVER_BUS_SWITCH_REQ, ctx->admin_out_chunk, sizeof(*out), 0UL, tspub, tspub );
339 45 : ctx->admin_out_chunk = fd_dcache_compact_next( ctx->admin_out_chunk, sizeof(*out), ctx->admin_out_chunk0, ctx->admin_out_wmark );
340 45 : ctx->id_switch.pending_key = key;
341 45 : ctx->id_switch.fresh = 0;
342 45 : return ctx->id_switch.request_id;
343 45 : }
344 :
345 : /* Ends the switch in flight and returns 1 if the caller should take the result. */
346 : static int
347 : on_switch_resp_msg( fd_failover_tile_ctx_t * ctx,
348 45 : ulong nonce ) {
349 45 : if( FD_UNLIKELY( !is_switch_pending( ctx ) || nonce!=ctx->id_switch.request_id ) ) {
350 0 : FD_LOG_WARNING(( "dropping a stale identity switch answer" ));
351 0 : return 0;
352 0 : }
353 45 : ctx->id_switch.pending_key = FD_FAILOVER_SWITCH_KEY_CNT;
354 45 : if( FD_UNLIKELY( ctx->id_switch.discard ) ) {
355 3 : ctx->id_switch.discard = 0;
356 3 : return 0;
357 3 : }
358 42 : ctx->id_switch.fresh = 1;
359 42 : return 1;
360 45 : }
361 :
362 : static ulong
363 39 : take_switch_result( fd_failover_tile_ctx_t * ctx ) {
364 39 : ctx->id_switch.fresh = 0;
365 39 : ctx->id_switch.overdue = 0;
366 39 : return ctx->id_switch.result.result;
367 39 : }
368 :
369 : /* The identity switch in flight passed its deadline. */
370 : static void
371 0 : mark_switch_overdue( fd_failover_tile_ctx_t * ctx ) {
372 0 : ctx->stuck = 1;
373 0 : if( FD_LIKELY( ctx->id_switch.overdue ) ) return;
374 0 : ctx->id_switch.overdue = 1;
375 0 : FD_LOG_WARNING(( "identity switch %lu to %s is overdue, its outcome is unknown, we keep waiting and refuse another switch",
376 0 : ctx->id_switch.request_id, ctx->id_switch.pending_key==FD_FAILOVER_SWITCH_KEY_STAKED ? "the identity" : "the junk key" ));
377 0 : }
378 :
379 : /* Inputs: the vote history, gossip and the operator */
380 :
381 : static void
382 : prepare_history( fd_failover_tile_ctx_t * ctx,
383 6 : fd_votor_hist_msg_t const * hist ) {
384 6 : uchar state[ FD_FAILOVER_ALPENGLOW_STATE_MAX ];
385 6 : ulong state_sz = 0UL;
386 6 : if( FD_UNLIKELY( fd_votor_hist_ser( &hist->hist, state, FD_FAILOVER_ALPENGLOW_STATE_MAX, &state_sz ) || fd_votor_hist_tip( &hist->hist )!=hist->vote_slot ) ) {
387 3 : if( FD_UNLIKELY( !ctx->hist_warned ) ) FD_LOG_WARNING(( "votor produced a vote history that does not match its slot metadata" ));
388 3 : ctx->hist_warned = 1;
389 3 : return;
390 3 : }
391 3 : ctx->hist_warned = 0;
392 3 : fd_memcpy( ctx->current_history.state, state, state_sz );
393 3 : ctx->current_history.valid = 1;
394 3 : ctx->current_history.tip = hist->vote_slot;
395 3 : ctx->current_history.sz = state_sz;
396 3 : }
397 :
398 : static void
399 : on_votor_hist_msg( fd_failover_tile_ctx_t * ctx,
400 15 : fd_votor_hist_msg_t const * hist ) {
401 15 : if( FD_LIKELY( hist->replay_slot!=FD_FAILOVER_SLOT_NULL && ( ctx->replay_slot==FD_FAILOVER_SLOT_NULL || hist->replay_slot>ctx->replay_slot ) ) ) ctx->replay_slot = hist->replay_slot;
402 15 : if( FD_LIKELY( hist->root_slot!=FD_FAILOVER_SLOT_NULL ) ) ctx->root_slot = hist->root_slot;
403 15 : ctx->installed_ranked = hist->ranked;
404 15 : if( FD_UNLIKELY( !fd_memeq( hist->identity.uc, ctx->hello.staked_pubkey, 32UL ) || hist->vote_slot==FD_FAILOVER_SLOT_NULL ) ) return;
405 6 : int producing = holds_identity( ctx ) || ctx->action==FD_FAILOVER_ACTION_PROMOTE_SWITCH || ctx->action==FD_FAILOVER_ACTION_DEMOTE_DRAIN;
406 6 : int voted = hist->has_vote || ( producing && ctx->last_vote_slot==FD_FAILOVER_SLOT_NULL );
407 6 : int changed = producing && ctx->current_history.valid && hist->vote_slot>=ctx->last_vote_slot;
408 6 : if( FD_LIKELY( !voted && !changed ) ) return;
409 6 : ctx->last_vote_slot = hist->vote_slot;
410 6 : if( voted && ( ctx->own_floor==FD_FAILOVER_SLOT_NULL || hist->vote_slot>ctx->own_floor ) ) ctx->own_floor = hist->vote_slot;
411 6 : if( hist->has_vote ) ctx->own_voted = 1;
412 6 : prepare_history( ctx, hist );
413 6 : }
414 :
415 : /* Logs a step of a command at NOTICE. */
416 156 : #define LOG_CMD_STEP( ctx, ... ) do { \
417 156 : FD_LOG_NOTICE(( __VA_ARGS__ )); \
418 264 : fd_cstr_printf( (ctx)->cmd_step[ (ctx)->cmd_step_head%FD_FAILOVER_STEP_CNT ], FD_FAILOVER_BUS_STEP_MAX, NULL, __VA_ARGS__ ); \
419 156 : (ctx)->cmd_step_head++; \
420 156 : if( (ctx)->cmd_step_head-(ctx)->cmd_step_tail>FD_FAILOVER_STEP_CNT ) (ctx)->cmd_step_tail++; \
421 156 : } while(0)
422 :
423 : static void
424 : on_gossip_contact_info_msg( fd_failover_tile_ctx_t * ctx,
425 3 : long now ) {
426 3 : uint addr = ctx->gossip_socket.addr;
427 3 : if( FD_UNLIKELY( !addr || !fd_memeq( ctx->gossip_origin, ctx->hello.staked_pubkey, 32UL ) ) ) return;
428 3 : if( FD_UNLIKELY( addr==ctx->own_gossip.addr && ctx->gossip_socket.port==ctx->own_gossip.port ) ) return;
429 3 : ctx->staked_seen_at = now;
430 3 : if( FD_UNLIKELY( ctx->peer_history.valid ) ) {
431 0 : FD_LOG_NOTICE(( "gossip shows the identity at `" FD_IP4_ADDR_FMT "`, discarding the saved final state, keeping the coverage floors", FD_IP4_ADDR_FMT_ARGS( addr ) ));
432 0 : ctx->peer_history.valid = 0;
433 0 : }
434 3 : if( FD_LIKELY( addr==ctx->staked_addr ) ) return;
435 3 : FD_LOG_NOTICE(( "gossip shows the identity at `" FD_IP4_ADDR_FMT "`, a handoff from this machine dials it", FD_IP4_ADDR_FMT_ARGS( addr ) ));
436 3 : ctx->staked_addr = addr;
437 3 : if( FD_UNLIKELY( ctx->request.id && ctx->request.until && !ctx->request.addr && !ctx->cmd_addr && is_requesting( ctx ) ) ) {
438 3 : ctx->request.addr = addr;
439 3 : ctx->request.until = fd_long_sat_add( now, ctx->request.quick ? FD_FAILOVER_FORCE_WAIT_NANOS : FD_FAILOVER_DEADLINE_NANOS );
440 3 : fd_failover_channel_init_dialer( ctx->channel, addr, get_dial_port( ctx ) );
441 3 : LOG_CMD_STEP( ctx, "handoff %lu: requesting handoff from the machine running the identity at `" FD_IP4_ADDR_FMT ":%hu` using gossip",
442 3 : ctx->request.id, FD_IP4_ADDR_FMT_ARGS( addr ), get_dial_port( ctx ) );
443 3 : }
444 3 : }
445 :
446 : static void
447 : on_admin_operator_action_msg( fd_failover_tile_ctx_t * ctx,
448 21 : fd_failover_operator_t const * operator ) {
449 21 : if( FD_UNLIKELY( operator->epoch<=ctx->id_switch.epoch ) ) return;
450 18 : ctx->id_switch.epoch = operator->epoch;
451 18 : ctx->id_switch.discard = is_switch_pending( ctx );
452 18 : ctx->id_switch.fresh = 0;
453 18 : ctx->id_switch.overdue = 0;
454 18 : if( FD_UNLIKELY( ctx->action!=FD_FAILOVER_ACTION_IDLE || ctx->request.id || ctx->handoff.delivery!=FD_FAILOVER_DELIVERY_NONE || ctx->reply.state==FD_FAILOVER_REPLY_PENDING ) ) {
455 3 : FD_LOG_WARNING(( "the identity changed in %s, dropping request %lu, handoff %lu and reply %lu, the other machine may already run the identity",
456 3 : get_action_name( ctx->action ), ctx->request.id, ctx->handoff.id, ctx->reply.handoff_id ));
457 3 : }
458 18 : finish_cmd( ctx, FD_FAILOVER_CONTROL_RESULT_IDENTITY_CHANGED );
459 18 : hang_up( ctx, fd_failover_clock() );
460 18 : reset_controller( ctx );
461 18 : if( FD_UNLIKELY( !fd_memeq( operator->failover_identity, ctx->hello.staked_pubkey, 32UL ) ) ) {
462 3 : ctx->staked_addr = 0U;
463 3 : ctx->staked_seen_at = 0L;
464 3 : ctx->holder_seen_at = 0L;
465 3 : ctx->peer_floor = FD_FAILOVER_SLOT_NULL;
466 3 : ctx->own_floor = FD_FAILOVER_SLOT_NULL;
467 3 : ctx->own_voted = 0;
468 3 : ctx->target_since = fd_failover_clock();
469 3 : }
470 18 : ulong boot_id = ctx->hello.boot_id;
471 18 : do FD_TEST( fd_rng_secure( &ctx->hello.boot_id, sizeof(ulong) ) ); while( FD_UNLIKELY( !ctx->hello.boot_id || ctx->hello.boot_id==boot_id ) );
472 18 : fd_memcpy( ctx->hello.staked_pubkey, operator->failover_identity, 32UL );
473 18 : set_installed( ctx, operator->identity );
474 18 : fd_tls_sign_t signer = { .ctx=ctx, .sign_fn=sign_ed25519 };
475 18 : if( FD_UNLIKELY( fd_failover_channel_set_identity( ctx->channel, signer, (fd_failover_proof_signer_t){ .ctx=ctx, .sign_fn=sign_proof }, &ctx->hello ) ) ) {
476 0 : FD_LOG_ERR(( "failover TLS setup failed after the identity changed" ));
477 0 : }
478 18 : sync_session( ctx );
479 18 : FD_BASE58_ENCODE_32_BYTES( operator->identity, identity );
480 18 : FD_BASE58_ENCODE_32_BYTES( operator->failover_identity, failover_identity );
481 18 : FD_LOG_NOTICE(( "failover now hands off `%s` and this machine runs `%s`, with a new boot %016lx", failover_identity, identity, ctx->hello.boot_id ));
482 18 : }
483 :
484 : /* Demotion: the holder gives up the identity */
485 :
486 : /* Our DEMOTED has been taken to logical completion. */
487 : static void
488 : resolve_handoff( fd_failover_tile_ctx_t * ctx,
489 21 : ulong result ) {
490 21 : if( FD_UNLIKELY( ctx->tx.valid && ctx->tx.type==(ushort)FD_FAILOVER_MSG_DEMOTED ) ) ctx->tx.valid = 0;
491 21 : if( FD_LIKELY( ctx->action==FD_FAILOVER_ACTION_DEMOTE_WAIT_ACK ) ) {
492 : /* reset to idle */
493 6 : set_action( ctx, FD_FAILOVER_ACTION_IDLE );
494 6 : }
495 21 : ctx->handoff.delivery = FD_FAILOVER_DELIVERY_NONE;
496 21 : ctx->handoff.result = result;
497 21 : }
498 :
499 : static void
500 : abort_demotion( fd_failover_tile_ctx_t * ctx,
501 3 : char const * why ) {
502 3 : FD_LOG_WARNING(( "handoff %lu: %s, %s", ctx->handoff.id, why, holds_identity( ctx ) ? "this machine keeps the identity" : "nobody runs the identity" ));
503 3 : if( FD_UNLIKELY( ctx->handoff.delivery!=FD_FAILOVER_DELIVERY_NONE ) ) queue_handoff_result( ctx, &ctx->handoff.target, ctx->handoff.id, FD_FAILOVER_CONTROL_RESULT_HANDOFF_ABORTED );
504 3 : ctx->handoff.delivery = FD_FAILOVER_DELIVERY_NONE;
505 3 : ctx->stuck = 1;
506 :
507 : /* reset to idle */
508 3 : set_action( ctx, FD_FAILOVER_ACTION_IDLE );
509 3 : }
510 :
511 : /* The final history has to be the history of our last vote. */
512 : static int
513 90 : is_final_history_ok( fd_failover_tile_ctx_t const * ctx ) {
514 90 : return ctx->current_history.valid && ctx->current_history.sz && ctx->last_vote_slot!=FD_FAILOVER_SLOT_NULL && ctx->current_history.tip==ctx->last_vote_slot;
515 90 : }
516 :
517 : /* Whether we can hand off now to a machine whose replay is at
518 : replay_slot.
519 : Returns FD_ADMINCTL_RESULT_SUCCESS or an error code. */
520 : static ulong
521 : check_handoff( fd_failover_tile_ctx_t const * ctx,
522 57 : ulong replay_slot ) {
523 57 : if( FD_UNLIKELY( !is_final_history_ok( ctx ) ) ) return FD_FAILOVER_CONTROL_RESULT_NO_FINAL_HISTORY;
524 54 : fd_votor_hist_t hist[1];
525 54 : ulong anchor = fd_votor_hist_de( ctx->current_history.state, ctx->current_history.sz, hist ) ? ctx->last_vote_slot : hist->anchor;
526 54 : if( FD_UNLIKELY( replay_slot==FD_FAILOVER_SLOT_NULL || fd_ulong_sat_add( replay_slot, FD_FAILOVER_DEADLINE_SLOTS )<anchor ) ) return FD_FAILOVER_CONTROL_RESULT_REPLAY_BEHIND;
527 48 : return FD_ADMINCTL_RESULT_SUCCESS;
528 54 : }
529 :
530 : /* Gives up the identity for the peer that asked (triggers the demotion flow). */
531 : static void
532 : accept_handoff( fd_failover_tile_ctx_t * ctx,
533 48 : ulong handoff_id ) {
534 48 : fd_memset( &ctx->handoff, 0, sizeof(ctx->handoff) );
535 48 : ctx->handoff.id = handoff_id;
536 48 : ctx->handoff.delivery = FD_FAILOVER_DELIVERY_PENDING;
537 48 : ctx->handoff.target.boot_id = ctx->session.peer_boot_id;
538 48 : ctx->stuck = 0;
539 48 : FD_LOG_NOTICE(( "handoff %lu: accepted the request from peer boot %016lx, giving up the identity", handoff_id, ctx->handoff.target.boot_id ));
540 :
541 48 : set_action( ctx, FD_FAILOVER_ACTION_DEMOTE_SWITCH );
542 48 : }
543 :
544 : /* Drain vote-history stream up to the key switching watermark, then enqueue the DEMOTED message. */
545 : static void
546 36 : step_demote_drain( fd_failover_tile_ctx_t * ctx ) {
547 36 : if( FD_UNLIKELY( !fd_seq_ge( fd_seq_inc( ctx->hist_seen_seq, 1UL ), ctx->id_switch.result.history_watermark ) ) ) {
548 3 : if( FD_UNLIKELY( is_overdue( ctx ) ) ) abort_demotion( ctx, "the vote history stream did not report its last votes in time, sending nothing" );
549 3 : return;
550 3 : }
551 33 : if( FD_UNLIKELY( !is_final_history_ok( ctx ) ) ) { abort_demotion( ctx, "no final vote history for the last vote, sending nothing" ); return; }
552 33 : ctx->handoff.payload_sz = fd_failover_demoted_encode( ctx->handoff.payload, ctx->handoff.id, ctx->handoff.target.boot_id, ctx->last_vote_slot, ctx->current_history.state, ctx->current_history.sz );
553 33 : if( FD_UNLIKELY( !ctx->handoff.payload_sz ) ) { abort_demotion( ctx, "the final vote history does not encode, sending nothing" ); return; }
554 33 : ctx->handoff.result = FD_FAILOVER_HANDOFF_PENDING;
555 33 : FD_LOG_NOTICE(( "handoff %lu: this machine runs its junk key now, sending the final vote history ending at slot %lu", ctx->handoff.id, ctx->last_vote_slot ));
556 33 : (void)queue_demoted( ctx );
557 :
558 33 : set_action( ctx, FD_FAILOVER_ACTION_DEMOTE_WAIT_ACK );
559 33 : }
560 :
561 : /* Asks the admin tile to switch to the junk key. */
562 : static void
563 : step_demote_switch( fd_failover_tile_ctx_t * ctx,
564 75 : fd_stem_context_t * stem ) {
565 75 : if( FD_UNLIKELY( !is_switch_pending( ctx ) && !ctx->id_switch.fresh ) ) {
566 39 : if( FD_UNLIKELY( request_switch( ctx, stem, FD_FAILOVER_SWITCH_KEY_JUNK )==ULONG_MAX ) ) { abort_demotion( ctx, "an identity switch could not be requested" ); return; }
567 39 : FD_LOG_NOTICE(( "handoff %lu: asked the admin tile to install the junk key", ctx->handoff.id ));
568 39 : return;
569 39 : }
570 36 : if( FD_LIKELY( !ctx->id_switch.fresh ) ) {
571 0 : if( FD_UNLIKELY( is_overdue( ctx ) ) ) mark_switch_overdue( ctx );
572 0 : return;
573 0 : }
574 36 : ulong result = take_switch_result( ctx );
575 36 : if( FD_UNLIKELY( result!=FD_FAILOVER_SWITCH_OK ) ) {
576 3 : abort_demotion( ctx, result==FD_FAILOVER_SWITCH_ERR_DISABLED ? "failover is disabled" : result==FD_FAILOVER_SWITCH_ERR_AUTHORIZED_VOTER ? "the identity is an authorized voter" : "the identity switch failed" );
577 3 : return;
578 3 : }
579 :
580 33 : set_installed( ctx, ctx->spare_pubkey );
581 33 : set_action( ctx, FD_FAILOVER_ACTION_DEMOTE_DRAIN );
582 33 : step_demote_drain( ctx );
583 33 : }
584 :
585 : /* DEMOTED is sent out and the peer answers with either PROMOTE_ACK or PROMOTE_REJECTED. */
586 : static void
587 21 : step_demote_wait_ack( fd_failover_tile_ctx_t * ctx ) {
588 21 : if( FD_UNLIKELY( is_overdue( ctx ) ) ) {
589 21 : FD_LOG_WARNING(( "handoff %lu: the peer has not answered, this machine runs the junk key, run `failover promote` to retry", ctx->handoff.id ));
590 21 : ctx->stuck = 1;
591 21 : if( FD_UNLIKELY( ctx->tx.valid && ctx->tx.type==(ushort)FD_FAILOVER_MSG_DEMOTED ) ) {
592 21 : ctx->tx.valid = 0;
593 21 : }
594 :
595 : /* reset to idle */
596 21 : set_action( ctx, FD_FAILOVER_ACTION_IDLE );
597 21 : return;
598 21 : }
599 0 : if( FD_UNLIKELY( ctx->handoff.delivery==FD_FAILOVER_DELIVERY_PENDING && !ctx->tx.valid ) ) {
600 0 : (void)queue_demoted( ctx );
601 0 : }
602 0 : }
603 :
604 : /* Handoff request: the taker asks the holder for the identity */
605 :
606 : /* Our request is paused at its deadline. */
607 : static inline int
608 72 : is_request_paused( fd_failover_tile_ctx_t const * ctx ) {
609 72 : return ctx->request.id && !ctx->request.until && is_requesting( ctx );
610 72 : }
611 :
612 : /* Stops dialing without ending the request. */
613 : static void
614 : pause_request( fd_failover_tile_ctx_t * ctx,
615 : long now,
616 : ulong result,
617 42 : char const * why ) {
618 42 : FD_LOG_WARNING(( "handoff %lu paused in %s, %s, `failover promote` or `failover takeover` resumes it", ctx->request.id, get_action_name( ctx->action ), why ));
619 42 : ctx->request.until = 0L;
620 42 : ctx->stuck = 1;
621 42 : if( ctx->action==FD_FAILOVER_ACTION_HANDOFF_WAIT_PEER ) finish_cmd( ctx, result );
622 42 : hang_up( ctx, now );
623 42 : }
624 :
625 : /* Ends our request with result, an FD_FAILOVER_HANDOFF_*, and hangs up. */
626 : static void
627 : end_request( fd_failover_tile_ctx_t * ctx,
628 : long now,
629 51 : ulong result ) {
630 51 : FD_TEST( !is_switch_pending( ctx ) && is_requesting( ctx ) );
631 51 : FD_LOG_NOTICE(( "handoff %lu: request ended (%s), %s", ctx->request.id, get_handoff_name( result ), holds_identity( ctx ) ? "this machine runs the identity" : "this machine does not run the identity" ));
632 51 : ctx->request.id = 0UL;
633 51 : ctx->request.until = 0L;
634 51 : ctx->tx.valid = 0;
635 51 : ctx->session.close_after_send = 0;
636 51 : ctx->stuck = result!=FD_FAILOVER_HANDOFF_TAKEN;
637 51 : if( ctx->reply.state==FD_FAILOVER_REPLY_PENDING ) ctx->reply.state = FD_FAILOVER_REPLY_KEPT;
638 :
639 : /* reset to idle */
640 51 : set_action( ctx, FD_FAILOVER_ACTION_IDLE );
641 51 : hang_up( ctx, now );
642 51 : }
643 :
644 : static void
645 6 : await_request_result( fd_failover_tile_ctx_t * ctx ) {
646 6 : if( FD_UNLIKELY( !ctx->request.until ) ) fd_failover_channel_init_dialer( ctx->channel, ctx->request.addr, get_dial_port( ctx ) );
647 6 : ctx->request.until = fd_long_sat_add( fd_failover_clock(), FD_FAILOVER_DEADLINE_NANOS );
648 :
649 6 : set_action( ctx, FD_FAILOVER_ACTION_HANDOFF_WAIT_RESULT );
650 6 : }
651 :
652 : /* Votor adoption: the taker adopts a vote history */
653 :
654 : /* The coverage floor is the higher of the peer's reported last vote and
655 : our highest signed vote. */
656 : static ulong
657 54 : get_coverage_floor( fd_failover_tile_ctx_t const * ctx ) {
658 54 : if( FD_UNLIKELY( ctx->peer_floor==FD_FAILOVER_SLOT_NULL ) ) return ctx->own_floor;
659 12 : if( FD_UNLIKELY( ctx->own_floor ==FD_FAILOVER_SLOT_NULL ) ) return ctx->peer_floor;
660 0 : return fd_ulong_max( ctx->peer_floor, ctx->own_floor );
661 12 : }
662 :
663 : /* Checks whether a forced promotion may go ahead now. Returns
664 : FD_ADMINCTL_RESULT_SUCCESS or the refusal code. */
665 : static ulong
666 99 : check_promotion( fd_failover_tile_ctx_t const * ctx ) {
667 99 : if( FD_UNLIKELY( holds_identity( ctx ) ) ) return FD_FAILOVER_CONTROL_RESULT_ALREADY_HOLDS;
668 99 : if( FD_UNLIKELY( ctx->installed_ranked ) ) return FD_FAILOVER_CONTROL_RESULT_HOLDS_OTHER;
669 96 : int cancellable = ctx->action==FD_FAILOVER_ACTION_DEMOTE_WAIT_ACK || ( ctx->request.id && is_requesting( ctx ) );
670 96 : if( FD_UNLIKELY( ( ctx->action!=FD_FAILOVER_ACTION_IDLE && !cancellable ) || is_switch_pending( ctx ) ) ) return FD_FAILOVER_CONTROL_RESULT_IN_PROGRESS;
671 96 : return FD_ADMINCTL_RESULT_SUCCESS;
672 96 : }
673 :
674 : /* Sets the finality anchor of the vote history that will be adopted via promotion. */
675 : static void
676 54 : set_adopt_anchor( fd_failover_tile_ctx_t * ctx ) {
677 54 : ctx->adopt_anchor = FD_FAILOVER_SLOT_NULL;
678 54 : ctx->empty_vote_after = FD_FAILOVER_SLOT_NULL;
679 54 : if( FD_UNLIKELY( ctx->promotion.source==FD_FAILOVER_SOURCE_EMPTY ) ) return;
680 9 : fd_votor_hist_t hist[1];
681 9 : if( FD_LIKELY( !fd_votor_hist_de( ctx->promotion.adopt.state, ctx->promotion.adopt.sz, hist ) ) ) ctx->adopt_anchor = hist->anchor;
682 9 : }
683 :
684 : /* Whether replay reached the saved history's finality anchor. */
685 : static int
686 18 : is_replay_ready( fd_failover_tile_ctx_t * ctx ) {
687 18 : if( FD_LIKELY( ctx->promotion.source!=FD_FAILOVER_SOURCE_EMPTY ) ) {
688 3 : return ctx->adopt_anchor==FD_FAILOVER_SLOT_NULL || ctx->replay_slot>=ctx->adopt_anchor;
689 3 : }
690 15 : ulong floor = ctx->promotion.floor;
691 15 : ctx->empty_vote_after = floor==FD_FAILOVER_SLOT_NULL ? ctx->replay_slot : fd_ulong_max( ctx->replay_slot, floor );
692 15 : FD_STORE( ulong, ctx->promotion.adopt.state, ctx->empty_vote_after );
693 15 : ctx->promotion.adopt.sz = FD_VOTOR_ADOPT_EMPTY_SZ;
694 15 : return 1;
695 18 : }
696 :
697 : /* Sends the promotion's history or empty bound to the votor for the
698 : target identity to the votor tile (which then runs failover_adopt_hist).
699 :
700 : Returns the request id or ULONG_MAX if nothing to send. */
701 : static ulong
702 : publish_adopt_state( fd_failover_tile_ctx_t * ctx,
703 12 : fd_stem_context_t * stem ) {
704 12 : if( FD_UNLIKELY( ctx->adopt_out_idx==ULONG_MAX || !ctx->promotion.adopt.sz ) ) return ULONG_MAX;
705 12 : if( FD_UNLIKELY( !++ctx->adopt_request_id ) ) ctx->adopt_request_id++;
706 12 : uchar * req = fd_chunk_to_laddr( ctx->adopt_out_mem, ctx->adopt_out_chunk );
707 12 : ulong sz = FD_VOTOR_ADOPT_REQ_HDR_SZ+ctx->promotion.adopt.sz;
708 12 : fd_memcpy( req, ctx->hello.staked_pubkey, FD_VOTOR_ADOPT_REQ_HDR_SZ );
709 12 : fd_memcpy( req+FD_VOTOR_ADOPT_REQ_HDR_SZ, ctx->promotion.adopt.state, ctx->promotion.adopt.sz );
710 12 : ulong tspub = fd_frag_meta_ts_comp( fd_tickcount() );
711 12 : fd_stem_publish( stem, ctx->adopt_out_idx, ctx->adopt_request_id, ctx->adopt_out_chunk, sz, 0UL, tspub, tspub );
712 12 : ctx->adopt_out_chunk = fd_dcache_compact_next( ctx->adopt_out_chunk, sz, ctx->adopt_out_chunk0, ctx->adopt_out_wmark );
713 12 : ctx->adopt_result_fresh = 0;
714 12 : return ctx->adopt_request_id;
715 12 : }
716 :
717 : static ulong
718 9 : get_reject_result( uchar reason ) {
719 9 : switch( reason ) {
720 3 : case (uchar)FD_FAILOVER_REJECT_REPLAY_BEHIND: return FD_FAILOVER_CONTROL_RESULT_REPLAY_BEHIND;
721 0 : case (uchar)FD_FAILOVER_REJECT_HOLDS_IDENTITY: return FD_FAILOVER_CONTROL_RESULT_PEER_HOLDS;
722 0 : case (uchar)FD_FAILOVER_REJECT_SWITCH_FAILED: return FD_FAILOVER_CONTROL_RESULT_SWITCH_FAILED;
723 6 : default: return FD_FAILOVER_CONTROL_RESULT_ADOPTION_FAILED;
724 9 : }
725 9 : }
726 :
727 : /* Ends an operator promotion at IDLE, or answers the peer and waits for
728 : its result on a cooperative handoff. Returns 1 if it was from a peer. */
729 : static int
730 : end_promotion( fd_failover_tile_ctx_t * ctx,
731 : ushort type,
732 12 : uchar reason ) {
733 12 : if( FD_LIKELY( !ctx->promotion.from_peer ) ) {
734 : /* reset to idle */
735 6 : set_action( ctx, FD_FAILOVER_ACTION_IDLE );
736 6 : return 0;
737 6 : }
738 6 : ctx->peer_history.valid = 0;
739 6 : ctx->promotion.from_peer = 0;
740 6 : await_request_result( ctx );
741 6 : finish_reply( ctx, &ctx->promotion.peer, ctx->promotion.handoff_id, type, reason );
742 6 : return 1;
743 12 : }
744 :
745 : /* Ends the promotion without switching to the identity.. */
746 : static void
747 : reject_promotion( fd_failover_tile_ctx_t * ctx,
748 : uchar reason,
749 9 : int bad_history ) {
750 9 : if( FD_UNLIKELY( bad_history && ctx->promotion.source==FD_FAILOVER_SOURCE_PEER ) ) ctx->peer_history.valid = 0;
751 9 : ctx->stuck = 1;
752 9 : FD_LOG_WARNING(( "%s failed (%s), this machine does not take the identity", ctx->promotion.label, get_reject_name( reason ) ));
753 9 : finish_cmd( ctx, get_reject_result( reason ) );
754 9 : end_promotion( ctx, (ushort)FD_FAILOVER_MSG_PROMOTE_REJECTED, reason );
755 9 : }
756 :
757 : /* Promotion: the taker installs the identity it adopted */
758 :
759 : static void
760 : start_promotion( fd_failover_tile_ctx_t * ctx,
761 : ulong source,
762 : fd_failover_peer_t const * peer,
763 : ulong handoff_id,
764 54 : ulong flags ) {
765 54 : if( FD_UNLIKELY( ctx->handoff.delivery!=FD_FAILOVER_DELIVERY_NONE ) ) {
766 3 : resolve_handoff( ctx, FD_FAILOVER_HANDOFF_CANCELLED );
767 3 : }
768 54 : fd_memset( &ctx->promotion, 0, sizeof(ctx->promotion) );
769 54 : if( source==FD_FAILOVER_SOURCE_PEER ) ctx->promotion.adopt = ctx->peer_history;
770 45 : else ctx->promotion.adopt.tip = FD_FAILOVER_SLOT_NULL;
771 54 : if( peer ) {
772 9 : ctx->promotion.peer = *peer;
773 9 : ctx->promotion.handoff_id = handoff_id;
774 9 : ctx->promotion.from_peer = 1;
775 9 : fd_cstr_printf( ctx->promotion.label, sizeof(ctx->promotion.label), NULL, "handoff %lu", handoff_id );
776 45 : } else {
777 45 : fd_cstr_printf( ctx->promotion.label, sizeof(ctx->promotion.label), NULL, "operator promotion" );
778 45 : }
779 54 : ctx->promotion.source = source;
780 54 : ctx->promotion.force = !!( flags & FD_ADMINCTL_FAILOVER_FLAG_TAKEOVER );
781 54 : ctx->promotion.soft = ctx->promotion.force && !( flags & FD_ADMINCTL_FAILOVER_FLAG_FORCE );
782 54 : ctx->promotion.floor = get_coverage_floor( ctx );
783 54 : ctx->current_history = ctx->promotion.adopt;
784 54 : ctx->last_vote_slot = ctx->promotion.adopt.tip;
785 54 : ctx->stuck = 0;
786 54 : set_adopt_anchor( ctx );
787 54 : LOG_CMD_STEP( ctx, "%s: adopting %s%s", ctx->promotion.label, get_source_name( source ),
788 54 : ( flags & FD_ADMINCTL_FAILOVER_FLAG_FORCE ) ? ", --force skips the checks on the other machine" :
789 54 : ( flags & FD_ADMINCTL_FAILOVER_FLAG_TAKEOVER ) ? ", the other machine showed no signs of life" : "" );
790 :
791 54 : set_action( ctx, FD_FAILOVER_ACTION_PROMOTE_WAIT_REPLAY );
792 54 : }
793 :
794 : /* Requests votor to adopt the history. */
795 : static int
796 : publish_adoption( fd_failover_tile_ctx_t * ctx,
797 12 : fd_stem_context_t * stem ) {
798 12 : ulong id = publish_adopt_state( ctx, stem );
799 12 : if( FD_UNLIKELY( id==ULONG_MAX ) ) { reject_promotion( ctx, FD_FAILOVER_REJECT_ADOPTION_FAILED, 0 ); return 0; }
800 12 : LOG_CMD_STEP( ctx, "%s: asking the votor to adopt %s", ctx->promotion.label, get_source_name( ctx->promotion.source ) );
801 12 : ctx->promotion.adopt_id = id;
802 12 : ctx->promotion.retry = 0;
803 12 : return 1;
804 12 : }
805 :
806 : /* Waits for replay to reach the history's finality anchor, then asks the votor to adopt it. */
807 : static void
808 : step_promote_wait_replay( fd_failover_tile_ctx_t * ctx,
809 12 : fd_stem_context_t * stem ) {
810 12 : if( FD_UNLIKELY( is_overdue( ctx ) ) ) {
811 3 : if( ctx->replay_slot==FD_FAILOVER_SLOT_NULL ) LOG_CMD_STEP( ctx, "%s: replay reported no slot before the deadline", ctx->promotion.label );
812 3 : else LOG_CMD_STEP( ctx, "%s: replay at slot %lu did not reach the vote history's finality anchor %lu before the deadline", ctx->promotion.label, ctx->replay_slot, ctx->adopt_anchor );
813 3 : reject_promotion( ctx, ctx->promotion.from_peer || ctx->promotion.gave_up ? FD_FAILOVER_REJECT_ADOPTION_FAILED : FD_FAILOVER_REJECT_REPLAY_BEHIND, 0 );
814 3 : return;
815 3 : }
816 9 : if( FD_UNLIKELY( ctx->replay_slot==FD_FAILOVER_SLOT_NULL ) ) return;
817 9 : if( FD_UNLIKELY( !is_replay_ready( ctx ) ) ) return; /* empty-history case */
818 9 : if( FD_UNLIKELY( !publish_adoption( ctx, stem ) ) ) return;
819 :
820 9 : set_action( ctx, FD_FAILOVER_ACTION_PROMOTE_WAIT_ADOPT );
821 9 : }
822 :
823 : /* Waits for the votor's response and then handles it. */
824 : static void
825 : step_promote_wait_adopt( fd_failover_tile_ctx_t * ctx,
826 21 : fd_stem_context_t * stem ) {
827 21 : if( FD_UNLIKELY( is_overdue( ctx ) ) ) { reject_promotion( ctx, FD_FAILOVER_REJECT_ADOPTION_FAILED, 0 ); return; }
828 18 : if( FD_UNLIKELY( ctx->promotion.retry ) ) {
829 6 : if( FD_LIKELY( ctx->replay_slot<=ctx->promotion.retry_slot ) ) return;
830 3 : (void)publish_adoption( ctx, stem );
831 3 : return;
832 6 : }
833 12 : if( FD_LIKELY( !ctx->adopt_result_fresh ) ) return;
834 12 : ctx->adopt_result_fresh = 0;
835 12 : if( FD_UNLIKELY( ctx->adopt_result_id!=ctx->promotion.adopt_id ) ) return;
836 12 : ulong result = ctx->adopt_result.result;
837 12 : char const * err = get_adopt_err_name( result );
838 12 : if( FD_UNLIKELY( result==FD_VOTOR_ADOPT_ERR_UNREPLAYED || result==FD_VOTOR_ADOPT_ERR_CAPACITY ) ) {
839 : /* retry once replay advances */
840 9 : if( FD_UNLIKELY( !ctx->promotion.from_peer && !ctx->promotion.gave_up ) ) {
841 3 : LOG_CMD_STEP( ctx, "%s: the votor is not ready to adopt (%s), retry once consensus is up", ctx->promotion.label, err );
842 3 : reject_promotion( ctx, FD_FAILOVER_REJECT_REPLAY_BEHIND, 0 );
843 3 : return;
844 3 : }
845 6 : LOG_CMD_STEP( ctx, "%s: the votor is not ready to adopt the vote history (%s), asking again as replay moves", ctx->promotion.label, err );
846 6 : ctx->promotion.retry = 1;
847 6 : ctx->promotion.retry_slot = ctx->replay_slot;
848 6 : return;
849 9 : }
850 3 : if( FD_UNLIKELY( result!=FD_VOTOR_ADOPT_SUCCESS ) ) {
851 0 : reject_promotion( ctx, FD_FAILOVER_REJECT_ADOPTION_FAILED, result==FD_VOTOR_ADOPT_ERR_DECODE || result==FD_VOTOR_ADOPT_ERR_INVALID || result==FD_VOTOR_ADOPT_ERR_STALE );
852 0 : return;
853 0 : }
854 :
855 : /* check tip and coverage floor */
856 3 : ulong tip = ctx->promotion.adopt.tip;
857 3 : if( FD_UNLIKELY( tip!=FD_FAILOVER_SLOT_NULL && ctx->adopt_result.vote_slot!=tip ) ) { reject_promotion( ctx, FD_FAILOVER_REJECT_ADOPTION_MISMATCH, 1 ); return; }
858 3 : ulong floor = ctx->promotion.floor;
859 3 : ulong reached = ctx->promotion.source==FD_FAILOVER_SOURCE_EMPTY ? ctx->adopt_result.vote_bound : ctx->adopt_result.vote_slot;
860 3 : int covered = floor==FD_FAILOVER_SLOT_NULL || ( reached!=FD_FAILOVER_SLOT_NULL && reached>=floor );
861 3 : if( ctx->adopt_result.vote_slot==FD_FAILOVER_SLOT_NULL ) LOG_CMD_STEP( ctx, "%s: the votor adopted %s, no last vote, no vote at or below slot %lu", ctx->promotion.label, get_source_name( ctx->promotion.source ), ctx->adopt_result.vote_bound );
862 3 : else LOG_CMD_STEP( ctx, "%s: the votor adopted %s, last vote at slot %lu, no vote at or below slot %lu", ctx->promotion.label, get_source_name( ctx->promotion.source ), ctx->adopt_result.vote_slot, ctx->adopt_result.vote_bound );
863 3 : if( FD_UNLIKELY( !covered && !ctx->promotion.force ) ) {
864 0 : ctx->promotion.retry = 1;
865 0 : ctx->promotion.retry_slot = FD_FAILOVER_SLOT_NULL;
866 0 : return;
867 0 : }
868 3 : if( FD_UNLIKELY( ctx->promotion.holder_seen && ( !ctx->promotion.force || ctx->promotion.soft ) ) ) { reject_promotion( ctx, FD_FAILOVER_REJECT_HOLDS_IDENTITY, 0 ); return; }
869 3 : if( FD_UNLIKELY( tip==FD_FAILOVER_SLOT_NULL ) ) ctx->last_vote_slot = ctx->adopt_result.vote_slot;
870 3 : if( FD_UNLIKELY( request_switch( ctx, stem, FD_FAILOVER_SWITCH_KEY_STAKED )==ULONG_MAX ) ) { reject_promotion( ctx, FD_FAILOVER_REJECT_ADOPTION_FAILED, 0 ); return; }
871 3 : LOG_CMD_STEP( ctx, "%s: asked the admin tile to install the identity", ctx->promotion.label );
872 :
873 : /* success, ask the admin tile to install the real identity! */
874 3 : set_action( ctx, FD_FAILOVER_ACTION_PROMOTE_SWITCH );
875 3 : }
876 :
877 : /* Waits for the admin tile to install the identity. */
878 : static void
879 3 : step_promote_switch( fd_failover_tile_ctx_t * ctx ) {
880 3 : if( FD_LIKELY( !ctx->id_switch.fresh ) ) {
881 0 : if( FD_UNLIKELY( is_overdue( ctx ) ) ) mark_switch_overdue( ctx );
882 0 : return;
883 0 : }
884 3 : if( FD_UNLIKELY( take_switch_result( ctx )!=FD_FAILOVER_SWITCH_OK ) ) { reject_promotion( ctx, FD_FAILOVER_REJECT_SWITCH_FAILED, 0 ); return; }
885 3 : if( !ctx->promotion.from_peer ) hang_up( ctx, fd_failover_clock() );
886 :
887 : /* look at me, I'm the holder now */
888 3 : set_installed( ctx, ctx->hello.staked_pubkey );
889 3 : ctx->peer_history.valid = 0;
890 3 : ctx->own_voted = 1;
891 3 : ctx->stuck = 0;
892 3 : FD_LOG_NOTICE(( "%s: this machine runs the identity now, last vote at slot %lu", ctx->promotion.label, ctx->last_vote_slot ));
893 3 : finish_cmd( ctx, FD_ADMINCTL_RESULT_SUCCESS );
894 :
895 : /* send PROMOTE_ACK back to the peer for cooperative handoffs */
896 3 : end_promotion( ctx, (ushort)FD_FAILOVER_MSG_PROMOTE_ACK, (uchar)FD_FAILOVER_REJECT_NONE );
897 3 : }
898 :
899 : /* The controller step dispatcher */
900 :
901 : /* Advances the controller state (main dispatcher). */
902 : static void
903 : step_controller( fd_failover_tile_ctx_t * ctx,
904 105 : fd_stem_context_t * stem ) {
905 105 : if( FD_UNLIKELY( ctx->reply.state==FD_FAILOVER_REPLY_PENDING && !ctx->tx.valid ) ) queue_reply( ctx );
906 105 : switch( ctx->action ) {
907 0 : case FD_FAILOVER_ACTION_IDLE:
908 0 : case FD_FAILOVER_ACTION_HANDOFF_WAIT_PEER:
909 0 : case FD_FAILOVER_ACTION_HANDOFF_WAIT_RESULT: return;
910 75 : case FD_FAILOVER_ACTION_DEMOTE_SWITCH: step_demote_switch( ctx, stem ); return;
911 3 : case FD_FAILOVER_ACTION_DEMOTE_DRAIN: step_demote_drain( ctx ); return;
912 21 : case FD_FAILOVER_ACTION_DEMOTE_WAIT_ACK: step_demote_wait_ack( ctx ); return;
913 3 : case FD_FAILOVER_ACTION_PROMOTE_WAIT_REPLAY: step_promote_wait_replay( ctx, stem ); return;
914 0 : case FD_FAILOVER_ACTION_PROMOTE_WAIT_ADOPT: step_promote_wait_adopt( ctx, stem ); return;
915 3 : case FD_FAILOVER_ACTION_PROMOTE_SWITCH: step_promote_switch( ctx ); return;
916 0 : default: FD_LOG_ERR(( "unexpected failover action %lu", ctx->action ));
917 105 : }
918 105 : }
919 :
920 : /* Peer messages and polling. Each handler returns 1 when the message
921 : was handled, including one it ignored on purpose, and 0 when it was
922 : malformed, which drops the session. */
923 :
924 : /* HANDOFF_REQUEST, the peer asks us to give up the identity. */
925 : static int
926 : on_handoff_request_msg( fd_failover_tile_ctx_t * ctx,
927 81 : ulong payload_sz ) {
928 81 : fd_failover_handoff_request_t request;
929 81 : if( FD_UNLIKELY( !fd_failover_handoff_request_decode( &request, ctx->rx, payload_sz ) ) ) return 0;
930 81 : ulong result = FD_ADMINCTL_RESULT_SUCCESS;
931 81 : if( FD_UNLIKELY( request.target_boot_id!=ctx->hello.boot_id ) ) {
932 6 : result = FD_FAILOVER_CONTROL_RESULT_PEER_RESTARTED;
933 75 : } else if( FD_UNLIKELY( request.handoff_id==ctx->handoff.id && is_bound_peer( ctx, &ctx->handoff.target ) ) ) {
934 15 : if( is_switching_out( ctx ) ) return 1;
935 12 : if( ctx->handoff.delivery!=FD_FAILOVER_DELIVERY_NONE ) {
936 6 : if( !ctx->tx.valid ) (void)queue_demoted( ctx );
937 6 : return 1;
938 6 : }
939 6 : result = ctx->handoff.result==FD_FAILOVER_HANDOFF_TAKEN ? FD_ADMINCTL_RESULT_SUCCESS : FD_FAILOVER_CONTROL_RESULT_HANDOFF_ABORTED;
940 60 : } else if( ctx->action!=FD_FAILOVER_ACTION_IDLE || is_switch_pending( ctx ) || ctx->tx.valid || ctx->session.close_after_send || ctx->handoff.delivery!=FD_FAILOVER_DELIVERY_NONE ) {
941 3 : result = FD_FAILOVER_CONTROL_RESULT_IN_PROGRESS;
942 57 : } else if( !holds_identity( ctx ) ) {
943 0 : result = FD_FAILOVER_CONTROL_RESULT_NOT_HOLDER;
944 57 : } else if( fd_failover_channel_peer_hello( ctx->channel )->holds ) {
945 0 : result = FD_FAILOVER_CONTROL_RESULT_ALREADY_HOLDS;
946 57 : } else {
947 57 : result = check_handoff( ctx, request.replay_slot );
948 : /* trigger the demotion flow */
949 57 : if( FD_LIKELY( result==FD_ADMINCTL_RESULT_SUCCESS ) ) { accept_handoff( ctx, request.handoff_id ); return 1; }
950 57 : }
951 24 : FD_LOG_WARNING(( "handoff %lu refused (result %#lx)", request.handoff_id, result ));
952 24 : queue_handoff_result( ctx, NULL, request.handoff_id, result );
953 24 : return 1;
954 81 : }
955 :
956 : /* HANDOFF_RESULT, the peer recorded how our request ended. */
957 : static int
958 : on_handoff_result_msg( fd_failover_tile_ctx_t * ctx,
959 : ulong payload_sz,
960 3 : long now ) {
961 3 : fd_failover_handoff_result_t result;
962 3 : if( FD_UNLIKELY( !fd_failover_handoff_result_decode( &result, ctx->rx, payload_sz ) ) ) return 0;
963 3 : if( FD_UNLIKELY( !ctx->request.id || result.handoff_id!=ctx->request.id || !is_bound_peer( ctx, &ctx->request.peer ) || !is_requesting( ctx ) ) ) return 1;
964 3 : int failed = result.result!=FD_ADMINCTL_RESULT_SUCCESS || !holds_identity( ctx );
965 3 : if( FD_UNLIKELY( failed && result.result==FD_FAILOVER_CONTROL_RESULT_NO_FINAL_HISTORY &&
966 3 : ctx->action==FD_FAILOVER_ACTION_HANDOFF_WAIT_PEER && ctx->request.send_cnt<FD_FAILOVER_REQUEST_RETRIES ) ) {
967 0 : LOG_CMD_STEP( ctx, "handoff %lu: the other machine has no final vote state yet, asking again (send %lu)", result.handoff_id, ctx->request.send_cnt );
968 0 : return 1;
969 0 : }
970 3 : if( FD_UNLIKELY( failed ) ) {
971 3 : FD_LOG_WARNING(( "handoff %lu ended (result %#lx), %s", result.handoff_id, result.result, holds_identity( ctx ) ? "this machine runs the identity and the other may run it too" : "this machine does not run the identity" ));
972 3 : ulong cmd_result = result.result;
973 3 : if( FD_UNLIKELY( cmd_result==FD_FAILOVER_CONTROL_RESULT_ALREADY_HOLDS ) ) cmd_result = holds_identity( ctx ) ? FD_FAILOVER_CONTROL_RESULT_PEER_HOLDS : FD_FAILOVER_CONTROL_RESULT_IN_PROGRESS;
974 3 : if( FD_UNLIKELY( cmd_result==FD_ADMINCTL_RESULT_SUCCESS ) ) cmd_result = FD_FAILOVER_CONTROL_RESULT_HANDOFF_ABORTED;
975 3 : finish_cmd( ctx, cmd_result );
976 3 : }
977 3 : end_request( ctx, now, failed ? FD_FAILOVER_HANDOFF_DECLINED : FD_FAILOVER_HANDOFF_TAKEN );
978 3 : return 1;
979 3 : }
980 :
981 : /* DEMOTED, the peer gave up the identity and hands us its final history.
982 : We call start_promotion to kick off promotion. */
983 : static int
984 : on_demoted_msg( fd_failover_tile_ctx_t * ctx,
985 6 : ulong payload_sz ) {
986 6 : fd_failover_demoted_t demoted;
987 6 : if( FD_UNLIKELY( !fd_failover_demoted_decode( &demoted, ctx->rx, payload_sz ) || demoted.mode!=(uchar)FD_FAILOVER_MODE_ALPENGLOW ) ) return 0;
988 6 : if( FD_UNLIKELY( !ctx->request.id || ctx->request.id!=demoted.handoff_id || !is_bound_peer( ctx, &ctx->request.peer ) ) ) return 1;
989 6 : if( FD_UNLIKELY( demoted.target_boot_id!=ctx->hello.boot_id ) ) return 0;
990 6 : if( FD_UNLIKELY( ctx->promotion.from_peer && ctx->promotion.handoff_id==demoted.handoff_id ) ) return 1;
991 6 : if( FD_UNLIKELY( ctx->reply.state!=FD_FAILOVER_REPLY_NONE && ctx->reply.handoff_id==demoted.handoff_id && is_bound_peer( ctx, &ctx->reply.peer ) ) ) { queue_reply( ctx ); return 1; }
992 6 : uchar reason = holds_identity( ctx ) ? FD_FAILOVER_REJECT_HOLDS_IDENTITY :
993 6 : ctx->action!=FD_FAILOVER_ACTION_HANDOFF_WAIT_PEER || is_switch_pending( ctx ) ? FD_FAILOVER_REJECT_BUSY : FD_FAILOVER_REJECT_NONE;
994 6 : if( FD_UNLIKELY( reason!=FD_FAILOVER_REJECT_NONE ) ) {
995 0 : FD_LOG_WARNING(( "handoff %lu: refusing the peer's final state (%s)", demoted.handoff_id, get_reject_name( reason ) ));
996 0 : finish_reply( ctx, &ctx->request.peer, demoted.handoff_id, (ushort)FD_FAILOVER_MSG_PROMOTE_REJECTED, reason );
997 0 : return 1;
998 0 : }
999 6 : fd_failover_history_t history = { .valid=1, .tip=demoted.last_vote_slot, .sz=(ulong)demoted.state_len };
1000 6 : fd_memcpy( history.state, ctx->rx+sizeof(fd_failover_demoted_t), history.sz );
1001 6 : if( ctx->peer_floor==FD_FAILOVER_SLOT_NULL || history.tip>ctx->peer_floor ) ctx->peer_floor = history.tip;
1002 6 : ctx->peer_history = history;
1003 6 : start_promotion( ctx, FD_FAILOVER_SOURCE_PEER, &ctx->request.peer, demoted.handoff_id, 0 );
1004 6 : return 1;
1005 6 : }
1006 :
1007 : /* PROMOTE_ACK or PROMOTE_REJECTED, the peer's response to our DEMOTED. */
1008 : static int
1009 : on_demoted_answer_msg( fd_failover_tile_ctx_t * ctx,
1010 : ushort type,
1011 15 : ulong payload_sz ) {
1012 15 : int taken = type==(ushort)FD_FAILOVER_MSG_PROMOTE_ACK;
1013 15 : fd_failover_promote_ack_t ack;
1014 15 : fd_failover_promote_rejected_t rej;
1015 15 : if( FD_UNLIKELY( taken ? !fd_failover_promote_ack_decode( &ack, ctx->rx, payload_sz ) : !fd_failover_promote_rejected_decode( &rej, ctx->rx, payload_sz ) ) ) return 0;
1016 15 : ulong handoff_id = taken ? ack.handoff_id : rej.handoff_id;
1017 15 : if( FD_UNLIKELY( ctx->handoff.delivery==FD_FAILOVER_DELIVERY_NONE || handoff_id!=ctx->handoff.id || !is_bound_peer( ctx, &ctx->handoff.target ) || is_switching_out( ctx ) ) ) return 1;
1018 12 : if( taken ) FD_LOG_NOTICE (( "handoff %lu: the peer took the identity, this machine keeps the junk key", handoff_id ));
1019 3 : else FD_LOG_WARNING(( "handoff %lu: the peer declined the identity (%s), nobody runs it", handoff_id, get_reject_name( rej.reason ) ));
1020 12 : if( taken ) ctx->holder_seen_at = fd_long_max( ctx->holder_seen_at, fd_failover_clock() );
1021 12 : ctx->peer_history.valid = 0;
1022 12 : ctx->stuck = !taken;
1023 12 : resolve_handoff( ctx, taken ? FD_FAILOVER_HANDOFF_TAKEN : FD_FAILOVER_HANDOFF_DECLINED );
1024 12 : queue_handoff_result( ctx, NULL, handoff_id, taken ? FD_ADMINCTL_RESULT_SUCCESS : FD_FAILOVER_CONTROL_RESULT_HANDOFF_ABORTED );
1025 12 : return 1;
1026 15 : }
1027 :
1028 : /* Handles a control message. Anything malformed drops the session. */
1029 : static void
1030 : handle_control( fd_failover_tile_ctx_t * ctx,
1031 : ushort type,
1032 : ulong payload_sz,
1033 105 : long now ) {
1034 105 : int handled;
1035 105 : switch( type ) {
1036 81 : case (ushort)FD_FAILOVER_MSG_HANDOFF_REQUEST: handled = on_handoff_request_msg( ctx, payload_sz ); break;
1037 3 : case (ushort)FD_FAILOVER_MSG_HANDOFF_RESULT: handled = on_handoff_result_msg ( ctx, payload_sz, now ); break;
1038 6 : case (ushort)FD_FAILOVER_MSG_DEMOTED: handled = on_demoted_msg ( ctx, payload_sz ); break;
1039 12 : case (ushort)FD_FAILOVER_MSG_PROMOTE_ACK:
1040 15 : case (ushort)FD_FAILOVER_MSG_PROMOTE_REJECTED: handled = on_demoted_answer_msg ( ctx, type, payload_sz ); break;
1041 0 : default: handled = 0; break;
1042 105 : }
1043 105 : if( FD_LIKELY( handled ) ) return;
1044 0 : fd_failover_channel_protocol_error( ctx->channel, now );
1045 0 : sync_session( ctx );
1046 0 : }
1047 :
1048 : /* Whether at, our clock or 0 for never, is within FD_FAILOVER_GOSSIP_FRESH_NANOS. */
1049 : static inline int
1050 : is_recent( long at,
1051 75 : long now ) {
1052 75 : return at && now>=at && now-at<FD_FAILOVER_GOSSIP_FRESH_NANOS;
1053 75 : }
1054 :
1055 : /* The machine running the identity refreshed its contact info in gossip
1056 : within FD_FAILOVER_GOSSIP_FRESH_NANOS. */
1057 : static inline int
1058 : is_holder_fresh( fd_failover_tile_ctx_t const * ctx,
1059 39 : long now ) {
1060 39 : return is_recent( ctx->staked_seen_at, now );
1061 39 : }
1062 :
1063 : static void
1064 63 : note_holder_heard( fd_failover_tile_ctx_t * ctx ) {
1065 63 : if( fd_failover_channel_state( ctx->channel )!=FD_FAILOVER_SESSION_PAIRED || !fd_failover_channel_peer_hello( ctx->channel )->holds ) return;
1066 39 : ctx->holder_seen_at = fd_long_max( ctx->holder_seen_at, fd_failover_channel_peer_heard_at( ctx->channel ) );
1067 39 : }
1068 :
1069 : static int
1070 : is_holder_live( fd_failover_tile_ctx_t const * ctx,
1071 48 : long now ) {
1072 48 : if( fd_failover_channel_state( ctx->channel )==FD_FAILOVER_SESSION_PAIRED && fd_failover_channel_peer_hello( ctx->channel )->holds ) return 1;
1073 42 : if( now>=ctx->target_since && now-ctx->target_since<FD_FAILOVER_GOSSIP_FRESH_NANOS ) return 1;
1074 39 : return is_holder_fresh( ctx, now ) || is_recent( ctx->holder_seen_at, now );
1075 42 : }
1076 :
1077 : /* Pumps the TCP+TLS peer channel and drives our handoff request on it. */
1078 : static void
1079 : poll_peer( fd_failover_tile_ctx_t * ctx,
1080 : long now,
1081 81 : int * charge_busy ) {
1082 81 : sync_session( ctx );
1083 81 : if( FD_UNLIKELY( ctx->request.id && ctx->request.until && now>=ctx->request.until && is_requesting( ctx ) ) ) {
1084 18 : if( ctx->action==FD_FAILOVER_ACTION_HANDOFF_WAIT_RESULT && holds_identity( ctx ) && !is_switch_pending( ctx ) ) {
1085 3 : FD_LOG_WARNING(( "handoff %lu: the peer did not confirm our answer before the deadline, ending the request", ctx->request.id ));
1086 3 : end_request( ctx, now, FD_FAILOVER_HANDOFF_TAKEN );
1087 3 : }
1088 15 : else if( ctx->request.send_cnt ) pause_request( ctx, now, FD_FAILOVER_CONTROL_RESULT_TIMEOUT, "no answer before the deadline" );
1089 12 : else if( !ctx->request.addr ) pause_request( ctx, now, FD_FAILOVER_CONTROL_RESULT_NO_HOLDER_ADDRESS, "gossip did not show the identity before the deadline" );
1090 9 : else if( !ctx->request.peer.boot_id ) pause_request( ctx, now, FD_FAILOVER_CONTROL_RESULT_NO_PEER, "no peer authenticated before the deadline" );
1091 6 : else pause_request( ctx, now, FD_FAILOVER_CONTROL_RESULT_TIMEOUT, "no handoff request sent before the deadline" );
1092 18 : return;
1093 18 : }
1094 63 : if( FD_UNLIKELY( ctx->session.close_after_send && !ctx->tx.valid && !fd_failover_channel_tx_pending( ctx->channel ) ) ) {
1095 0 : ctx->session.close_after_send = 0;
1096 0 : fd_failover_channel_hangup( ctx->channel, now );
1097 0 : sync_session( ctx );
1098 0 : return;
1099 0 : }
1100 63 : ushort type;
1101 63 : ulong payload_sz;
1102 63 : int delivered = fd_failover_channel_poll( ctx->channel, now, charge_busy, &type, ctx->rx, &payload_sz );
1103 63 : sync_session( ctx );
1104 63 : note_holder_heard( ctx );
1105 63 : if( FD_UNLIKELY( delivered ) ) {
1106 0 : handle_control( ctx, type, payload_sz, now );
1107 0 : sync_session( ctx );
1108 0 : }
1109 63 : if( FD_LIKELY( !ctx->request.id || fd_failover_channel_state( ctx->channel )!=FD_FAILOVER_SESSION_PAIRED ) ) return;
1110 57 : fd_failover_hello_t const * peer = fd_failover_channel_peer_hello( ctx->channel );
1111 57 : if( FD_UNLIKELY( ctx->request.peer.boot_id && !is_bound_peer( ctx, &ctx->request.peer ) ) ) {
1112 27 : if( ctx->action==FD_FAILOVER_ACTION_HANDOFF_WAIT_RESULT && !is_switch_pending( ctx ) ) {
1113 12 : FD_LOG_WARNING(( "handoff %lu: the peer restarted before it confirmed our answer, ending the request", ctx->request.id ));
1114 12 : end_request( ctx, now, holds_identity( ctx ) ? FD_FAILOVER_HANDOFF_TAKEN : FD_FAILOVER_HANDOFF_DECLINED );
1115 15 : } else if( ctx->request.until ) {
1116 9 : ctx->request.peer_restarted = 1;
1117 9 : pause_request( ctx, now, FD_FAILOVER_CONTROL_RESULT_PEER_RESTARTED, "the peer restarted" );
1118 9 : }
1119 27 : return;
1120 27 : }
1121 30 : if( FD_LIKELY( !ctx->request.until || ctx->request.sent ) ) return;
1122 18 : if( FD_UNLIKELY( !ctx->request.peer.boot_id ) ) {
1123 12 : if( FD_UNLIKELY( !peer->holds ) ) {
1124 3 : if( FD_UNLIKELY( !ctx->request.addr ) ) return;
1125 3 : FD_LOG_WARNING(( "handoff %lu: dialed `" FD_IP4_ADDR_FMT ":%hu`, which does not run the identity", ctx->request.id, FD_IP4_ADDR_FMT_ARGS( ctx->request.addr ), get_dial_port( ctx ) ));
1126 3 : finish_cmd( ctx, FD_FAILOVER_CONTROL_RESULT_NOT_HOLDER );
1127 3 : end_request( ctx, now, FD_FAILOVER_HANDOFF_NOT_HOLDER );
1128 3 : return;
1129 3 : }
1130 9 : ctx->request.peer.boot_id = peer->boot_id;
1131 9 : }
1132 :
1133 : /* send the HANDOFF_REQUEST over the channel */
1134 15 : fd_failover_handoff_request_t request = { .handoff_id=ctx->request.id, .target_boot_id=ctx->request.peer.boot_id, .replay_slot=ctx->replay_slot };
1135 15 : if( FD_UNLIKELY( fd_failover_channel_send( ctx->channel, now, FD_FAILOVER_MSG_HANDOFF_REQUEST, (uchar const *)&request, sizeof(request) ) ) ) return;
1136 15 : ctx->request.sent = 1;
1137 15 : ctx->request.send_cnt = fd_ulong_sat_add( ctx->request.send_cnt, 1UL );
1138 15 : if( FD_UNLIKELY( ctx->request.quick ) ) ctx->request.until = fd_long_max( ctx->request.until, fd_long_sat_add( now, FD_FAILOVER_DEADLINE_NANOS ) );
1139 15 : LOG_CMD_STEP( ctx, "handoff %lu: asked peer boot %016lx for the identity (send %lu)", ctx->request.id, ctx->request.peer.boot_id, ctx->request.send_cnt );
1140 15 : *charge_busy = 1;
1141 15 : }
1142 :
1143 : /* Operator commands and status */
1144 :
1145 : static ulong
1146 3 : get_settled_result( fd_failover_tile_ctx_t const * ctx ) {
1147 3 : if( FD_LIKELY( holds_identity( ctx ) ) ) return FD_ADMINCTL_RESULT_SUCCESS;
1148 0 : return ctx->reply.type==(ushort)FD_FAILOVER_MSG_PROMOTE_REJECTED ? get_reject_result( ctx->reply.reason ) : FD_FAILOVER_CONTROL_RESULT_HANDOFF_ABORTED;
1149 3 : }
1150 :
1151 : /* Stages the current request with this run's address, flags and
1152 : deadline, and starts dialing.
1153 :
1154 : Returns the address it will dial, zero when none is known yet. */
1155 : static uint
1156 : stage_request( fd_failover_tile_ctx_t * ctx,
1157 : fd_adminctl_failover_req_t const * req,
1158 54 : long now ) {
1159 54 : uint addr = req->addr ? req->addr : ctx->staked_addr;
1160 54 : ctx->cmd_addr = req->addr;
1161 54 : ctx->cmd_port = req->port;
1162 54 : ctx->request.addr = addr;
1163 54 : ctx->request.quick = !!( req->flags & FD_ADMINCTL_FAILOVER_FLAG_TAKEOVER );
1164 54 : ctx->request.until = fd_long_sat_add( now, ctx->request.quick && !ctx->request.send_cnt ? FD_FAILOVER_FORCE_WAIT_NANOS : FD_FAILOVER_DEADLINE_NANOS );
1165 54 : ctx->request.sent = 0;
1166 54 : ctx->stuck = 0;
1167 54 : fd_failover_channel_init_dialer( ctx->channel, addr, get_dial_port( ctx ) );
1168 54 : return addr;
1169 54 : }
1170 :
1171 : /* `failover promote` asks the machine running the identity for it.
1172 : Opens a handoff request, to be driven by poll_peer().
1173 :
1174 : Returns SUCCESS when the command is settled and
1175 : FD_FAILOVER_CMD_PENDING when it waits for the outcome. */
1176 : static ulong
1177 : run_handoff_cmd( fd_failover_tile_ctx_t * ctx,
1178 : fd_adminctl_failover_req_t const * req,
1179 66 : long now ) {
1180 66 : int paused = is_request_paused( ctx ) && !is_switch_pending( ctx );
1181 66 : if( FD_UNLIKELY( paused && ctx->request.peer_restarted ) ) {
1182 6 : if( ctx->action==FD_FAILOVER_ACTION_HANDOFF_WAIT_RESULT ) {
1183 3 : FD_LOG_WARNING(( "handoff %lu: the authenticated peer changed before it confirmed our answer, ending the confirmation wait", ctx->request.id ));
1184 3 : ulong result = get_settled_result( ctx );
1185 3 : end_request( ctx, now, holds_identity( ctx ) ? FD_FAILOVER_HANDOFF_TAKEN : FD_FAILOVER_HANDOFF_DECLINED );
1186 3 : return result;
1187 3 : }
1188 3 : FD_LOG_NOTICE(( "handoff %lu was bound to peer boot %016lx, the authenticated peer changed, ending it and starting a new handoff", ctx->request.id, ctx->request.peer.boot_id ));
1189 3 : end_request( ctx, now, FD_FAILOVER_HANDOFF_RESTARTED );
1190 3 : paused = 0;
1191 3 : }
1192 63 : if( FD_UNLIKELY( paused ) ) {
1193 24 : uint addr = stage_request( ctx, req, now );
1194 24 : if( FD_LIKELY( addr ) ) {
1195 18 : LOG_CMD_STEP( ctx, "handoff %lu: resuming at `" FD_IP4_ADDR_FMT ":%hu` using %s", ctx->request.id, FD_IP4_ADDR_FMT_ARGS( ctx->request.addr ), get_dial_port( ctx ),
1196 18 : ctx->cmd_addr ? "`--address`" : "gossip" );
1197 18 : } else {
1198 6 : LOG_CMD_STEP( ctx, "handoff %lu: resuming once gossip shows the machine running the identity, pass --address to dial one now", ctx->request.id );
1199 6 : }
1200 24 : return ctx->action==FD_FAILOVER_ACTION_HANDOFF_WAIT_RESULT ? get_settled_result( ctx ) : FD_FAILOVER_CMD_PENDING;
1201 24 : }
1202 39 : if( FD_UNLIKELY( ctx->action==FD_FAILOVER_ACTION_DEMOTE_WAIT_ACK && !is_switch_pending( ctx ) ) ) return FD_FAILOVER_CONTROL_RESULT_HANDED_OFF;
1203 36 : if( FD_UNLIKELY( ctx->action!=FD_FAILOVER_ACTION_IDLE || is_switch_pending( ctx ) ) ) return FD_FAILOVER_CONTROL_RESULT_IN_PROGRESS;
1204 36 : if( FD_UNLIKELY( holds_identity( ctx ) ) ) return FD_FAILOVER_CONTROL_RESULT_ALREADY_HOLDS;
1205 33 : if( FD_UNLIKELY( ctx->installed_ranked ) ) return FD_FAILOVER_CONTROL_RESULT_HOLDS_OTHER;
1206 30 : fd_memset( &ctx->request, 0, sizeof(ctx->request) );
1207 30 : ctx->request.id = ctx->handoff_base+(++ctx->handoff_cnt);
1208 30 : if( FD_UNLIKELY( !ctx->request.id ) ) ctx->request.id = ++ctx->handoff_cnt;
1209 30 : ctx->session.close_after_send = 0;
1210 30 : uint addr = stage_request( ctx, req, now );
1211 30 : if( FD_LIKELY( addr ) ) {
1212 30 : LOG_CMD_STEP( ctx, "handoff %lu: requesting handoff from the machine running the identity at `" FD_IP4_ADDR_FMT ":%hu` using %s",
1213 30 : ctx->request.id, FD_IP4_ADDR_FMT_ARGS( addr ), get_dial_port( ctx ), ctx->cmd_addr ? "`--address`" : "gossip" );
1214 30 : } else {
1215 0 : LOG_CMD_STEP( ctx, "handoff %lu: waiting for gossip to show the machine running the identity, or for `--address` once this command returns", ctx->request.id );
1216 0 : }
1217 :
1218 30 : set_action( ctx, FD_FAILOVER_ACTION_HANDOFF_WAIT_PEER );
1219 30 : return FD_FAILOVER_CMD_PENDING;
1220 33 : }
1221 :
1222 : /* `failover takeover` (after the handoff failed) takes the identity
1223 : without coordination, with --force even from a machine that may be
1224 : live. */
1225 : static ulong
1226 : run_promote_cmd( fd_failover_tile_ctx_t * ctx,
1227 : fd_adminctl_failover_req_t const * req,
1228 108 : long now ) {
1229 108 : int force = !!( req->flags & FD_ADMINCTL_FAILOVER_FLAG_FORCE );
1230 108 : if( FD_UNLIKELY( !( req->flags & FD_ADMINCTL_FAILOVER_FLAG_TAKEOVER ) ) ) return FD_ADMINCTL_RESULT_UNKNOWN_COMMAND;
1231 102 : static uchar const no_target[ 32 ] = { 0 };
1232 102 : if( FD_UNLIKELY( !fd_memeq( req->target, no_target, 32UL ) && !fd_memeq( req->target, ctx->hello.staked_pubkey, 32UL ) ) ) return FD_FAILOVER_CONTROL_RESULT_IDENTITY_CHANGED;
1233 99 : ulong result = check_promotion( ctx );
1234 99 : if( FD_UNLIKELY( result!=FD_ADMINCTL_RESULT_SUCCESS ) ) return result;
1235 96 : if( FD_UNLIKELY( !force && ctx->handoff.delivery!=FD_FAILOVER_DELIVERY_NONE ) ) {
1236 24 : return FD_FAILOVER_CONTROL_RESULT_HANDED_OFF;
1237 24 : }
1238 72 : if( FD_UNLIKELY( !force && is_holder_live( ctx, now ) ) ) return FD_FAILOVER_CONTROL_RESULT_HOLDER_LIVE;
1239 45 : if( ctx->request.id && is_requesting( ctx ) ) end_request( ctx, now, FD_FAILOVER_HANDOFF_CANCELLED );
1240 45 : int gave_up = ctx->handoff.delivery!=FD_FAILOVER_DELIVERY_NONE;
1241 45 : if( gave_up ) {
1242 6 : ctx->peer_history.valid = 0;
1243 6 : resolve_handoff( ctx, FD_FAILOVER_HANDOFF_CANCELLED );
1244 6 : }
1245 45 : start_promotion( ctx, FD_FAILOVER_SOURCE_EMPTY, NULL, 0UL, req->flags & (FD_ADMINCTL_FAILOVER_FLAG_TAKEOVER|FD_ADMINCTL_FAILOVER_FLAG_FORCE) );
1246 45 : ctx->promotion.gave_up = gave_up;
1247 45 : return FD_FAILOVER_CMD_PENDING;
1248 72 : }
1249 :
1250 : static ulong
1251 : run_preflight_cmd( fd_failover_tile_ctx_t const * ctx,
1252 9 : fd_adminctl_failover_req_t const * req ) {
1253 9 : if( fd_memeq( req->target, ctx->hello.staked_pubkey, 32UL ) ) return FD_ADMINCTL_RESULT_SUCCESS;
1254 9 : if( FD_UNLIKELY( is_switch_pending( ctx ) || ( ctx->action!=FD_FAILOVER_ACTION_IDLE && !is_request_paused( ctx ) ) ) ) {
1255 0 : return FD_FAILOVER_CONTROL_RESULT_IN_PROGRESS;
1256 0 : }
1257 9 : if( FD_UNLIKELY( holds_identity( ctx ) && ctx->own_voted ) ) return FD_FAILOVER_CONTROL_RESULT_HOLDS_OTHER;
1258 9 : return FD_ADMINCTL_RESULT_SUCCESS;
1259 9 : }
1260 :
1261 : /* Runs an operator command. Returns either its result or
1262 : FD_FAILOVER_CMD_PENDING. */
1263 : static ulong
1264 : apply_control( fd_failover_tile_ctx_t * ctx,
1265 : fd_adminctl_failover_req_t const * req,
1266 183 : long now ) {
1267 183 : if( FD_UNLIKELY( ctx->cmd_nonce ) ) return FD_FAILOVER_CONTROL_RESULT_BUSY;
1268 183 : switch( req->cmd ) {
1269 66 : case FD_ADMINCTL_FAILOVER_CMD_HANDOFF: return run_handoff_cmd( ctx, req, now );
1270 108 : case FD_ADMINCTL_FAILOVER_CMD_PROMOTE: return run_promote_cmd( ctx, req, now );
1271 9 : case FD_ADMINCTL_FAILOVER_CMD_PREFLIGHT: return run_preflight_cmd( ctx, req );
1272 0 : default: return FD_ADMINCTL_RESULT_UNSUPPORTED;
1273 183 : }
1274 183 : }
1275 :
1276 : /* Answers one bus frame with the result for nonce. */
1277 : static void
1278 : publish_bus_result( fd_failover_tile_ctx_t * ctx,
1279 : fd_stem_context_t * stem,
1280 : ulong nonce,
1281 3 : ulong result ) {
1282 3 : fd_failover_bus_msg_t * out = fd_chunk_to_laddr( ctx->admin_out_mem, ctx->admin_out_chunk );
1283 3 : fd_memset( out, 0, sizeof(*out) );
1284 3 : out->nonce = nonce;
1285 3 : out->result = result;
1286 3 : FD_STORE( ulong, out->payload, ctx->last_vote_slot );
1287 3 : ulong tspub = fd_frag_meta_ts_comp( fd_tickcount() );
1288 3 : fd_stem_publish( stem, ctx->admin_out_idx, FD_FAILOVER_BUS_RESPONSE, ctx->admin_out_chunk, sizeof(*out), 0UL, tspub, tspub );
1289 3 : ctx->admin_out_chunk = fd_dcache_compact_next( ctx->admin_out_chunk, sizeof(*out), ctx->admin_out_chunk0, ctx->admin_out_wmark );
1290 3 : }
1291 :
1292 : /* Publishes the outcome finish_cmd recorded. */
1293 : static void
1294 : publish_cmd_answer( fd_failover_tile_ctx_t * ctx,
1295 0 : fd_stem_context_t * stem ) {
1296 0 : publish_bus_result( ctx, stem, ctx->cmd_nonce, ctx->cmd_result );
1297 0 : ctx->cmd_nonce = 0UL;
1298 0 : ctx->cmd_done = 0;
1299 0 : ctx->cmd_step_tail = ctx->cmd_step_head;
1300 0 : }
1301 :
1302 : /* Publishes the oldest unsent step of the pending command. */
1303 : static void
1304 : publish_cmd_step( fd_failover_tile_ctx_t * ctx,
1305 0 : fd_stem_context_t * stem ) {
1306 0 : fd_failover_bus_msg_t * out = fd_chunk_to_laddr( ctx->admin_out_mem, ctx->admin_out_chunk );
1307 0 : fd_memset( out, 0, sizeof(*out) );
1308 0 : out->nonce = ctx->cmd_nonce;
1309 0 : fd_cstr_ncpy( (char *)out->payload, ctx->cmd_step[ ctx->cmd_step_tail%FD_FAILOVER_STEP_CNT ], FD_FAILOVER_BUS_STEP_MAX );
1310 0 : ulong tspub = fd_frag_meta_ts_comp( fd_tickcount() );
1311 0 : fd_stem_publish( stem, ctx->admin_out_idx, FD_FAILOVER_BUS_STEP, ctx->admin_out_chunk, sizeof(*out), 0UL, tspub, tspub );
1312 0 : ctx->admin_out_chunk = fd_dcache_compact_next( ctx->admin_out_chunk, sizeof(*out), ctx->admin_out_chunk0, ctx->admin_out_wmark );
1313 0 : ctx->cmd_step_tail++;
1314 0 : }
1315 :
1316 : /* Runs one operator command. The admin tile validated the ABI already. */
1317 : static void
1318 : serve_bus_request( fd_failover_tile_ctx_t * ctx,
1319 : fd_stem_context_t * stem,
1320 6 : long now ) {
1321 6 : fd_adminctl_failover_req_t req;
1322 6 : fd_memcpy( &req, ctx->bus_req.payload, sizeof(req) );
1323 6 : char const * cmd_name = fd_adminctl_failover_cmd_typed( req.cmd, req.flags );
1324 6 : ulong step_head = ctx->cmd_step_head;
1325 6 : ulong result = apply_control( ctx, &req, now );
1326 6 : if( FD_UNLIKELY( result==FD_FAILOVER_CMD_PENDING ) ) {
1327 3 : ctx->cmd_nonce = ctx->bus_req.nonce;
1328 3 : ctx->cmd_step_tail = step_head;
1329 3 : return;
1330 3 : }
1331 3 : if( FD_UNLIKELY( result==FD_FAILOVER_CONTROL_RESULT_ALREADY_HOLDS ) ) FD_LOG_NOTICE(( "`%s` done, this machine already runs the identity", cmd_name ));
1332 3 : else if( FD_UNLIKELY( result!=FD_ADMINCTL_RESULT_SUCCESS ) ) FD_LOG_WARNING(( "`%s` refused (result %#lx)", cmd_name, result ));
1333 3 : publish_bus_result( ctx, stem, ctx->bus_req.nonce, result );
1334 3 : }
|
