Line data    Source code

       1             : #include "fd_fec_chainer.h"
       2             : 
       3             : void *
       4           0 : fd_fec_chainer_new( void * shmem, ulong fec_max, ulong seed ) {
       5             : 
       6           0 :   if( FD_UNLIKELY( !shmem ) ) {
       7           0 :     FD_LOG_WARNING(( "NULL mem" ));
       8           0 :     return NULL;
       9           0 :   }
      10             : 
      11           0 :   if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)shmem, fd_fec_chainer_align() ) ) ) {
      12           0 :     FD_LOG_WARNING(( "misaligned mem" ));
      13           0 :     return NULL;
      14           0 :   }
      15             : 
      16           0 :   ulong footprint = fd_fec_chainer_footprint( fec_max );
      17           0 :   if( FD_UNLIKELY( !footprint ) ) {
      18           0 :     FD_LOG_WARNING(( "bad fec_max (%lu)", fec_max ));
      19           0 :     return NULL;
      20           0 :   }
      21             : 
      22           0 :   fd_wksp_t * wksp = fd_wksp_containing( shmem );
      23           0 :   if( FD_UNLIKELY( !wksp ) ) {
      24           0 :     FD_LOG_WARNING(( "shmem must be part of a workspace" ));
      25           0 :     return NULL;
      26           0 :   }
      27             : 
      28           0 :   fd_memset( shmem, 0, footprint );
      29             : 
      30           0 :   fd_fec_chainer_t * chainer;
      31           0 :   int lg_fec_max = fd_ulong_find_msb( fd_ulong_pow2_up( fec_max ) );
      32             : 
      33           0 :   FD_SCRATCH_ALLOC_INIT( l, shmem );
      34           0 :   chainer         = FD_SCRATCH_ALLOC_APPEND( l, fd_fec_chainer_align(),  sizeof( fd_fec_chainer_t )               );
      35           0 :   void * ancestry = FD_SCRATCH_ALLOC_APPEND( l, fd_fec_ancestry_align(), fd_fec_ancestry_footprint( fec_max )     );
      36           0 :   void * frontier = FD_SCRATCH_ALLOC_APPEND( l, fd_fec_frontier_align(), fd_fec_frontier_footprint( fec_max )     );
      37           0 :   void * orphaned = FD_SCRATCH_ALLOC_APPEND( l, fd_fec_orphaned_align(), fd_fec_orphaned_footprint( fec_max )     );
      38           0 :   void * pool     = FD_SCRATCH_ALLOC_APPEND( l, fd_fec_pool_align(),     fd_fec_pool_footprint( fec_max )         );
      39           0 :   void * parents  = FD_SCRATCH_ALLOC_APPEND( l, fd_fec_parents_align(),  fd_fec_parents_footprint( lg_fec_max )   );
      40           0 :   void * children = FD_SCRATCH_ALLOC_APPEND( l, fd_fec_children_align(), fd_fec_children_footprint( lg_fec_max )  );
      41           0 :   void * queue    = FD_SCRATCH_ALLOC_APPEND( l, fd_fec_queue_align(),    fd_fec_queue_footprint( fec_max )        );
      42           0 :   void * out      = FD_SCRATCH_ALLOC_APPEND( l, fd_fec_out_align(),      fd_fec_out_footprint( fec_max )          );
      43           0 :   FD_TEST( FD_SCRATCH_ALLOC_FINI( l, fd_fec_chainer_align() ) == (ulong)shmem + footprint );
      44             : 
      45           0 :   chainer->ancestry = fd_fec_ancestry_new( ancestry, fec_max, seed );
      46           0 :   chainer->frontier = fd_fec_frontier_new( frontier, fec_max, seed );
      47           0 :   chainer->orphaned = fd_fec_orphaned_new( orphaned, fec_max, seed );
      48           0 :   chainer->pool     = fd_fec_pool_new    ( pool,     fec_max       );
      49           0 :   chainer->parents  = fd_fec_parents_new ( parents,  lg_fec_max    );
      50           0 :   chainer->children = fd_fec_children_new( children, lg_fec_max    );
      51           0 :   chainer->queue    = fd_fec_queue_new   ( queue,    fec_max       );
      52           0 :   chainer->out      = fd_fec_out_new     ( out,      fec_max       );
      53             : 
      54           0 :   return shmem;
      55           0 : }
      56             : 
      57             : fd_fec_chainer_t *
      58           0 : fd_fec_chainer_join( void * shfec_chainer ) {
      59           0 :   fd_fec_chainer_t * chainer = (fd_fec_chainer_t *)shfec_chainer;
      60             : 
      61           0 :   if( FD_UNLIKELY( !chainer ) ) {
      62           0 :     FD_LOG_WARNING(( "NULL chainer" ));
      63           0 :     return NULL;
      64           0 :   }
      65             : 
      66           0 :   chainer->ancestry = fd_fec_ancestry_join( chainer->ancestry );
      67           0 :   chainer->frontier = fd_fec_frontier_join( chainer->frontier );
      68           0 :   chainer->orphaned = fd_fec_orphaned_join( chainer->orphaned );
      69           0 :   chainer->pool     = fd_fec_pool_join    ( chainer->pool     );
      70           0 :   chainer->parents  = fd_fec_parents_join ( chainer->parents  );
      71           0 :   chainer->children = fd_fec_children_join( chainer->children );
      72           0 :   chainer->queue    = fd_fec_queue_join   ( chainer->queue    );
      73           0 :   chainer->out      = fd_fec_out_join     ( chainer->out      );
      74             : 
      75           0 :   return chainer;
      76           0 : }
      77             : 
      78             : void *
      79           0 : fd_fec_chainer_leave( fd_fec_chainer_t * chainer ) {
      80             : 
      81           0 :   if( FD_UNLIKELY( !chainer ) ) {
      82           0 :     FD_LOG_WARNING(( "NULL chainer" ));
      83           0 :     return NULL;
      84           0 :   }
      85             : 
      86           0 :   return (void *)chainer;
      87           0 : }
      88             : 
      89             : void *
      90           0 : fd_fec_chainer_delete( void * shchainer ) {
      91           0 :   fd_fec_chainer_t * chainer = (fd_fec_chainer_t *)shchainer;
      92             : 
      93           0 :   if( FD_UNLIKELY( !chainer ) ) {
      94           0 :     FD_LOG_WARNING(( "NULL chainer" ));
      95           0 :     return NULL;
      96           0 :   }
      97             : 
      98           0 :   if( FD_UNLIKELY( !fd_ulong_is_aligned((ulong)chainer, fd_fec_chainer_align() ) ) ) {
      99           0 :     FD_LOG_WARNING(( "misaligned chainer" ));
     100           0 :     return NULL;
     101           0 :   }
     102             : 
     103           0 :   return chainer;
     104           0 : }
     105             : 
     106             : fd_fec_ele_t *
     107           0 : fd_fec_chainer_init( fd_fec_chainer_t * chainer, ulong slot, fd_hash_t const * merkle_root ) {
     108           0 :   FD_TEST( fd_fec_pool_free( chainer->pool ) );
     109           0 :   fd_fec_ele_t * root = fd_fec_pool_ele_acquire( chainer->pool );
     110           0 :   FD_TEST( root );
     111           0 :   root->key           = slot << 32 | ( 0 ); // maintain invariant that no fec_set_idx=UINT_MAX lives in pool_ele
     112           0 :   root->slot          = slot;
     113           0 :   root->fec_set_idx   = 0;
     114           0 :   root->data_cnt      = 0;
     115           0 :   root->data_complete = 1;
     116           0 :   root->slot_complete = 1;
     117           0 :   root->parent_off    = 0;
     118           0 :   memcpy( root->merkle_root, merkle_root, FD_SHRED_MERKLE_ROOT_SZ );
     119           0 :   memset( root->chained_merkle_root, 0, FD_SHRED_MERKLE_ROOT_SZ );
     120             : 
     121             :   /* For the next slot that chains off the init slot, it will use the
     122             :      parent_map and key with slot | UINT_MAX to look for its parent, so
     123             :      we need to provide the artificial parent link between the last
     124             :      fec_set_idx and UINT_MAX. This way it can query for
     125             :      UINT_MAX -> UINT_MAX-1 -> get_pool_ele(UINT_MAX-1)*/
     126             : 
     127           0 :   fd_fec_parent_t * p = fd_fec_parents_insert( chainer->parents, slot << 32 | UINT_MAX );
     128           0 :   p->parent_key       = (slot << 32) | ( 0 );
     129             : 
     130           0 :   fd_fec_frontier_ele_insert( chainer->frontier, root, chainer->pool );
     131           0 :   chainer->root = fd_fec_pool_idx( chainer->pool, root );
     132           0 :   return root;
     133           0 : }
     134             : 
     135             : void *
     136           0 : fd_fec_chainer_fini( fd_fec_chainer_t * chainer ) {
     137           0 :   return (void *)chainer;
     138           0 : }
     139             : 
     140             : FD_FN_PURE fd_fec_ele_t *
     141           0 : fd_fec_chainer_query( fd_fec_chainer_t * chainer, ulong slot, uint fec_set_idx ) {
     142           0 :   ulong key = slot << 32 | fec_set_idx;
     143           0 :   fd_fec_ele_t * fec;
     144           0 :   fec =                  fd_fec_frontier_ele_query( chainer->frontier, &key, NULL, chainer->pool );
     145           0 :   fec = fd_ptr_if( !fec, fd_fec_ancestry_ele_query( chainer->ancestry, &key, NULL, chainer->pool ), fec );
     146           0 :   fec = fd_ptr_if( !fec, fd_fec_orphaned_ele_query( chainer->orphaned, &key, NULL, chainer->pool ), fec );
     147           0 :   return fec;
     148           0 : }
     149             : 
     150             : static fd_fec_ele_t *
     151           0 : fd_fec_chainer_remove( fd_fec_chainer_t * chainer, ulong key ) {
     152           0 :   fd_fec_ele_t * fec = fd_fec_frontier_ele_remove( chainer->frontier, &key, NULL, chainer->pool );
     153           0 :   fec = fd_ptr_if( !fec, fd_fec_ancestry_ele_remove( chainer->ancestry, &key, NULL, chainer->pool ), fec );
     154           0 :   fec = fd_ptr_if( !fec, fd_fec_orphaned_ele_remove( chainer->orphaned, &key, NULL, chainer->pool ), fec );
     155           0 :   return fec;
     156           0 : }
     157             : 
     158             : static int
     159           0 : is_last_fec( ulong key ){
     160           0 :   return (uint)fd_ulong_extract( key, 0, 31 ) == UINT_MAX;
     161           0 : }
     162             : 
     163             : static void
     164           0 : link_orphans( fd_fec_chainer_t * chainer ) {
     165           0 :   while( FD_LIKELY( !fd_fec_queue_empty( chainer->queue ) ) ) {
     166           0 :     ulong          key = fd_fec_queue_pop_head( chainer->queue );
     167           0 :     fd_fec_ele_t * ele = fd_fec_orphaned_ele_query( chainer->orphaned, &key, NULL, chainer->pool );
     168             : 
     169           0 :     if( FD_LIKELY( !ele ) ) continue;
     170             : 
     171             :     /* Query for the parent_key. */
     172             : 
     173           0 :     fd_fec_parent_t * parent_key = fd_fec_parents_query( chainer->parents, key, NULL );
     174           0 :     if( FD_UNLIKELY( !parent_key ) ) continue; /* still orphaned */
     175             : 
     176             :     /* If the parent is in the frontier, remove the parent and insert
     177             :        into ancestry.  Otherwise check for parent in ancestry. */
     178             : 
     179           0 :     if( FD_UNLIKELY( is_last_fec( parent_key->parent_key ) ) ) {
     180             : 
     181             :       /* If the parent was the last fec of the previous slot, the
     182             :          parent_key will be UINT_MAX. Need to query for the actual
     183             :          fec_set_idx of the last FEC. This is the double query */
     184             : 
     185           0 :       parent_key = fd_fec_parents_query( chainer->parents, parent_key->parent_key, NULL );
     186           0 :       if( !parent_key ) continue; /* still orphaned */
     187           0 :     }
     188             : 
     189           0 :     fd_fec_ele_t * parent = fd_fec_frontier_ele_remove( chainer->frontier, &parent_key->parent_key, NULL, chainer->pool );
     190           0 :     if( FD_LIKELY( parent ) ) fd_fec_ancestry_ele_insert( chainer->ancestry, parent, chainer->pool );
     191           0 :     else parent = fd_fec_ancestry_ele_query( chainer->ancestry, &parent_key->parent_key, NULL, chainer->pool );
     192             : 
     193             :     /* If the parent is not in frontier or ancestry, ele is still
     194             :        orphaned. Note it is possible to have inserted ele's parent but
     195             :        have ele still be orphaned, because parent is also orphaned. */
     196             : 
     197           0 :     if( FD_UNLIKELY( !parent ) ) continue;
     198             : 
     199             :     /* Remove ele from orphaned. */
     200             : 
     201           0 :     fd_fec_ele_t * remove = fd_fec_orphaned_ele_remove( chainer->orphaned, &ele->key, NULL, chainer->pool );
     202           0 :     FD_TEST( remove == ele );
     203             : 
     204             :     /* Verify the chained merkle root. */
     205             : 
     206           0 :     fd_hash_t null = { 0 };
     207             :     // FD_LOG_NOTICE(( "parent %lu %u %s ele %lu %u %s", parent->slot, parent->fec_set_idx, FD_BASE58_ENC_32_ALLOCA( parent->merkle_root ), ele->slot, ele->fec_set_idx, FD_BASE58_ENC_32_ALLOCA( ele->merkle_root ) ));
     208           0 :     if ( FD_UNLIKELY( memcmp( parent->merkle_root, ele->chained_merkle_root, FD_SHRED_MERKLE_ROOT_SZ ) ) &&
     209           0 :                       memcmp( parent->merkle_root, &null,                    FD_SHRED_MERKLE_ROOT_SZ ) ) {
     210           0 :       FD_LOG_ERR(( "bad chained merkle root. slot: %lu fec_set_idx: %u merkle_root: %s chained_merkle_root: %s", ele->slot, ele->fec_set_idx, FD_BASE58_ENC_32_ALLOCA( parent->merkle_root ), FD_BASE58_ENC_32_ALLOCA( ele->chained_merkle_root ) ));
     211           0 :     }
     212             : 
     213             :     /* Insert into frontier (ele is either advancing a fork or starting
     214             :        a new fork) and deliver to `out`. */
     215             : 
     216           0 :     fd_fec_frontier_ele_insert( chainer->frontier, ele, chainer->pool );
     217           0 :     fd_hash_t merkle_root;
     218           0 :     memcpy( merkle_root.uc, ele->merkle_root, FD_SHRED_MERKLE_ROOT_SZ );
     219           0 :     fd_hash_t chained_root;
     220           0 :     memcpy( chained_root.uc, ele->chained_merkle_root, FD_SHRED_MERKLE_ROOT_SZ );
     221           0 :     fd_fec_out_t out = {
     222           0 :       .err            = FD_FEC_CHAINER_SUCCESS,
     223           0 :       .slot           = ele->slot,
     224           0 :       .parent_off     = ele->parent_off,
     225           0 :       .fec_set_idx    = ele->fec_set_idx,
     226           0 :       .data_cnt       = ele->data_cnt,
     227           0 :       .data_complete  = ele->data_complete,
     228           0 :       .slot_complete  = ele->slot_complete,
     229           0 :       .merkle_root    = merkle_root,
     230           0 :       .chained_root   = chained_root
     231           0 :     };
     232           0 :     fd_fec_out_push_tail( chainer->out, out );
     233             : 
     234             :     /* Check whether any of ele's children are orphaned and can be
     235             :        chained into the frontier. */
     236             : 
     237             :     /* TODO this BFS can be structured differently without using any
     238             :        additional memory by reusing ele->next. */
     239             : 
     240           0 :     if( FD_UNLIKELY( ele->slot_complete ) ) {
     241           0 :       fd_fec_children_t * fec_children = fd_fec_children_query( chainer->children, ele->slot, NULL );
     242           0 :       if( FD_UNLIKELY( !fec_children ) ) continue;
     243           0 :       for( ulong off = fd_slot_child_offs_const_iter_init( fec_children->child_offs );
     244           0 :            !fd_slot_child_offs_const_iter_done( off );
     245           0 :            off = fd_slot_child_offs_const_iter_next( fec_children->child_offs, off ) ) {
     246           0 :         ulong child_key = (ele->slot + off) << 32; /* always fec_set_idx 0 */
     247           0 :         fd_fec_ele_t * child = fd_fec_orphaned_ele_query( chainer->orphaned, &child_key, NULL, chainer->pool );
     248           0 :         if( FD_LIKELY( child ) ) {
     249           0 :           fd_fec_queue_push_tail( chainer->queue, child_key );
     250           0 :         }
     251           0 :       }
     252           0 :     } else {
     253           0 :       ulong child_key = (ele->slot << 32) | (ele->key + ele->data_cnt);
     254           0 :       fd_fec_queue_push_tail( chainer->queue, child_key );
     255           0 :     }
     256           0 :   }
     257           0 : }
     258             : 
     259             : fd_fec_ele_t *
     260             : fd_fec_chainer_insert( fd_fec_chainer_t * chainer,
     261             :                        ulong              slot,
     262             :                        uint               fec_set_idx,
     263             :                        ushort             data_cnt,
     264             :                        int                data_complete,
     265             :                        int                slot_complete,
     266             :                        ushort             parent_off,
     267             :                        fd_hash_t const *  merkle_root,
     268           0 :                        fd_hash_t const *  chained_merkle_root ) {
     269           0 :   ulong key = slot << 32 | fec_set_idx;
     270             :   // FD_LOG_NOTICE(( "inserting %lu %u %u %d %d", slot, fec_set_idx, data_cnt, data_complete, slot_complete ));
     271             : 
     272           0 :   if( FD_UNLIKELY( fd_fec_chainer_query( chainer, slot, fec_set_idx ) ) ) {
     273           0 :     FD_LOG_ERR(( "equivocating FEC. slot: %lu fec_set_idx: %u merkle_root: %s chained_merkle_root: %s", slot, fec_set_idx, FD_BASE58_ENC_32_ALLOCA( merkle_root ), FD_BASE58_ENC_32_ALLOCA( chained_merkle_root ) ));
     274           0 :   }
     275             : 
     276           0 : # if FD_FEC_CHAINER_USE_HANDHOLDING
     277           0 :   FD_TEST( fd_fec_pool_free( chainer->pool ) ); /* FIXME lru? */
     278           0 :   FD_TEST( fd_fec_parents_key_cnt( chainer->parents ) < fd_fec_parents_key_max( chainer->parents ) );
     279           0 :   FD_TEST( fd_fec_children_key_cnt( chainer->children ) < fd_fec_children_key_max( chainer->children ) );
     280           0 : # endif
     281             : 
     282           0 :   fd_fec_ele_t * ele = fd_fec_pool_ele_acquire( chainer->pool );
     283           0 :   ele->key           = key;
     284           0 :   ele->slot          = slot;
     285           0 :   ele->fec_set_idx   = fec_set_idx;
     286           0 :   ele->data_cnt      = data_cnt;
     287           0 :   ele->data_complete = data_complete;
     288           0 :   ele->slot_complete = slot_complete;
     289           0 :   ele->parent_off    = parent_off;
     290           0 :   memcpy( ele->merkle_root, merkle_root, FD_SHRED_MERKLE_ROOT_SZ );
     291           0 :   memcpy( ele->chained_merkle_root, chained_merkle_root, FD_SHRED_MERKLE_ROOT_SZ );
     292             : 
     293             :   /* If it is the first FEC set, derive and insert parent_key->key into
     294             :      the parents map and parent_slot->slot into the children map. */
     295             : 
     296           0 :   if( FD_UNLIKELY( fec_set_idx == 0 ) ) {
     297           0 :     ulong parent_slot = slot - parent_off;
     298             : 
     299           0 :     fd_fec_parent_t * parent_key = fd_fec_parents_insert( chainer->parents, key );
     300           0 :     parent_key->parent_key       = (parent_slot << 32) | UINT_MAX;
     301             : 
     302           0 :     fd_fec_children_t * fec_children = fd_fec_children_query( chainer->children, parent_slot, NULL );
     303           0 :     if( FD_LIKELY( !fec_children ) ) {
     304           0 :       fec_children = fd_fec_children_insert( chainer->children, parent_slot );
     305           0 :       fd_slot_child_offs_null( fec_children->child_offs );
     306           0 :     }
     307           0 :     fd_slot_child_offs_insert( fec_children->child_offs, parent_off );
     308           0 :   }
     309             : 
     310             :   /* Derive and insert the child_key->key into the parents map. */
     311             : 
     312           0 :   ulong child_key = ( slot << 32 ) | ( fec_set_idx + data_cnt );
     313           0 :   if( FD_UNLIKELY( slot_complete ) ) {
     314             : 
     315             :     /* This is the last FEC set. There is a special case to add
     316             :        a UINT_MAX -> fec_set_idx link because the child slots will chain
     317             :        off of UINT_MAX, but the pool_ele will be keyed on fec_set_idx. */
     318             : 
     319           0 :     fd_fec_parent_t * parent = fd_fec_parents_insert( chainer->parents, slot << 32 | UINT_MAX );
     320           0 :     parent->parent_key       = key;
     321             : 
     322           0 :   } else {
     323             : 
     324             :     /* This is not the last FEC set. */
     325             :     /* Key the child to point to ele (child's parent). */
     326             : 
     327           0 :     if( !fd_fec_parents_query( chainer->parents, child_key, NULL ) ) {
     328           0 :       fd_fec_parent_t * parent = fd_fec_parents_insert( chainer->parents, child_key );
     329           0 :       parent->parent_key = key;
     330           0 :     } else {
     331           0 :       fd_fec_parent_t * parent = fd_fec_parents_query( chainer->parents, child_key, NULL );
     332           0 :       if( parent->parent_key != key ) {
     333           0 :         FD_LOG_ERR(( "inconsistent keys %lu %u %lu %u", slot, fec_set_idx, parent->parent_key >> 32, (uint)parent->parent_key ));
     334           0 :       }
     335           0 :     }
     336           0 :   }
     337             : 
     338             :   /* Push ele into the BFS deque and the orphaned map for processing. */
     339             : 
     340           0 :   fd_fec_queue_push_tail( chainer->queue, key );
     341           0 :   fd_fec_orphaned_ele_insert( chainer->orphaned, ele, chainer->pool );
     342             : 
     343           0 :   link_orphans( chainer );
     344             : 
     345           0 :   return ele;
     346           0 : }
     347             : 
     348             : void
     349           0 : fd_fec_chainer_publish( fd_fec_chainer_t * chainer, ulong new_root_slot ) {
     350           0 :   fd_fec_ele_t * old_root = fd_fec_pool_ele( chainer->pool, chainer->root );
     351           0 :   fd_fec_ele_t * new_root = fd_fec_chainer_query( chainer, new_root_slot, 0 );
     352             : 
     353           0 :   FD_TEST( old_root );
     354           0 :   if( FD_UNLIKELY( !new_root ) ) {
     355             :     /* It is possible to not have a fec element for the new root during
     356             :     second incremental snapshot load */
     357             : 
     358           0 :     new_root = fd_fec_pool_ele_acquire( chainer->pool );
     359           0 :     new_root->key           = new_root_slot << 32; /* fec_set_idx 0, similar to chainer_init */
     360           0 :     new_root->slot          = new_root_slot;
     361           0 :     new_root->fec_set_idx   = 0;
     362           0 :     new_root->data_cnt      = 0;
     363           0 :     new_root->data_complete = 1;
     364           0 :     new_root->slot_complete = 1;
     365           0 :     new_root->parent_off    = 0;
     366           0 :     memset( new_root->chained_merkle_root, 0, FD_SHRED_MERKLE_ROOT_SZ );
     367             : 
     368           0 :     fd_fec_parent_t * p = fd_fec_parents_insert( chainer->parents, new_root_slot << 32 | UINT_MAX );
     369           0 :     p->parent_key       = new_root_slot << 32;
     370             : 
     371           0 :     fd_fec_frontier_ele_insert( chainer->frontier, new_root, chainer->pool );
     372           0 :   }
     373             : 
     374             :   /* Prune children of the old root */
     375             : 
     376           0 :   fd_fec_queue_push_tail( chainer->queue, old_root->key );
     377             : 
     378           0 :   while( FD_LIKELY( !fd_fec_queue_empty( chainer->queue ) ) ) {
     379           0 :     ulong key = fd_fec_queue_pop_head( chainer->queue );
     380           0 :     fd_fec_ele_t * ele = fd_fec_chainer_query( chainer, key >> 32, (uint)key );
     381           0 :     if( FD_UNLIKELY( !ele ) ) continue;
     382             : 
     383           0 :     if( FD_UNLIKELY( ele->slot_complete ) ) {
     384           0 :       fd_fec_children_t * fec_children = fd_fec_children_query( chainer->children, ele->slot, NULL );
     385           0 :       if( FD_UNLIKELY( fec_children ) ) {
     386           0 :         for( ulong off = fd_slot_child_offs_const_iter_init( fec_children->child_offs );
     387           0 :             !fd_slot_child_offs_const_iter_done( off );
     388           0 :             off = fd_slot_child_offs_const_iter_next( fec_children->child_offs, off ) ) {
     389           0 :           ulong child_slot = ele->slot + off;
     390             : 
     391           0 :           if( FD_UNLIKELY( child_slot == new_root_slot ) ) continue;
     392             : 
     393           0 :           fd_fec_queue_push_tail( chainer->queue, child_slot << 32 | 0 );
     394           0 :         }
     395           0 :       }
     396           0 :     } else {
     397           0 :       ulong child_key = (ele->slot << 32) | (ele->key + ele->data_cnt);
     398           0 :       fd_fec_queue_push_tail( chainer->queue, child_key );
     399           0 :     }
     400             : 
     401             :     /* Remove ele from the chainer. */
     402             : 
     403           0 :     fd_fec_ele_t * remove = fd_fec_chainer_remove( chainer, ele->key );
     404           0 :     FD_TEST( remove == ele );
     405           0 :   }
     406             : 
     407             :   /* Update the root_fec */
     408             : 
     409           0 :   chainer->root = fd_fec_pool_idx( chainer->pool, new_root );
     410           0 : }