LCOV - code coverage report
Current view: top level - disco/topo - fd_topob.c (source / functions) Hit Total Coverage
Test: cov.lcov Lines: 0 479 0.0 %
Date: 2025-07-01 05:00:49 Functions: 0 12 0.0 %

          Line data    Source code
       1             : #include "fd_topob.h"
       2             : 
       3             : #include "../../util/pod/fd_pod_format.h"
       4             : #include "fd_cpu_topo.h"
       5             : 
       6             : fd_topo_t *
       7             : fd_topob_new( void * mem,
       8           0 :               char const * app_name ) {
       9           0 :   fd_topo_t * topo = (fd_topo_t *)mem;
      10             : 
      11           0 :   if( FD_UNLIKELY( !topo ) ) {
      12           0 :     FD_LOG_WARNING( ( "NULL topo" ) );
      13           0 :     return NULL;
      14           0 :   }
      15             : 
      16           0 :   if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)topo, alignof(fd_topo_t) ) ) ) {
      17           0 :     FD_LOG_WARNING( ( "misaligned topo" ) );
      18           0 :     return NULL;
      19           0 :   }
      20             : 
      21           0 :   fd_memset( topo, 0, sizeof(fd_topo_t) );
      22             : 
      23           0 :   FD_TEST( fd_pod_new( topo->props, sizeof(topo->props) ) );
      24             : 
      25           0 :   if( FD_UNLIKELY( strlen( app_name )>=sizeof(topo->app_name) ) ) FD_LOG_ERR(( "app_name too long: %s", app_name ));
      26           0 :   strncpy( topo->app_name, app_name, sizeof(topo->app_name) );
      27             : 
      28           0 :   topo->max_page_size           = FD_SHMEM_GIGANTIC_PAGE_SZ;
      29           0 :   topo->gigantic_page_threshold = 4 * FD_SHMEM_HUGE_PAGE_SZ;
      30             : 
      31           0 :   return topo;
      32           0 : }
      33             : 
      34             : void
      35             : fd_topob_wksp( fd_topo_t *  topo,
      36           0 :                char const * name ) {
      37           0 :   if( FD_UNLIKELY( !topo || !name || !strlen( name ) ) ) FD_LOG_ERR(( "NULL args" ));
      38           0 :   if( FD_UNLIKELY( strlen( name )>=sizeof(topo->workspaces[ topo->wksp_cnt ].name ) ) ) FD_LOG_ERR(( "wksp name too long: %s", name ));
      39           0 :   if( FD_UNLIKELY( topo->wksp_cnt>=FD_TOPO_MAX_WKSPS ) ) FD_LOG_ERR(( "too many workspaces" ));
      40             : 
      41           0 :   fd_topo_wksp_t * wksp = &topo->workspaces[ topo->wksp_cnt ];
      42           0 :   strncpy( wksp->name, name, sizeof(wksp->name) );
      43           0 :   wksp->id = topo->wksp_cnt;
      44           0 :   topo->wksp_cnt++;
      45           0 : }
      46             : 
      47             : fd_topo_obj_t *
      48             : fd_topob_obj( fd_topo_t *  topo,
      49             :               char const * obj_name,
      50           0 :               char const * wksp_name ) {
      51           0 :   if( FD_UNLIKELY( !topo || !obj_name || !wksp_name ) ) FD_LOG_ERR(( "NULL args" ));
      52           0 :   if( FD_UNLIKELY( strlen( obj_name )>=sizeof(topo->objs[ topo->obj_cnt ].name ) ) ) FD_LOG_ERR(( "obj name too long: %s", obj_name ));
      53           0 :   if( FD_UNLIKELY( topo->obj_cnt>=FD_TOPO_MAX_OBJS ) ) FD_LOG_ERR(( "too many objects" ));
      54             : 
      55           0 :   ulong wksp_id = fd_topo_find_wksp( topo, wksp_name );
      56           0 :   if( FD_UNLIKELY( wksp_id==ULONG_MAX ) ) FD_LOG_ERR(( "workspace not found: %s", wksp_name ));
      57             : 
      58           0 :   fd_topo_obj_t * obj = &topo->objs[ topo->obj_cnt ];
      59           0 :   strncpy( obj->name, obj_name, sizeof(obj->name) );
      60           0 :   obj->id      = topo->obj_cnt;
      61           0 :   obj->wksp_id = wksp_id;
      62           0 :   topo->obj_cnt++;
      63             : 
      64           0 :   return obj;
      65           0 : }
      66             : 
      67             : fd_topo_link_t *
      68             : fd_topob_link( fd_topo_t *  topo,
      69             :                char const * link_name,
      70             :                char const * wksp_name,
      71             :                ulong        depth,
      72             :                ulong        mtu,
      73           0 :                ulong        burst ) {
      74           0 :   if( FD_UNLIKELY( !topo || !link_name || !wksp_name ) ) FD_LOG_ERR(( "NULL args" ));
      75           0 :   if( FD_UNLIKELY( strlen( link_name )>=sizeof(topo->links[ topo->link_cnt ].name ) ) ) FD_LOG_ERR(( "link name too long: %s", link_name ));
      76           0 :   if( FD_UNLIKELY( topo->link_cnt>=FD_TOPO_MAX_LINKS ) ) FD_LOG_ERR(( "too many links" ));
      77             : 
      78           0 :   ulong kind_id = 0UL;
      79           0 :   for( ulong i=0UL; i<topo->link_cnt; i++ ) {
      80           0 :     if( !strcmp( topo->links[ i ].name, link_name ) ) kind_id++;
      81           0 :   }
      82             : 
      83           0 :   fd_topo_link_t * link = &topo->links[ topo->link_cnt ];
      84           0 :   strncpy( link->name, link_name, sizeof(link->name) );
      85           0 :   link->id       = topo->link_cnt;
      86           0 :   link->kind_id  = kind_id;
      87           0 :   link->depth    = depth;
      88           0 :   link->mtu      = mtu;
      89           0 :   link->burst    = burst;
      90             : 
      91           0 :   fd_topo_obj_t * obj = fd_topob_obj( topo, "mcache", wksp_name );
      92           0 :   link->mcache_obj_id = obj->id;
      93           0 :   FD_TEST( fd_pod_insertf_ulong( topo->props, depth, "obj.%lu.depth", obj->id ) );
      94             : 
      95           0 :   if( mtu ) {
      96           0 :     obj = fd_topob_obj( topo, "dcache", wksp_name );
      97           0 :     link->dcache_obj_id = obj->id;
      98           0 :     FD_TEST( fd_pod_insertf_ulong( topo->props, depth, "obj.%lu.depth", obj->id ) );
      99           0 :     FD_TEST( fd_pod_insertf_ulong( topo->props, burst, "obj.%lu.burst", obj->id ) );
     100           0 :     FD_TEST( fd_pod_insertf_ulong( topo->props, mtu,   "obj.%lu.mtu",   obj->id ) );
     101           0 :   }
     102           0 :   topo->link_cnt++;
     103             : 
     104           0 :   return link;
     105           0 : }
     106             : 
     107             : void
     108             : fd_topob_tile_uses( fd_topo_t *      topo,
     109             :                     fd_topo_tile_t * tile,
     110             :                     fd_topo_obj_t *  obj,
     111           0 :                     int              mode ) {
     112           0 :   (void)topo;
     113             : 
     114           0 :   if( FD_UNLIKELY( tile->uses_obj_cnt>=FD_TOPO_MAX_TILE_OBJS ) ) FD_LOG_ERR(( "tile `%s` uses too many objects", tile->name ));
     115             : 
     116           0 :   tile->uses_obj_id[ tile->uses_obj_cnt ] = obj->id;
     117           0 :   tile->uses_obj_mode[ tile->uses_obj_cnt ] = mode;
     118           0 :   tile->uses_obj_cnt++;
     119           0 : }
     120             : 
     121             : fd_topo_tile_t *
     122             : fd_topob_tile( fd_topo_t *    topo,
     123             :                char const *   tile_name,
     124             :                char const *   tile_wksp,
     125             :                char const *   metrics_wksp,
     126             :                ulong          cpu_idx,
     127             :                int            is_agave,
     128           0 :                int            uses_keyswitch ) {
     129           0 :   if( FD_UNLIKELY( !topo || !tile_name || !tile_wksp || !metrics_wksp ) ) FD_LOG_ERR(( "NULL args" ));
     130           0 :   if( FD_UNLIKELY( strlen( tile_name )>=sizeof(topo->tiles[ topo->tile_cnt ].name ) ) ) FD_LOG_ERR(( "tile name too long: %s", tile_name ));
     131           0 :   if( FD_UNLIKELY( topo->tile_cnt>=FD_TOPO_MAX_TILES ) ) FD_LOG_ERR(( "too many tiles %lu", topo->tile_cnt ));
     132             : 
     133           0 :   ulong kind_id = 0UL;
     134           0 :   for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
     135           0 :     if( !strcmp( topo->tiles[ i ].name, tile_name ) ) kind_id++;
     136           0 :   }
     137             : 
     138           0 :   fd_topo_tile_t * tile = &topo->tiles[ topo->tile_cnt ];
     139           0 :   strncpy( tile->name, tile_name, sizeof(tile->name) );
     140           0 :   tile->id                  = topo->tile_cnt;
     141           0 :   tile->kind_id             = kind_id;
     142           0 :   tile->is_agave            = is_agave;
     143           0 :   tile->cpu_idx             = cpu_idx;
     144           0 :   tile->in_cnt              = 0UL;
     145           0 :   tile->out_cnt             = 0UL;
     146           0 :   tile->uses_obj_cnt        = 0UL;
     147             : 
     148           0 :   fd_topo_obj_t * tile_obj = fd_topob_obj( topo, "tile", tile_wksp );
     149           0 :   tile->tile_obj_id = tile_obj->id;
     150           0 :   fd_topob_tile_uses( topo, tile, tile_obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
     151             : 
     152           0 :   fd_topo_obj_t * obj = fd_topob_obj( topo, "metrics", metrics_wksp );
     153           0 :   tile->metrics_obj_id = obj->id;
     154           0 :   fd_topob_tile_uses( topo, tile, obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
     155             : 
     156           0 :   if( FD_LIKELY( uses_keyswitch ) ) {
     157           0 :     obj = fd_topob_obj( topo, "keyswitch", tile_wksp );
     158           0 :     tile->keyswitch_obj_id = obj->id;
     159           0 :     fd_topob_tile_uses( topo, tile, obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
     160           0 :   } else {
     161           0 :     tile->keyswitch_obj_id = ULONG_MAX;
     162           0 :   }
     163             : 
     164           0 :   topo->tile_cnt++;
     165           0 :   return tile;
     166           0 : }
     167             : 
     168             : void
     169             : fd_topob_tile_in( fd_topo_t *  topo,
     170             :                   char const * tile_name,
     171             :                   ulong        tile_kind_id,
     172             :                   char const * fseq_wksp,
     173             :                   char const * link_name,
     174             :                   ulong        link_kind_id,
     175             :                   int          reliable,
     176           0 :                   int          polled ) {
     177           0 :   if( FD_UNLIKELY( !topo || !tile_name || !fseq_wksp || !link_name ) ) FD_LOG_ERR(( "NULL args" ));
     178             : 
     179           0 :   ulong tile_id = fd_topo_find_tile( topo, tile_name, tile_kind_id );
     180           0 :   if( FD_UNLIKELY( tile_id==ULONG_MAX ) ) FD_LOG_ERR(( "tile not found: %s:%lu", tile_name, tile_kind_id ));
     181           0 :   fd_topo_tile_t * tile = &topo->tiles[ tile_id ];
     182             : 
     183           0 :   ulong link_id = fd_topo_find_link( topo, link_name, link_kind_id );
     184           0 :   if( FD_UNLIKELY( link_id==ULONG_MAX ) ) FD_LOG_ERR(( "link not found: %s:%lu", link_name, link_kind_id ));
     185           0 :   fd_topo_link_t * link = &topo->links[ link_id ];
     186             : 
     187           0 :   if( FD_UNLIKELY( tile->in_cnt>=FD_TOPO_MAX_TILE_IN_LINKS ) ) FD_LOG_ERR(( "too many in links: %s:%lu", tile_name, tile_kind_id ) );
     188           0 :   tile->in_link_id[ tile->in_cnt ] = link->id;
     189           0 :   tile->in_link_reliable[ tile->in_cnt ] = reliable;
     190           0 :   tile->in_link_poll[ tile->in_cnt ] = polled;
     191           0 :   fd_topo_obj_t * obj = fd_topob_obj( topo, "fseq", fseq_wksp );
     192           0 :   fd_topob_tile_uses( topo, tile, obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
     193           0 :   tile->in_link_fseq_obj_id[ tile->in_cnt ] = obj->id;
     194           0 :   tile->in_cnt++;
     195             : 
     196           0 :   fd_topob_tile_uses( topo, tile, &topo->objs[ link->mcache_obj_id ], FD_SHMEM_JOIN_MODE_READ_ONLY );
     197           0 :   if( FD_LIKELY( link->mtu ) ) {
     198           0 :     fd_topob_tile_uses( topo, tile, &topo->objs[ link->dcache_obj_id ], FD_SHMEM_JOIN_MODE_READ_ONLY );
     199           0 :   }
     200           0 : }
     201             : 
     202             : void
     203             : fd_topob_tile_out( fd_topo_t *  topo,
     204             :                    char const * tile_name,
     205             :                    ulong        tile_kind_id,
     206             :                    char const * link_name,
     207           0 :                    ulong        link_kind_id ) {
     208           0 :   ulong tile_id = fd_topo_find_tile( topo, tile_name, tile_kind_id );
     209           0 :   if( FD_UNLIKELY( tile_id==ULONG_MAX ) ) FD_LOG_ERR(( "tile not found: %s:%lu", tile_name, tile_kind_id ));
     210           0 :   fd_topo_tile_t * tile = &topo->tiles[ tile_id ];
     211             : 
     212           0 :   ulong link_id = fd_topo_find_link( topo, link_name, link_kind_id );
     213           0 :   if( FD_UNLIKELY( link_id==ULONG_MAX ) ) FD_LOG_ERR(( "link not found: %s:%lu", link_name, link_kind_id ));
     214           0 :   fd_topo_link_t * link = &topo->links[ link_id ];
     215             : 
     216           0 :   if( FD_UNLIKELY( tile->out_cnt>=FD_TOPO_MAX_TILE_OUT_LINKS ) ) FD_LOG_ERR(( "too many out links: %s", tile_name ));
     217           0 :   tile->out_link_id[ tile->out_cnt ] = link->id;
     218           0 :   tile->out_cnt++;
     219             : 
     220           0 :   fd_topob_tile_uses( topo, tile, &topo->objs[ link->mcache_obj_id ], FD_SHMEM_JOIN_MODE_READ_WRITE );
     221           0 :   if( FD_LIKELY( link->mtu ) ) {
     222           0 :     fd_topob_tile_uses( topo, tile, &topo->objs[ link->dcache_obj_id ], FD_SHMEM_JOIN_MODE_READ_WRITE );
     223           0 :   }
     224           0 : }
     225             : 
     226             : static void
     227           0 : validate( fd_topo_t const * topo ) {
     228             :   /* Objects have valid wksp_ids */
     229           0 :   for( ulong i=0UL; i<topo->obj_cnt; i++ ) {
     230           0 :     if( FD_UNLIKELY( topo->objs[ i ].wksp_id>=topo->wksp_cnt ) )
     231           0 :       FD_LOG_ERR(( "invalid workspace id %lu", topo->objs[ i ].wksp_id ));
     232           0 :   }
     233             : 
     234             :   /* Tile ins are valid */
     235           0 :   for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
     236           0 :     for( ulong j=0UL; j<topo->tiles[ i ].in_cnt; j++ ) {
     237           0 :       if( FD_UNLIKELY( topo->tiles[ i ].in_link_id[ j ]>=topo->link_cnt ) )
     238           0 :         FD_LOG_ERR(( "tile %lu (%s) has invalid in link %lu", i, topo->tiles[ i ].name, topo->tiles[ i ].in_link_id[ j ] ));
     239           0 :     }
     240           0 :   }
     241             : 
     242             :   /* Tile does not have duplicated ins */
     243           0 :   for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
     244           0 :     for( ulong j=0UL; j<topo->tiles[ i ].in_cnt; j++ ) {
     245           0 :       for( ulong k=0UL; k<topo->tiles[ i ].in_cnt; k++ ) {
     246           0 :         if( FD_UNLIKELY( j==k ) ) continue;
     247           0 :         if( FD_UNLIKELY( topo->tiles[ i ].in_link_id[ j ] == topo->tiles[ i ].in_link_id[ k ] ) )
     248           0 :           FD_LOG_ERR(( "tile %lu (%s) has duplicated in link %lu (%s)", i, topo->tiles[ i ].name,
     249           0 :               topo->tiles[ i ].in_link_id[ j ], topo->links[ topo->tiles[ i ].in_link_id[ j ] ].name ));
     250           0 :       }
     251           0 :     }
     252           0 :   }
     253             : 
     254             :   /* Tile does not have duplicated outs */
     255           0 :   for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
     256           0 :     for( ulong j=0UL; j<topo->tiles[ i ].out_cnt; j++ ) {
     257           0 :       for( ulong k=0UL; k<topo->tiles[ i ].out_cnt; k++ ) {
     258           0 :         if( FD_UNLIKELY( j==k ) ) continue;
     259           0 :         if( FD_UNLIKELY( topo->tiles[ i ].out_link_id[ j ] == topo->tiles[ i ].out_link_id[ k ] ) )
     260           0 :           FD_LOG_ERR(( "tile %lu (%s) has duplicated out link %lu (%s)", i, topo->tiles[ i ].name,
     261           0 :               topo->tiles[ i ].out_link_id[ j ], topo->links[ topo->tiles[ i ].out_link_id[ j ] ].name ));
     262           0 :       }
     263           0 :     }
     264           0 :   }
     265             : 
     266             :   /* Tile outs are different than ins */
     267           0 :   for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
     268           0 :     for( ulong j=0UL; j<topo->tiles[ i ].out_cnt; j++ ) {
     269           0 :       for( ulong k=0UL; k<topo->tiles[ i ].in_cnt; k++ ) {
     270           0 :         char const * link_name = topo->links[ topo->tiles[ i ].out_link_id[ j ] ].name;
     271             :         /* PoH tile "publishes" this on behalf of Agave, so it's not
     272             :            a real circular link. */
     273           0 :         if( FD_UNLIKELY( !strcmp( link_name, "stake_out" ) ||
     274           0 :                          !strcmp( link_name, "crds_shred" ) ) ) continue;
     275             : 
     276           0 :         if( FD_UNLIKELY( topo->tiles[ i ].out_link_id[ j ] == topo->tiles[ i ].in_link_id[ k ] ) )
     277           0 :           FD_LOG_ERR(( "tile %lu has out link %lu same as in", i, topo->tiles[ i ].out_link_id[ j ] ));
     278           0 :       }
     279           0 :     }
     280           0 :   }
     281             : 
     282             :   /* Non polling tile ins are also not reliable */
     283           0 :   for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
     284           0 :     for( ulong j=0UL; j<topo->tiles[ i ].in_cnt; j++ ) {
     285           0 :       if( FD_UNLIKELY( !topo->tiles[ i ].in_link_poll[ j ] && topo->tiles[ i ].in_link_reliable[ j ] ) )
     286           0 :         FD_LOG_ERR(( "tile %lu has in link %lu which is not polled but reliable", i, topo->tiles[ i ].in_link_id[ j ] ));
     287           0 :     }
     288           0 :   }
     289             : 
     290             :   /* Tile outs are valid */
     291           0 :   for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
     292           0 :     for( ulong j=0UL; j<topo->tiles[ i ].out_cnt; j++ ) {
     293           0 :       if( FD_UNLIKELY( topo->tiles[ i ].out_link_id[ j ] >= topo->link_cnt ) )
     294           0 :         FD_LOG_ERR(( "tile %lu has invalid out link %lu", i, topo->tiles[ i ].out_link_id[ j ] ));
     295           0 :     }
     296           0 :   }
     297             : 
     298             :   /* Workspace names are unique */
     299           0 :   for( ulong i=0UL; i<topo->wksp_cnt; i++ ) {
     300           0 :     for( ulong j=0UL; j<topo->wksp_cnt; j++ ) {
     301           0 :       if( FD_UNLIKELY( i==j ) ) continue;
     302           0 :       if( FD_UNLIKELY( !strcmp( topo->workspaces[ i ].name,  topo->workspaces[ j ].name ) ) )
     303           0 :         FD_LOG_ERR(( "duplicate workspace name %s", topo->workspaces[ i ].name ));
     304           0 :     }
     305           0 :   }
     306             : 
     307             :   /* Each workspace is identified correctly */
     308           0 :   for( ulong i=0UL; i<topo->wksp_cnt; i++ ) {
     309           0 :     if( FD_UNLIKELY( topo->workspaces[ i ].id != i ) )
     310           0 :       FD_LOG_ERR(( "workspace %lu has id %lu", i, topo->workspaces[ i ].id ));
     311           0 :   }
     312             : 
     313             :   /* Each link has exactly one producer */
     314           0 :   for( ulong i=0UL; i<topo->link_cnt; i++ ) {
     315           0 :     ulong producer_cnt = 0;
     316           0 :     for( ulong j=0UL; j<topo->tile_cnt; j++ ) {
     317           0 :       for( ulong k=0UL; k<topo->tiles[ j ].out_cnt; k++ ) {
     318           0 :         if( topo->tiles[ j ].out_link_id[ k ]==i ) producer_cnt++;
     319           0 :       }
     320           0 :     }
     321           0 :     if( FD_UNLIKELY( producer_cnt>1UL || ( producer_cnt==0UL && !topo->links[ i ].permit_no_producers ) ) )
     322           0 :       FD_LOG_ERR(( "link %lu (%s:%lu) has %lu producers", i, topo->links[ i ].name, topo->links[ i ].kind_id, producer_cnt ));
     323           0 :   }
     324             : 
     325             :   /* Each link has at least one consumer */
     326           0 :   for( ulong i=0UL; i<topo->link_cnt; i++ ) {
     327           0 :     ulong cnt = fd_topo_link_consumer_cnt( topo, &topo->links[ i ] );
     328           0 :     if( FD_UNLIKELY( cnt < 1UL && !topo->links[ i ].permit_no_consumers ) ) {
     329           0 :       FD_LOG_ERR(( "link %lu (%s:%lu) has 0 consumers", i, topo->links[ i ].name, topo->links[ i ].kind_id ));
     330           0 :     }
     331           0 :   }
     332           0 : }
     333             : 
     334             : void
     335             : fd_topob_auto_layout( fd_topo_t * topo,
     336           0 :                       int         reserve_agave_cores ) {
     337             :   /* Incredibly simple automatic layout system for now ... just assign
     338             :      tiles to CPU cores in NUMA sequential order, except for a few tiles
     339             :      which should be floating. */
     340             : 
     341           0 :   char const * FLOATING[] = {
     342           0 :     "netlnk",
     343           0 :     "metric",
     344           0 :     "cswtch",
     345           0 :     "bencho",
     346           0 :   };
     347             : 
     348           0 :   char const * ORDERED[] = {
     349           0 :     "benchg",
     350           0 :     "benchs",
     351           0 :     "net",
     352           0 :     "sock",
     353           0 :     "quic",
     354           0 :     "bundle",
     355           0 :     "verify",
     356           0 :     "dedup",
     357           0 :     "resolv", /* FRANK only */
     358           0 :     "pack",
     359           0 :     "bank",   /* FRANK only */
     360           0 :     "poh",    /* FRANK only */
     361           0 :     "poh",   /* FIREDANCER only */
     362           0 :     "shred",
     363           0 :     "store",  /* FRANK only */
     364           0 :     "storei", /* FIREDANCER only */
     365           0 :     "sign",
     366           0 :     "plugin",
     367           0 :     "gui",
     368           0 :     "gossip", /* FIREDANCER only */
     369           0 :     "repair", /* FIREDANCER only */
     370           0 :     "replay", /* FIREDANCER only */
     371           0 :     "exec",   /* FIREDANCER only */
     372           0 :     "writer", /* FIREDANCER only */
     373           0 :     "send",   /* FIREDANCER only */
     374           0 :     "tower",  /* FIREDANCER only */
     375           0 :     "rpcsrv", /* FIREDANCER only */
     376           0 :     "pktgen"
     377           0 :   };
     378             : 
     379           0 :   char const * CRITICAL_TILES[] = {
     380           0 :     "pack",
     381           0 :     "poh",
     382           0 :   };
     383             : 
     384           0 :   for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
     385           0 :     fd_topo_tile_t * tile = &topo->tiles[ i ];
     386           0 :     tile->cpu_idx = ULONG_MAX;
     387           0 :   }
     388             : 
     389           0 :   fd_topo_cpus_t cpus[1];
     390           0 :   fd_topo_cpus_init( cpus );
     391             : 
     392           0 :   ulong cpu_ordering[ FD_TILE_MAX ] = { 0UL };
     393           0 :   int   pairs_assigned[ FD_TILE_MAX ] = { 0 };
     394             : 
     395           0 :   ulong next_cpu_idx   = 0UL;
     396           0 :   for( ulong i=0UL; i<cpus->numa_node_cnt; i++ ) {
     397           0 :     for( ulong j=0UL; j<cpus->cpu_cnt; j++ ) {
     398           0 :       fd_topo_cpu_t * cpu = &cpus->cpu[ j ];
     399             : 
     400           0 :       if( FD_UNLIKELY( pairs_assigned[ j ] || cpu->numa_node!=i ) ) continue;
     401             : 
     402           0 :       FD_TEST( next_cpu_idx<FD_TILE_MAX );
     403           0 :       cpu_ordering[ next_cpu_idx++ ] = j;
     404             : 
     405           0 :       if( FD_UNLIKELY( cpu->sibling!=ULONG_MAX ) ) {
     406             :         /* If the CPU has a HT pair, place it immediately after so they
     407             :            are sequentially assigned. */
     408           0 :         FD_TEST( next_cpu_idx<FD_TILE_MAX );
     409           0 :         cpu_ordering[ next_cpu_idx++ ] = cpu->sibling;
     410           0 :         pairs_assigned[ cpu->sibling ] = 1;
     411           0 :       }
     412           0 :     }
     413           0 :   }
     414             : 
     415           0 :   FD_TEST( next_cpu_idx==cpus->cpu_cnt );
     416             : 
     417           0 :   int cpu_assigned[ FD_TILE_MAX ] = {0};
     418             : 
     419           0 :   ulong cpu_idx = 0UL;
     420           0 :   for( ulong i=0UL; i<sizeof(ORDERED)/sizeof(ORDERED[0]); i++ ) {
     421           0 :     for( ulong j=0UL; j<topo->tile_cnt; j++ ) {
     422           0 :       fd_topo_tile_t * tile = &topo->tiles[ j ];
     423           0 :       if( !strcmp( tile->name, ORDERED[ i ] ) ) {
     424           0 :         if( FD_UNLIKELY( cpu_idx>=cpus->cpu_cnt ) ) {
     425           0 :           FD_LOG_ERR(( "auto layout cannot set affinity for tile `%s:%lu` because all the CPUs are already assigned", tile->name, tile->kind_id ));
     426           0 :         } else {
     427             :           /* Certain tiles are latency and throughput critical and
     428             :              should not get a HT pair assigned. */
     429           0 :           fd_topo_cpu_t const * cpu = &cpus->cpu[ cpu_ordering[ cpu_idx ] ];
     430             : 
     431           0 :           int is_ht_critical = 0;
     432           0 :           if( FD_UNLIKELY( cpu->sibling!=ULONG_MAX ) ) {
     433           0 :             for( ulong k=0UL; k<sizeof(CRITICAL_TILES)/sizeof(CRITICAL_TILES[0]); k++ ) {
     434           0 :               if( !strcmp( tile->name, CRITICAL_TILES[ k ] ) ) {
     435           0 :                 is_ht_critical = 1;
     436           0 :                 break;
     437           0 :               }
     438           0 :             }
     439           0 :           }
     440             : 
     441           0 :           if( FD_UNLIKELY( is_ht_critical ) ) {
     442           0 :             ulong try_assign = cpu_idx;
     443           0 :             while( cpu_assigned[ cpu_ordering[ try_assign ] ] || (cpus->cpu[ cpu_ordering[ try_assign ] ].sibling!=ULONG_MAX && cpu_assigned[ cpus->cpu[ cpu_ordering[ try_assign ] ].sibling ]) ) {
     444           0 :               try_assign++;
     445           0 :               if( FD_UNLIKELY( try_assign>=cpus->cpu_cnt ) ) FD_LOG_ERR(( "auto layout cannot set affinity for tile `%s:%lu` because all the CPUs are already assigned or have a HT pair assigned", tile->name, tile->kind_id ));
     446           0 :             }
     447             : 
     448           0 :             ulong sibling = cpus->cpu[ cpu_ordering[ try_assign ] ].sibling;
     449           0 :             cpu_assigned[ cpu_ordering[ try_assign ] ] = 1;
     450           0 :             if( sibling!=ULONG_MAX ) {
     451           0 :               cpu_assigned[ sibling ] = 1;
     452           0 :             }
     453           0 :             tile->cpu_idx = cpu_ordering[ try_assign ];
     454           0 :             while( cpu_assigned[ cpu_ordering[ cpu_idx ] ] ) cpu_idx++;
     455           0 :           } else {
     456           0 :             cpu_assigned[ cpu_ordering[ cpu_idx ] ] = 1;
     457           0 :             tile->cpu_idx = cpu_ordering[ cpu_idx ];
     458           0 :             while( cpu_assigned[ cpu_ordering[ cpu_idx ] ] ) cpu_idx++;
     459           0 :           }
     460           0 :         }
     461           0 :       }
     462           0 :     }
     463           0 :   }
     464             : 
     465             :   /* Make sure all the tiles we haven't set are supposed to be floating. */
     466           0 :   for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
     467           0 :     fd_topo_tile_t * tile = &topo->tiles[ i ];
     468           0 :     if( tile->cpu_idx!=ULONG_MAX ) continue;
     469             : 
     470           0 :     int found = 0;
     471           0 :     for( ulong j=0UL; j<sizeof(FLOATING)/sizeof(FLOATING[0]); j++ ) {
     472           0 :       if( !strcmp( tile->name, FLOATING[ j ] ) ) {
     473           0 :         found = 1;
     474           0 :         break;
     475           0 :       }
     476           0 :     }
     477             : 
     478           0 :     if( FD_UNLIKELY( !found ) ) FD_LOG_WARNING(( "auto layout cannot affine tile `%s:%lu` because it is unknown. Leaving it floating", tile->name, tile->kind_id ));
     479           0 :   }
     480             : 
     481           0 :   if( FD_UNLIKELY( reserve_agave_cores ) ) {
     482           0 :     for( ulong i=cpu_idx; i<cpus->cpu_cnt; i++ ) {
     483           0 :       if( FD_UNLIKELY( !cpus->cpu[ cpu_ordering[ i ] ].online ) ) continue;
     484             : 
     485           0 :       if( FD_LIKELY( topo->agave_affinity_cnt<sizeof(topo->agave_affinity_cpu_idx)/sizeof(topo->agave_affinity_cpu_idx[0]) ) ) {
     486           0 :         topo->agave_affinity_cpu_idx[ topo->agave_affinity_cnt++ ] = cpu_ordering[ i ];
     487           0 :       }
     488           0 :     }
     489           0 :   }
     490           0 : }
     491             : 
     492             : ulong
     493             : fd_numa_node_idx( ulong cpu_idx );
     494             : 
     495             : static void
     496           0 : initialize_numa_assignments( fd_topo_t * topo ) {
     497             :   /* Assign workspaces to NUMA nodes.  The heuristic here is pretty
     498             :      simple for now: workspaces go on the NUMA node of the first
     499             :      tile which maps the largest object in the workspace. */
     500             : 
     501           0 :   for( ulong i=0UL; i<topo->wksp_cnt; i++ ) {
     502           0 :     ulong max_footprint = 0UL;
     503           0 :     ulong max_obj = ULONG_MAX;
     504             : 
     505           0 :     for( ulong j=0UL; j<topo->obj_cnt; j++ ) {
     506           0 :       fd_topo_obj_t * obj = &topo->objs[ j ];
     507           0 :       if( obj->wksp_id!=i ) continue;
     508             : 
     509           0 :       if( FD_UNLIKELY( !max_footprint || obj->footprint>max_footprint ) ) {
     510           0 :         max_footprint = obj->footprint;
     511           0 :         max_obj = j;
     512           0 :       }
     513           0 :     }
     514             : 
     515           0 :     if( FD_UNLIKELY( max_obj==ULONG_MAX ) ) FD_LOG_ERR(( "no object found for workspace %s", topo->workspaces[ i ].name ));
     516             : 
     517           0 :     int found_strict = 0;
     518           0 :     int found_lazy   = 0;
     519           0 :     for( ulong j=0UL; j<topo->tile_cnt; j++ ) {
     520           0 :       fd_topo_tile_t * tile = &topo->tiles[ j ];
     521           0 :       if( FD_UNLIKELY( tile->tile_obj_id==max_obj && tile->cpu_idx<FD_TILE_MAX ) ) {
     522           0 :         topo->workspaces[ i ].numa_idx = fd_numa_node_idx( tile->cpu_idx );
     523           0 :         FD_TEST( topo->workspaces[ i ].numa_idx!=ULONG_MAX );
     524           0 :         found_strict = 1;
     525           0 :         found_lazy = 1;
     526           0 :         break;
     527           0 :       } else if( FD_UNLIKELY( tile->tile_obj_id==max_obj && tile->cpu_idx>=FD_TILE_MAX ) ) {
     528           0 :         topo->workspaces[ i ].numa_idx = 0;
     529           0 :         found_lazy = 1;
     530           0 :         break;
     531           0 :       }
     532           0 :     }
     533             : 
     534           0 :     if( FD_LIKELY( !found_strict ) ) {
     535           0 :       for( ulong j=0UL; j<topo->tile_cnt; j++ ) {
     536           0 :         fd_topo_tile_t * tile = &topo->tiles[ j ];
     537           0 :         for( ulong k=0UL; k<tile->uses_obj_cnt; k++ ) {
     538           0 :           if( FD_LIKELY( tile->uses_obj_id[ k ]==max_obj && tile->cpu_idx<FD_TILE_MAX ) ) {
     539           0 :             topo->workspaces[ i ].numa_idx = fd_numa_node_idx( tile->cpu_idx );
     540           0 :             FD_TEST( topo->workspaces[ i ].numa_idx!=ULONG_MAX );
     541           0 :             found_lazy = 1;
     542           0 :             break;
     543           0 :           } else if( FD_UNLIKELY( tile->uses_obj_id[ k ]==max_obj ) && tile->cpu_idx>=FD_TILE_MAX ) {
     544           0 :             topo->workspaces[ i ].numa_idx = 0;
     545           0 :             found_lazy = 1;
     546             :             /* Don't break, keep looking -- a tile with a CPU assignment
     547             :                might also use object in which case we want to use that
     548             :                NUMA node. */
     549           0 :           }
     550           0 :         }
     551             : 
     552           0 :         if( FD_UNLIKELY( found_lazy ) ) break;
     553           0 :       }
     554           0 :     }
     555             : 
     556           0 :     if( FD_UNLIKELY( !found_lazy ) ) FD_LOG_ERR(( "no tile uses object %s for workspace %s", topo->objs[ max_obj ].name, topo->workspaces[ i ].name ));
     557           0 :   }
     558           0 : }
     559             : 
     560             : void
     561             : fd_topob_finish( fd_topo_t *                topo,
     562           0 :                  fd_topo_obj_callbacks_t ** callbacks ) {
     563           0 :   for( ulong z=0UL; z<topo->tile_cnt; z++ ) {
     564           0 :     fd_topo_tile_t * tile = &topo->tiles[ z ];
     565             : 
     566           0 :     ulong in_cnt = 0UL;
     567           0 :     for( ulong i=0UL; i<tile->in_cnt; i++ ) {
     568           0 :       if( FD_UNLIKELY( !tile->in_link_poll[ i ] ) ) continue;
     569           0 :       in_cnt++;
     570           0 :     }
     571             : 
     572           0 :     ulong cons_cnt = 0UL;
     573           0 :     for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
     574           0 :       fd_topo_tile_t * consumer_tile = &topo->tiles[ i ];
     575           0 :       for( ulong j=0UL; j<consumer_tile->in_cnt; j++ ) {
     576           0 :         for( ulong k=0UL; k<tile->out_cnt; k++ ) {
     577           0 :           if( FD_UNLIKELY( consumer_tile->in_link_id[ j ]==tile->out_link_id[ k ] && consumer_tile->in_link_reliable[ j ] ) ) {
     578           0 :             cons_cnt++;
     579           0 :           }
     580           0 :         }
     581           0 :       }
     582           0 :     }
     583             : 
     584           0 :     FD_TEST( !fd_pod_replacef_ulong( topo->props, in_cnt, "obj.%lu.in_cnt", tile->metrics_obj_id ) );
     585           0 :     FD_TEST( !fd_pod_replacef_ulong( topo->props, cons_cnt, "obj.%lu.cons_cnt", tile->metrics_obj_id ) );
     586           0 :   }
     587             : 
     588           0 :   for( ulong i=0UL; i<topo->wksp_cnt; i++ ) {
     589           0 :     fd_topo_wksp_t * wksp = &topo->workspaces[ i ];
     590             : 
     591           0 :     ulong loose_sz = 0UL;
     592           0 :     for( ulong j=0UL; j<topo->obj_cnt; j++ ) {
     593           0 :       fd_topo_obj_t * obj = &topo->objs[ j ];
     594           0 :       if( FD_UNLIKELY( obj->wksp_id!=wksp->id ) ) continue;
     595             : 
     596           0 :       fd_topo_obj_callbacks_t * cb = NULL;
     597           0 :       for( ulong i=0UL; callbacks[ i ]; i++ ) {
     598           0 :         if( FD_UNLIKELY( !strcmp( callbacks[ i ]->name, obj->name ) ) ) {
     599           0 :           cb = callbacks[ i ];
     600           0 :           break;
     601           0 :         }
     602           0 :       }
     603           0 :       if( FD_UNLIKELY( !cb ) ) FD_LOG_ERR(( "no callbacks for object %s", obj->name ));
     604             : 
     605           0 :       if( FD_UNLIKELY( cb->loose ) ) loose_sz += cb->loose( topo, obj );
     606           0 :     }
     607             : 
     608           0 :     ulong part_max = wksp->part_max;
     609           0 :     if( !part_max ) part_max = (loose_sz / (64UL << 10)); /* alloc + residual padding */
     610           0 :     part_max += 3; /* for initial alignment */
     611           0 :     ulong offset = fd_ulong_align_up( fd_wksp_private_data_off( part_max ), fd_topo_workspace_align() );
     612             : 
     613           0 :     for( ulong j=0UL; j<topo->obj_cnt; j++ ) {
     614           0 :       fd_topo_obj_t * obj = &topo->objs[ j ];
     615           0 :       if( FD_UNLIKELY( obj->wksp_id!=wksp->id ) ) continue;
     616             : 
     617           0 :       fd_topo_obj_callbacks_t * cb = NULL;
     618           0 :       for( ulong i=0UL; callbacks[ i ]; i++ ) {
     619           0 :         if( FD_UNLIKELY( !strcmp( callbacks[ i ]->name, obj->name ) ) ) {
     620           0 :           cb = callbacks[ i ];
     621           0 :           break;
     622           0 :         }
     623           0 :       }
     624           0 :       if( FD_UNLIKELY( !cb ) ) FD_LOG_ERR(( "no callbacks for object %s", obj->name ));
     625             : 
     626           0 :       ulong align_ = cb->align( topo, obj );
     627           0 :       if( FD_UNLIKELY( !fd_ulong_is_pow2( align_ ) ) ) FD_LOG_ERR(( "Return value of fdctl_obj_align(%s,%lu) is not a power of 2", obj->name, obj->id ));
     628           0 :       offset = fd_ulong_align_up( offset, align_ );
     629           0 :       obj->offset = offset;
     630           0 :       obj->footprint = cb->footprint( topo, obj );
     631           0 :       if( FD_UNLIKELY( 0!=strcmp( obj->name, "tile" ) && (!obj->footprint || obj->footprint>LONG_MAX) ) ) {
     632           0 :         FD_LOG_ERR(( "fdctl_obj_footprint(%s,%lu) failed", obj->name, obj->id ));
     633           0 :       }
     634           0 :       offset += obj->footprint;
     635           0 :     }
     636             : 
     637           0 :     ulong footprint = fd_ulong_align_up( offset, fd_topo_workspace_align() );
     638             : 
     639             :     /* Compute footprint for a workspace that can store our footprint,
     640             :        with an extra align of padding incase gaddr_lo is not aligned. */
     641           0 :     ulong total_wksp_footprint = fd_wksp_footprint( part_max, footprint + fd_topo_workspace_align() + loose_sz );
     642             : 
     643           0 :     ulong page_sz = topo->max_page_size;
     644           0 :     if( total_wksp_footprint < topo->gigantic_page_threshold ) page_sz = FD_SHMEM_HUGE_PAGE_SZ;
     645           0 :     if( FD_UNLIKELY( page_sz!=FD_SHMEM_HUGE_PAGE_SZ && page_sz!=FD_SHMEM_GIGANTIC_PAGE_SZ ) ) FD_LOG_ERR(( "invalid page_sz" ));
     646             : 
     647           0 :     ulong wksp_aligned_footprint = fd_ulong_align_up( total_wksp_footprint, page_sz );
     648             : 
     649             :     /* Give any leftover space in the underlying shared memory to the
     650             :        data region of the workspace, since we might as well use it. */
     651           0 :     wksp->part_max = part_max;
     652           0 :     wksp->known_footprint = footprint;
     653           0 :     wksp->total_footprint = wksp_aligned_footprint - fd_ulong_align_up( fd_wksp_private_data_off( part_max ), fd_topo_workspace_align() );
     654           0 :     wksp->page_sz = page_sz;
     655           0 :     wksp->page_cnt = wksp_aligned_footprint / page_sz;
     656           0 :   }
     657             : 
     658           0 :   initialize_numa_assignments( topo );
     659             : 
     660           0 :   validate( topo );
     661           0 : }

Generated by: LCOV version 1.14