Line data    Source code

       1             : #include "../../flamenco/runtime/fd_rocksdb.h"
       2             : #include <unistd.h>
       3             : #include <sys/stat.h>
       4             : 
       5             : struct fd_ledger_args {
       6             :   fd_wksp_t *           wksp;                    /* wksp for blockstore */
       7             :   char const *          cmd;                     /* user passed command to fd_ledger */
       8             :   ulong                 start_slot;              /* start slot for offline replay */
       9             :   ulong                 end_slot;                /* end slot for offline replay */
      10             :   uint                  hashseed;                /* hashseed */
      11             :   ulong                 shred_max;               /* maximum number of shreds*/
      12             :   ulong                 slot_history_max;        /* number of slots stored by blockstore*/
      13             :   char const *          mini_db_dir;             /* path to minifed rocksdb that's to be created */
      14             :   char const *          rocksdb_path;            /* path to rocksdb directory */
      15             : };
      16             : typedef struct fd_ledger_args fd_ledger_args_t;
      17             : 
      18             : /********************* Main Command Functions and Setup ***********************/
      19             : void
      20           0 : minify( fd_ledger_args_t * args ) {
      21             :     /* Example commmand:
      22             :     fd_ledger --cmd minify --rocksdb <LARGE_ROCKSDB> --minified-rocksdb <MINI_ROCKSDB>
      23             :               --start-slot <START_SLOT> --end-slot <END_SLOT>
      24             :   */
      25           0 :   if( args->rocksdb_path == NULL ) {
      26           0 :     FD_LOG_ERR(( "rocksdb path is NULL" ));
      27           0 :   }
      28           0 :   if( args->mini_db_dir == NULL ) {
      29           0 :     FD_LOG_ERR(( "minified rocksdb path is NULL" ));
      30           0 :   }
      31             : 
      32           0 :   fd_rocksdb_t big_rocksdb;
      33           0 :   char * err = fd_rocksdb_init( &big_rocksdb, args->rocksdb_path );
      34           0 :   if( FD_UNLIKELY( err!=NULL ) ) {
      35           0 :     FD_LOG_ERR(( "fd_rocksdb_init at path=%s returned error=%s", args->rocksdb_path, err ));
      36           0 :   }
      37             : 
      38             :   /* If the directory for the minified rocksdb already exists, error out */
      39           0 :   struct stat statbuf;
      40           0 :   if( stat( args->mini_db_dir, &statbuf ) == 0 ) {
      41           0 :     FD_LOG_ERR(( "path for mini_db_dir=%s already exists", args->mini_db_dir ));
      42           0 :   }
      43             : 
      44             :   /* Create a new smaller rocksdb */
      45           0 :   fd_rocksdb_t mini_rocksdb;
      46           0 :   fd_rocksdb_new( &mini_rocksdb, args->mini_db_dir );
      47             : 
      48             :   /* Correctly bound off start and end slot */
      49           0 :   ulong first_slot = fd_rocksdb_first_slot( &big_rocksdb, &err );
      50           0 :   ulong last_slot  = fd_rocksdb_last_slot( &big_rocksdb, &err );
      51           0 :   if( args->start_slot < first_slot ) { args->start_slot = first_slot; }
      52           0 :   if( args->end_slot > last_slot )    { args->end_slot = last_slot; }
      53             : 
      54           0 :   FD_LOG_NOTICE(( "copying over rocks db for range [%lu, %lu]", args->start_slot, args->end_slot ));
      55             : 
      56             :   /* Copy over all slot indexed columns */
      57           0 :   for( ulong cf_idx = 1; cf_idx < FD_ROCKSDB_CF_CNT; ++cf_idx ) {
      58           0 :     fd_rocksdb_copy_over_slot_indexed_range( &big_rocksdb, &mini_rocksdb, cf_idx,
      59           0 :                                               args->start_slot, args->end_slot );
      60           0 :   }
      61           0 :   FD_LOG_NOTICE(("copied over all slot indexed columns"));
      62             : 
      63             :   /* TODO: Currently, the address signatures column family isn't copied as it
      64             :            is indexed on the pubkey. */
      65             : 
      66           0 :   rocksdb_flushoptions_t * flush_options = rocksdb_flushoptions_create();
      67           0 :   rocksdb_flushoptions_set_wait( flush_options, 1 );
      68           0 :   char * flush_err = NULL;
      69           0 :   rocksdb_flush_cfs( mini_rocksdb.db, flush_options,
      70           0 :                      &mini_rocksdb.cf_handles[ 1 ],
      71           0 :                      FD_ROCKSDB_CF_CNT - 1, &flush_err );
      72           0 :   if( FD_UNLIKELY( flush_err ) ) {
      73           0 :     FD_LOG_WARNING(( "minify: flushing minified rocksdb failed: %s", flush_err ));
      74           0 :     rocksdb_free( flush_err );
      75           0 :   }
      76           0 :   rocksdb_flushoptions_destroy( flush_options );
      77             : 
      78           0 :   fd_rocksdb_destroy( &big_rocksdb );
      79           0 :   fd_rocksdb_destroy( &mini_rocksdb );
      80           0 : }
      81             : 
      82             : /* Parse user arguments and setup shared data structures used across commands */
      83             : int
      84           0 : initial_setup( int argc, char ** argv, fd_ledger_args_t * args ) {
      85           0 :   if( FD_UNLIKELY( argc==1 ) ) {
      86           0 :     return 1;
      87           0 :   }
      88             : 
      89           0 :   fd_boot( &argc, &argv );
      90             : 
      91           0 :   char const * cmd                   = fd_env_strip_cmdline_cstr  ( &argc, &argv, "--cmd",                   NULL, NULL                                               );
      92           0 :   ulong        slot_history_max      = fd_env_strip_cmdline_ulong ( &argc, &argv, "--slot-history",          NULL, 100UL                                              );
      93           0 :   ulong        shred_max             = fd_env_strip_cmdline_ulong ( &argc, &argv, "--shred-max",             NULL, 1UL << 17                                          );
      94           0 :   ulong        start_slot            = fd_env_strip_cmdline_ulong ( &argc, &argv, "--start-slot",            NULL, 0UL                                                );
      95           0 :   ulong        end_slot              = fd_env_strip_cmdline_ulong ( &argc, &argv, "--end-slot",              NULL, ULONG_MAX                                          );
      96           0 :   char const * mini_db_dir           = fd_env_strip_cmdline_cstr  ( &argc, &argv, "--minified-rocksdb",      NULL, NULL                                               );
      97           0 :   char const * rocksdb_path          = fd_env_strip_cmdline_cstr  ( &argc, &argv, "--rocksdb",               NULL, NULL                                               );
      98             : 
      99             :   // TODO: Add argument validation. Make sure that we aren't including any arguments that aren't parsed for
     100             : 
     101           0 :   char hostname[64];
     102           0 :   gethostname( hostname, sizeof(hostname) );
     103           0 :   ulong hashseed = fd_hash( 0, hostname, strnlen( hostname, sizeof(hostname) ) );
     104           0 :   args->hashseed = (uint)hashseed;
     105             : 
     106             :   /* Copy over arguments */
     107           0 :   args->cmd                     = cmd;
     108           0 :   args->start_slot              = start_slot;
     109           0 :   args->end_slot                = end_slot;
     110           0 :   args->shred_max               = shred_max;
     111           0 :   args->slot_history_max        = slot_history_max;
     112           0 :   args->mini_db_dir             = mini_db_dir;
     113           0 :   args->rocksdb_path            = rocksdb_path;
     114             : 
     115           0 :   if( args->rocksdb_path != NULL ) {
     116           0 :     FD_LOG_NOTICE(( "rocksdb=%s", args->rocksdb_path ));
     117           0 :   }
     118             : 
     119           0 :   return 0;
     120           0 : }
     121             : 
     122             : int main( int argc, char ** argv ) {
     123             :   /* Declaring this on the stack gets the alignment wrong when using asan */
     124             :   fd_ledger_args_t * args = fd_alloca( alignof(fd_ledger_args_t), sizeof(fd_ledger_args_t) );
     125             :   memset( args, 0, sizeof(fd_ledger_args_t) );
     126             :   initial_setup( argc, argv, args );
     127             : 
     128             :   if( args->cmd == NULL ) {
     129             :     FD_LOG_ERR(( "no command specified" ));
     130             :   } else if( strcmp( args->cmd, "minify" ) == 0 ) {
     131             :     minify( args );
     132             :   } else {
     133             :     FD_LOG_ERR(( "unknown command=%s", args->cmd ));
     134             :   }
     135             : 
     136             :   return 0;
     137             : }