Line data    Source code

       1             : #include "fd_rocksdb.h"
       2             : #include <stdbool.h>
       3             : #include <stdlib.h>
       4             : #include <stdio.h>
       5             : #include <unistd.h>
       6             : #include "../../util/bits/fd_bits.h"
       7             : 
       8             : char *
       9             : fd_rocksdb_init( fd_rocksdb_t * db,
      10           0 :                  char const *   db_name ) {
      11           0 :   fd_memset(db, 0, sizeof(fd_rocksdb_t));
      12             : 
      13           0 :   db->opts = rocksdb_options_create();
      14           0 :   db->cfgs[ FD_ROCKSDB_CFIDX_DEFAULT                  ] = "default";
      15           0 :   db->cfgs[ FD_ROCKSDB_CFIDX_META                     ] = "meta";
      16           0 :   db->cfgs[ FD_ROCKSDB_CFIDX_DEAD_SLOTS               ] = "dead_slots";
      17           0 :   db->cfgs[ FD_ROCKSDB_CFIDX_DUPLICATE_SLOTS          ] = "duplicate_slots";
      18           0 :   db->cfgs[ FD_ROCKSDB_CFIDX_ERASURE_META             ] = "erasure_meta";
      19           0 :   db->cfgs[ FD_ROCKSDB_CFIDX_ORPHANS                  ] = "orphans";
      20           0 :   db->cfgs[ FD_ROCKSDB_CFIDX_BANK_HASHES              ] = "bank_hashes";
      21           0 :   db->cfgs[ FD_ROCKSDB_CFIDX_ROOT                     ] = "root";
      22           0 :   db->cfgs[ FD_ROCKSDB_CFIDX_INDEX                    ] = "index";
      23           0 :   db->cfgs[ FD_ROCKSDB_CFIDX_DATA_SHRED               ] = "data_shred";
      24           0 :   db->cfgs[ FD_ROCKSDB_CFIDX_CODE_SHRED               ] = "code_shred";
      25           0 :   db->cfgs[ FD_ROCKSDB_CFIDX_TRANSACTION_STATUS       ] = "transaction_status";
      26           0 :   db->cfgs[ FD_ROCKSDB_CFIDX_ADDRESS_SIGNATURES       ] = "address_signatures";
      27           0 :   db->cfgs[ FD_ROCKSDB_CFIDX_TRANSACTION_MEMOS        ] = "transaction_memos";
      28           0 :   db->cfgs[ FD_ROCKSDB_CFIDX_TRANSACTION_STATUS_INDEX ] = "transaction_status_index";
      29           0 :   db->cfgs[ FD_ROCKSDB_CFIDX_REWARDS                  ] = "rewards";
      30           0 :   db->cfgs[ FD_ROCKSDB_CFIDX_BLOCKTIME                ] = "blocktime";
      31           0 :   db->cfgs[ FD_ROCKSDB_CFIDX_PERF_SAMPLES             ] = "perf_samples";
      32           0 :   db->cfgs[ FD_ROCKSDB_CFIDX_BLOCK_HEIGHT             ] = "block_height";
      33           0 :   db->cfgs[ FD_ROCKSDB_CFIDX_OPTIMISTIC_SLOTS         ] = "optimistic_slots";
      34           0 :   db->cfgs[ FD_ROCKSDB_CFIDX_MERKLE_ROOT_META         ] = "merkle_root_meta";
      35             : 
      36           0 :   rocksdb_options_t const * cf_options[ FD_ROCKSDB_CF_CNT ];
      37           0 :   for( ulong i=0UL; i<FD_ROCKSDB_CF_CNT; i++ )
      38           0 :     cf_options[ i ] = db->opts;
      39             : 
      40           0 :   char *err = NULL;
      41             : 
      42           0 :   db->db = rocksdb_open_for_read_only_column_families(
      43           0 :       db->opts,
      44           0 :       db_name,
      45           0 :       FD_ROCKSDB_CF_CNT,
      46           0 :       (char              const * const *)db->cfgs,
      47           0 :       (rocksdb_options_t const * const *)cf_options,
      48           0 :       db->cf_handles,
      49           0 :       false,
      50           0 :       &err );
      51             : 
      52           0 :   if( FD_UNLIKELY( err ) ) return err;
      53             : 
      54           0 :   db->ro = rocksdb_readoptions_create();
      55             : 
      56           0 :   return NULL;
      57           0 : }
      58             : 
      59             : void
      60             : fd_rocksdb_new( fd_rocksdb_t * db,
      61           0 :                 char const *   db_name ) {
      62           0 :   fd_memset(db, 0, sizeof(fd_rocksdb_t));
      63             : 
      64           0 :   db->opts = rocksdb_options_create();
      65             :   /* Create the db*/
      66           0 :   rocksdb_options_set_create_if_missing(db->opts, 1);
      67             : 
      68           0 :   db->cfgs[ FD_ROCKSDB_CFIDX_DEFAULT                  ] = "default";
      69           0 :   db->cfgs[ FD_ROCKSDB_CFIDX_META                     ] = "meta";
      70           0 :   db->cfgs[ FD_ROCKSDB_CFIDX_DEAD_SLOTS               ] = "dead_slots";
      71           0 :   db->cfgs[ FD_ROCKSDB_CFIDX_DUPLICATE_SLOTS          ] = "duplicate_slots";
      72           0 :   db->cfgs[ FD_ROCKSDB_CFIDX_ERASURE_META             ] = "erasure_meta";
      73           0 :   db->cfgs[ FD_ROCKSDB_CFIDX_ORPHANS                  ] = "orphans";
      74           0 :   db->cfgs[ FD_ROCKSDB_CFIDX_BANK_HASHES              ] = "bank_hashes";
      75           0 :   db->cfgs[ FD_ROCKSDB_CFIDX_ROOT                     ] = "root";
      76           0 :   db->cfgs[ FD_ROCKSDB_CFIDX_INDEX                    ] = "index";
      77           0 :   db->cfgs[ FD_ROCKSDB_CFIDX_DATA_SHRED               ] = "data_shred";
      78           0 :   db->cfgs[ FD_ROCKSDB_CFIDX_CODE_SHRED               ] = "code_shred";
      79           0 :   db->cfgs[ FD_ROCKSDB_CFIDX_TRANSACTION_STATUS       ] = "transaction_status";
      80           0 :   db->cfgs[ FD_ROCKSDB_CFIDX_ADDRESS_SIGNATURES       ] = "address_signatures";
      81           0 :   db->cfgs[ FD_ROCKSDB_CFIDX_TRANSACTION_MEMOS        ] = "transaction_memos";
      82           0 :   db->cfgs[ FD_ROCKSDB_CFIDX_TRANSACTION_STATUS_INDEX ] = "transaction_status_index";
      83           0 :   db->cfgs[ FD_ROCKSDB_CFIDX_REWARDS                  ] = "rewards";
      84           0 :   db->cfgs[ FD_ROCKSDB_CFIDX_BLOCKTIME                ] = "blocktime";
      85           0 :   db->cfgs[ FD_ROCKSDB_CFIDX_PERF_SAMPLES             ] = "perf_samples";
      86           0 :   db->cfgs[ FD_ROCKSDB_CFIDX_BLOCK_HEIGHT             ] = "block_height";
      87           0 :   db->cfgs[ FD_ROCKSDB_CFIDX_OPTIMISTIC_SLOTS         ] = "optimistic_slots";
      88           0 :   db->cfgs[ FD_ROCKSDB_CFIDX_MERKLE_ROOT_META         ] = "merkle_root_meta";
      89             : 
      90             :   /* Create the rocksdb */
      91           0 :   char * err = NULL;
      92           0 :   db->db = rocksdb_open(db->opts, db_name, &err);
      93           0 :   if ( err != NULL ) {
      94           0 :     FD_LOG_ERR(("rocksdb creation failed: %s", err));
      95           0 :   }
      96             : 
      97           0 :   db->wo = rocksdb_writeoptions_create();
      98             : 
      99             :   /* Create column families, default already exists at index 0 */
     100           0 :   for ( ulong i = 1; i < FD_ROCKSDB_CF_CNT; ++i ) {
     101           0 :     db->cf_handles[i] = rocksdb_create_column_family(db->db, db->opts, db->cfgs[i], &err);
     102           0 :   }
     103           0 :   rocksdb_options_set_compression( db->opts, rocksdb_lz4_compression );
     104           0 : }
     105             : 
     106           0 : void fd_rocksdb_destroy(fd_rocksdb_t *db) {
     107             : 
     108           0 :   for( ulong i=0UL; i<FD_ROCKSDB_CF_CNT; i++ ) {
     109           0 :     if( db->cf_handles[i] ) {
     110           0 :       rocksdb_column_family_handle_destroy( db->cf_handles[i] );
     111           0 :       db->cf_handles[i] = NULL;
     112           0 :     }
     113           0 :   }
     114             : 
     115           0 :   if( db->ro ) {
     116           0 :     rocksdb_readoptions_destroy( db->ro );
     117           0 :     db->ro = NULL;
     118           0 :   }
     119             : 
     120           0 :   if( db->opts ) {
     121           0 :     rocksdb_options_destroy( db->opts );
     122           0 :     db->opts = NULL;
     123           0 :   }
     124             : 
     125           0 :   if( db->db ) {
     126           0 :     rocksdb_close( db->db );
     127           0 :     db->db = NULL;
     128           0 :   }
     129             : 
     130           0 :   if( db->wo ) {
     131           0 :     rocksdb_writeoptions_destroy( db->wo );
     132           0 :   }
     133           0 : }
     134             : 
     135           0 : ulong fd_rocksdb_last_slot(fd_rocksdb_t *db, char **err) {
     136           0 :   rocksdb_iterator_t* iter = rocksdb_create_iterator_cf(db->db, db->ro, db->cf_handles[FD_ROCKSDB_CFIDX_ROOT]);
     137           0 :   rocksdb_iter_seek_to_last(iter);
     138           0 :   if (!rocksdb_iter_valid(iter)) {
     139           0 :     rocksdb_iter_destroy(iter);
     140           0 :     *err = "db column for root is empty";
     141           0 :     return 0;
     142           0 :   }
     143             : 
     144           0 :   size_t klen = 0;
     145           0 :   const char *key = rocksdb_iter_key(iter, &klen); // There is no need to free key
     146           0 :   unsigned long slot = fd_ulong_bswap(*((unsigned long *) key));
     147           0 :   rocksdb_iter_destroy(iter);
     148           0 :   return slot;
     149           0 : }
     150             : 
     151             : ulong
     152             : fd_rocksdb_first_slot( fd_rocksdb_t * db,
     153           0 :                        char **        err ) {
     154             : 
     155           0 :   rocksdb_iterator_t* iter = rocksdb_create_iterator_cf(db->db, db->ro, db->cf_handles[FD_ROCKSDB_CFIDX_ROOT]);
     156           0 :   rocksdb_iter_seek_to_first(iter);
     157           0 :   if( FD_UNLIKELY( !rocksdb_iter_valid(iter) ) ) {
     158           0 :     rocksdb_iter_destroy(iter);
     159           0 :     *err = "db column for root is empty";
     160           0 :     return 0;
     161           0 :   }
     162             : 
     163           0 :   ulong klen = 0;
     164           0 :   char const * key = rocksdb_iter_key( iter, &klen ); // There is no need to free key
     165           0 :   ulong slot = fd_ulong_bswap( *((ulong *)key));
     166           0 :   rocksdb_iter_destroy(iter);
     167           0 :   return slot;
     168           0 : }
     169             : 
     170             : fd_slot_meta_t *
     171             : fd_rocksdb_get_meta( fd_rocksdb_t *   db,
     172           0 :                      ulong            slot ) {
     173           0 :   ulong ks = fd_ulong_bswap(slot);
     174           0 :   size_t vallen = 0;
     175             : 
     176           0 :   char * err  = NULL;
     177           0 :   char * meta = rocksdb_get_cf( db->db,
     178           0 :                                 db->ro,
     179           0 :                                 db->cf_handles[FD_ROCKSDB_CFIDX_META],
     180           0 :                                 (const char *) &ks,
     181           0 :                                 sizeof(ks),
     182           0 :                                 &vallen,
     183           0 :                                 &err );
     184             : 
     185           0 :   if( NULL != err ) {
     186           0 :     FD_LOG_WARNING(( "%s", err ));
     187           0 :     free( err );
     188           0 :     return NULL;
     189           0 :   }
     190             : 
     191           0 :   if (0 == vallen)
     192           0 :     return NULL;
     193             : 
     194           0 :   fd_bincode_decode_ctx_t ctx;
     195           0 :   ctx.data = meta;
     196           0 :   ctx.dataend = &meta[vallen];
     197             : 
     198           0 :   ulong total_sz = 0UL;
     199           0 :   if( fd_slot_meta_decode_footprint( &ctx, &total_sz ) ) {
     200           0 :     FD_LOG_ERR(( "fd_slot_meta_decode failed" ));
     201           0 :   }
     202             : 
     203           0 :   fd_slot_meta_t * mem = aligned_alloc( fd_slot_meta_align(), total_sz );
     204           0 :   if( FD_UNLIKELY( !mem ) ) {
     205           0 :     FD_LOG_ERR(( "aligned_alloc failed" ));
     206           0 :   }
     207             : 
     208           0 :   fd_slot_meta_decode( mem, &ctx );
     209             : 
     210           0 :   free(meta);
     211             : 
     212           0 :   return mem;
     213           0 : }
     214             : 
     215             : void *
     216           0 : fd_rocksdb_root_iter_new     ( void * ptr ) {
     217           0 :   fd_memset(ptr, 0, sizeof(fd_rocksdb_root_iter_t));
     218           0 :   return ptr;
     219           0 : }
     220             : 
     221             : fd_rocksdb_root_iter_t *
     222           0 : fd_rocksdb_root_iter_join    ( void * ptr ) {
     223           0 :   return (fd_rocksdb_root_iter_t *) ptr;
     224           0 : }
     225             : 
     226             : void *
     227           0 : fd_rocksdb_root_iter_leave   ( fd_rocksdb_root_iter_t * ptr ) {
     228           0 :   return ptr;
     229           0 : }
     230             : 
     231             : fd_slot_meta_t *
     232             : fd_rocksdb_root_iter_seek( fd_rocksdb_root_iter_t * self,
     233             :                            fd_rocksdb_t *           db,
     234           0 :                            ulong                    slot ) {
     235           0 :   self->db = db;
     236             : 
     237           0 :   if( FD_UNLIKELY( !self->iter ) )
     238           0 :     self->iter = rocksdb_create_iterator_cf(self->db->db, self->db->ro, self->db->cf_handles[FD_ROCKSDB_CFIDX_ROOT]);
     239             : 
     240           0 :   ulong ks = fd_ulong_bswap( slot );
     241             : 
     242           0 :   rocksdb_iter_seek( self->iter, (char const *)&ks, sizeof(ulong) );
     243           0 :   if( FD_UNLIKELY( !rocksdb_iter_valid(self->iter) ) )
     244           0 :     return NULL;
     245             : 
     246           0 :   size_t klen = 0;
     247           0 :   char const * key = rocksdb_iter_key( self->iter, &klen ); // There is no need to free key
     248           0 :   ulong kslot = fd_ulong_bswap( *((ulong *)key) );
     249             : 
     250           0 :   if( FD_UNLIKELY( kslot != slot ) ) {
     251           0 :     FD_LOG_WARNING(( "fd_rocksdb_root_iter_seek: wanted slot %lu, found %lu",
     252           0 :                      slot, kslot ));
     253           0 :     return NULL;
     254           0 :   }
     255             : 
     256           0 :   return fd_rocksdb_get_meta( self->db, slot );
     257           0 : }
     258             : 
     259             : int
     260           0 : fd_rocksdb_root_iter_slot  ( fd_rocksdb_root_iter_t * self, ulong *slot ) {
     261           0 :   if ((NULL == self->db) || (NULL == self->iter))
     262           0 :     return -1;
     263             : 
     264           0 :   if (!rocksdb_iter_valid(self->iter))
     265           0 :     return -2;
     266             : 
     267           0 :   size_t klen = 0;
     268           0 :   const char *key = rocksdb_iter_key(self->iter, &klen); // There is no need to free key
     269           0 :   *slot = fd_ulong_bswap(*((unsigned long *) key));
     270           0 :   return 0;
     271           0 : }
     272             : 
     273             : fd_slot_meta_t *
     274           0 : fd_rocksdb_root_iter_next( fd_rocksdb_root_iter_t * self ) {
     275           0 :   if ((NULL == self->db) || (NULL == self->iter))
     276           0 :     return NULL;
     277             : 
     278           0 :   if (!rocksdb_iter_valid(self->iter))
     279           0 :     return NULL;
     280             : 
     281           0 :   rocksdb_iter_next(self->iter);
     282             : 
     283           0 :   if (!rocksdb_iter_valid(self->iter))
     284           0 :     return NULL;
     285             : 
     286           0 :   size_t klen = 0;
     287           0 :   const char *key = rocksdb_iter_key(self->iter, &klen); // There is no need to free key
     288             : 
     289           0 :   return fd_rocksdb_get_meta( self->db, fd_ulong_bswap(*((unsigned long *) key)) );
     290           0 : }
     291             : 
     292             : void
     293           0 : fd_rocksdb_root_iter_destroy ( fd_rocksdb_root_iter_t * self ) {
     294           0 :   if (NULL != self->iter) {
     295           0 :     rocksdb_iter_destroy(self->iter);
     296           0 :     self->iter = 0;
     297           0 :   }
     298           0 :   self->db = NULL;
     299           0 : }
     300             : 
     301             : ulong
     302           0 : fd_rocksdb_get_slot( ulong cf_idx, char const * key ) {
     303           0 :   switch (cf_idx) {
     304           0 :     case FD_ROCKSDB_CFIDX_TRANSACTION_STATUS:
     305           0 :       return fd_ulong_bswap(*((ulong *) &key[72])); /* (signature,slot)*/
     306           0 :     case FD_ROCKSDB_CFIDX_ADDRESS_SIGNATURES:
     307           0 :       return fd_ulong_bswap(*((ulong *) &key[40])); /* (pubkey,slot,u32,signature) */
     308           0 :     default: /* all other cfs have the slot at the start */
     309           0 :       return fd_ulong_bswap( *((ulong *)&key[0]) ); /* The key is just the slot number */
     310           0 :   }
     311             : 
     312           0 :   return fd_ulong_bswap( *((ulong *)key) );
     313           0 : }
     314             : 
     315             : void
     316           0 : fd_rocksdb_iter_seek_to_slot_if_possible( rocksdb_iterator_t * iter, const ulong cf_idx, const ulong slot ) {
     317           0 :   ulong k = fd_ulong_bswap(slot);
     318           0 :   switch (cf_idx) {
     319             :     /* These cfs do not have the slot at the start, we can't seek based on slot prefix */
     320           0 :     case FD_ROCKSDB_CFIDX_TRANSACTION_STATUS:
     321           0 :     case FD_ROCKSDB_CFIDX_ADDRESS_SIGNATURES:
     322           0 :       rocksdb_iter_seek_to_first( iter );
     323           0 :       break;
     324           0 :     default: /* all other cfs have the slot at the start, seek based on slot prefix */
     325           0 :       rocksdb_iter_seek( iter, (const char *)&k, 8);
     326           0 :       break;
     327           0 :   }
     328           0 : }
     329             : 
     330             : int
     331             : fd_rocksdb_copy_over_slot_indexed_range( fd_rocksdb_t * src,
     332             :                                          fd_rocksdb_t * dst,
     333             :                                          ulong          cf_idx,
     334             :                                          ulong          start_slot,
     335           0 :                                          ulong          end_slot ) {
     336           0 :   FD_LOG_NOTICE(( "fd_rocksdb_copy_over_slot_indexed_range: %lu", cf_idx ));
     337             : 
     338           0 :   if ( cf_idx == FD_ROCKSDB_CFIDX_TRANSACTION_MEMOS  ||
     339           0 :        cf_idx == FD_ROCKSDB_CFIDX_TRANSACTION_STATUS ||
     340           0 :        cf_idx == FD_ROCKSDB_CFIDX_ADDRESS_SIGNATURES ) {
     341           0 :     FD_LOG_NOTICE(( "fd_rocksdb_copy_over_range: skipping cf_idx=%lu because not slot indexed", cf_idx ));
     342           0 :     return 0;
     343           0 :   }
     344             : 
     345           0 :   rocksdb_iterator_t * iter = rocksdb_create_iterator_cf( src->db, src->ro, src->cf_handles[cf_idx] );
     346           0 :   if ( FD_UNLIKELY( iter == NULL ) ) {
     347           0 :     FD_LOG_ERR(( "rocksdb_create_iterator_cf failed for cf_idx=%lu", cf_idx ));
     348           0 :   }
     349             : 
     350           0 :   for ( fd_rocksdb_iter_seek_to_slot_if_possible( iter, cf_idx, start_slot ); rocksdb_iter_valid( iter ); rocksdb_iter_next( iter ) ) {
     351           0 :     ulong klen = 0;
     352           0 :     char const * key = rocksdb_iter_key( iter, &klen ); // There is no need to free key
     353             : 
     354           0 :     ulong slot = fd_rocksdb_get_slot( cf_idx, key );
     355           0 :     if ( slot < start_slot ) {
     356           0 :       continue;
     357           0 :     }
     358           0 :     else if ( slot > end_slot ) {
     359           0 :       break;
     360           0 :     }
     361             : 
     362           0 :     ulong vlen = 0;
     363           0 :     char const * value = rocksdb_iter_value( iter, &vlen );
     364             : 
     365           0 :     fd_rocksdb_insert_entry( dst, cf_idx, key, klen, value, vlen );
     366           0 :   }
     367           0 :   rocksdb_iter_destroy( iter );
     368           0 :   return 0;
     369           0 : }
     370             : 
     371             : int
     372             : fd_rocksdb_insert_entry( fd_rocksdb_t * db,
     373             :                          ulong          cf_idx,
     374             :                          const char *   key,
     375             :                          ulong          klen,
     376             :                          const char *   value,
     377             :                          ulong          vlen )
     378           0 : {
     379           0 :   char * err = NULL;
     380           0 :   rocksdb_put_cf( db->db, db->wo, db->cf_handles[cf_idx],
     381           0 :                   key, klen, value, vlen, &err );
     382           0 :   if( FD_UNLIKELY( err != NULL ) ) {
     383           0 :     FD_LOG_WARNING(( "rocksdb_put_cf failed with error %s", err ));
     384           0 :     return -1;
     385           0 :   }
     386           0 :   return 0;
     387           0 : }