Line data Source code
1 : #if !FD_HAS_ROCKSDB
2 : #error "fd_ledger requires RocksDB"
3 : #endif
4 :
5 : #include "../../util/fd_util.h"
6 : #include <rocksdb/c.h>
7 : #include <sys/stat.h>
8 :
9 : static char const * const fd_rocksdb_cf_names[] = {
10 : "default",
11 : "meta",
12 : "dead_slots",
13 : "duplicate_slots",
14 : "erasure_meta",
15 : "orphans",
16 : "bank_hashes",
17 : "root",
18 : "index",
19 : "data_shred",
20 : "code_shred",
21 : "transaction_status",
22 : "address_signatures",
23 : "transaction_memos",
24 : "rewards",
25 : "blocktime",
26 : "perf_samples",
27 : "block_height",
28 : "optimistic_slots",
29 : "merkle_root_meta",
30 : };
31 :
32 0 : #define FD_ROCKSDB_CF_CNT (sizeof(fd_rocksdb_cf_names)/sizeof(fd_rocksdb_cf_names[0]))
33 :
34 0 : #define FD_ROCKSDB_CFIDX_ROOT (7UL)
35 0 : #define FD_ROCKSDB_CFIDX_TRANSACTION_STATUS (11UL)
36 0 : #define FD_ROCKSDB_CFIDX_ADDRESS_SIGNATURES (12UL)
37 0 : #define FD_ROCKSDB_CFIDX_TRANSACTION_MEMOS (13UL)
38 :
39 : struct fd_rocksdb {
40 : rocksdb_t * db;
41 : rocksdb_column_family_handle_t * cf_handles[ FD_ROCKSDB_CF_CNT ];
42 : rocksdb_options_t * opts;
43 : rocksdb_readoptions_t * ro;
44 : rocksdb_writeoptions_t * wo;
45 : };
46 : typedef struct fd_rocksdb fd_rocksdb_t;
47 :
48 : static char *
49 : fd_rocksdb_init( fd_rocksdb_t * db,
50 0 : char const * db_name ) {
51 0 : fd_memset( db, 0, sizeof(fd_rocksdb_t) );
52 :
53 0 : db->opts = rocksdb_options_create();
54 :
55 0 : rocksdb_options_t const * cf_options[ FD_ROCKSDB_CF_CNT ];
56 0 : for( ulong i=0UL; i<FD_ROCKSDB_CF_CNT; i++ )
57 0 : cf_options[ i ] = db->opts;
58 :
59 0 : char * err = NULL;
60 :
61 0 : db->db = rocksdb_open_for_read_only_column_families(
62 0 : db->opts,
63 0 : db_name,
64 0 : FD_ROCKSDB_CF_CNT,
65 0 : fd_rocksdb_cf_names,
66 0 : (rocksdb_options_t const * const *)cf_options,
67 0 : db->cf_handles,
68 0 : 0,
69 0 : &err );
70 :
71 0 : if( FD_UNLIKELY( err ) ) return err;
72 :
73 0 : db->ro = rocksdb_readoptions_create();
74 :
75 0 : return NULL;
76 0 : }
77 :
78 : static void
79 : fd_rocksdb_new( fd_rocksdb_t * db,
80 0 : char const * db_name ) {
81 0 : fd_memset( db, 0, sizeof(fd_rocksdb_t) );
82 :
83 0 : db->opts = rocksdb_options_create();
84 0 : rocksdb_options_set_create_if_missing( db->opts, 1 );
85 0 : rocksdb_options_set_compression( db->opts, rocksdb_lz4_compression );
86 :
87 0 : char * err = NULL;
88 0 : db->db = rocksdb_open( db->opts, db_name, &err );
89 0 : if( FD_UNLIKELY( err ) ) {
90 0 : FD_LOG_ERR(( "rocksdb creation failed: %s", err ));
91 0 : }
92 :
93 0 : db->wo = rocksdb_writeoptions_create();
94 :
95 0 : for( ulong i=1UL; i<FD_ROCKSDB_CF_CNT; i++ ) {
96 0 : db->cf_handles[i] = rocksdb_create_column_family( db->db, db->opts, fd_rocksdb_cf_names[i], &err );
97 0 : }
98 0 : }
99 :
100 : static void
101 0 : fd_rocksdb_destroy( fd_rocksdb_t * db ) {
102 :
103 0 : for( ulong i=0UL; i<FD_ROCKSDB_CF_CNT; i++ ) {
104 0 : if( db->cf_handles[i] ) {
105 0 : rocksdb_column_family_handle_destroy( db->cf_handles[i] );
106 0 : db->cf_handles[i] = NULL;
107 0 : }
108 0 : }
109 :
110 0 : if( db->ro ) {
111 0 : rocksdb_readoptions_destroy( db->ro );
112 0 : db->ro = NULL;
113 0 : }
114 :
115 0 : if( db->opts ) {
116 0 : rocksdb_options_destroy( db->opts );
117 0 : db->opts = NULL;
118 0 : }
119 :
120 0 : if( db->db ) {
121 0 : rocksdb_close( db->db );
122 0 : db->db = NULL;
123 0 : }
124 :
125 0 : if( db->wo ) {
126 0 : rocksdb_writeoptions_destroy( db->wo );
127 0 : }
128 0 : }
129 :
130 : static ulong
131 : fd_rocksdb_root_slot( fd_rocksdb_t * db,
132 : int last,
133 0 : char ** err ) {
134 0 : rocksdb_iterator_t * iter = rocksdb_create_iterator_cf( db->db, db->ro, db->cf_handles[FD_ROCKSDB_CFIDX_ROOT] );
135 0 : if( last ) rocksdb_iter_seek_to_last ( iter );
136 0 : else rocksdb_iter_seek_to_first( iter );
137 :
138 0 : if( FD_UNLIKELY( !rocksdb_iter_valid( iter ) ) ) {
139 0 : rocksdb_iter_destroy( iter );
140 0 : *err = "db column for root is empty";
141 0 : return 0;
142 0 : }
143 :
144 0 : ulong klen = 0;
145 0 : char const * key = rocksdb_iter_key( iter, &klen );
146 0 : ulong slot = fd_ulong_bswap( FD_LOAD( ulong, key ) );
147 0 : rocksdb_iter_destroy( iter );
148 0 : return slot;
149 0 : }
150 :
151 : static void
152 : fd_rocksdb_copy_over_slot_indexed_range( fd_rocksdb_t * src,
153 : fd_rocksdb_t * dst,
154 : ulong cf_idx,
155 : ulong start_slot,
156 0 : ulong end_slot ) {
157 0 : FD_LOG_NOTICE(( "fd_rocksdb_copy_over_slot_indexed_range: %lu", cf_idx ));
158 :
159 0 : if( cf_idx==FD_ROCKSDB_CFIDX_TRANSACTION_MEMOS ||
160 0 : cf_idx==FD_ROCKSDB_CFIDX_TRANSACTION_STATUS ||
161 0 : cf_idx==FD_ROCKSDB_CFIDX_ADDRESS_SIGNATURES ) {
162 0 : FD_LOG_NOTICE(( "fd_rocksdb_copy_over_range: skipping cf_idx=%lu because not slot indexed", cf_idx ));
163 0 : return;
164 0 : }
165 :
166 0 : rocksdb_iterator_t * iter = rocksdb_create_iterator_cf( src->db, src->ro, src->cf_handles[cf_idx] );
167 0 : if( FD_UNLIKELY( !iter ) ) {
168 0 : FD_LOG_ERR(( "rocksdb_create_iterator_cf failed for cf_idx=%lu", cf_idx ));
169 0 : }
170 :
171 0 : ulong start_key = fd_ulong_bswap( start_slot );
172 0 : for( rocksdb_iter_seek( iter, (char const *)&start_key, sizeof(start_key) ); rocksdb_iter_valid( iter ); rocksdb_iter_next( iter ) ) {
173 0 : ulong klen = 0;
174 0 : char const * key = rocksdb_iter_key( iter, &klen );
175 :
176 0 : ulong slot = fd_ulong_bswap( FD_LOAD( ulong, key ) );
177 0 : if( slot<start_slot ) {
178 0 : continue;
179 0 : } else if( slot>end_slot ) {
180 0 : break;
181 0 : }
182 :
183 0 : ulong vlen = 0;
184 0 : char const * value = rocksdb_iter_value( iter, &vlen );
185 :
186 0 : char * err = NULL;
187 0 : rocksdb_put_cf( dst->db, dst->wo, dst->cf_handles[cf_idx], key, klen, value, vlen, &err );
188 0 : if( FD_UNLIKELY( err ) ) {
189 0 : FD_LOG_WARNING(( "rocksdb_put_cf failed with error %s", err ));
190 0 : rocksdb_free( err );
191 0 : }
192 0 : }
193 0 : rocksdb_iter_destroy( iter );
194 0 : }
195 :
196 : /********************* Main Command Functions and Setup ***********************/
197 : static void
198 : minify( char const * rocksdb_path,
199 : char const * mini_db_dir,
200 : ulong start_slot,
201 0 : ulong end_slot ) {
202 : /* Example command:
203 : fd_ledger --cmd minify --rocksdb <LARGE_ROCKSDB> --minified-rocksdb <MINI_ROCKSDB>
204 : --start-slot <START_SLOT> --end-slot <END_SLOT> */
205 0 : if( FD_UNLIKELY( !rocksdb_path ) ) {
206 0 : FD_LOG_ERR(( "rocksdb path is NULL" ));
207 0 : }
208 0 : if( FD_UNLIKELY( !mini_db_dir ) ) {
209 0 : FD_LOG_ERR(( "minified rocksdb path is NULL" ));
210 0 : }
211 :
212 0 : fd_rocksdb_t big_rocksdb;
213 0 : char * err = fd_rocksdb_init( &big_rocksdb, rocksdb_path );
214 0 : if( FD_UNLIKELY( err ) ) {
215 0 : FD_LOG_ERR(( "fd_rocksdb_init at path=%s returned error=%s", rocksdb_path, err ));
216 0 : }
217 :
218 : /* If the directory for the minified rocksdb already exists, error out */
219 0 : struct stat statbuf;
220 0 : if( stat( mini_db_dir, &statbuf ) == 0 ) {
221 0 : FD_LOG_ERR(( "path for mini_db_dir=%s already exists", mini_db_dir ));
222 0 : }
223 :
224 : /* Create a new smaller rocksdb */
225 0 : fd_rocksdb_t mini_rocksdb;
226 0 : fd_rocksdb_new( &mini_rocksdb, mini_db_dir );
227 :
228 : /* Correctly bound off start and end slot */
229 0 : ulong first_slot = fd_rocksdb_root_slot( &big_rocksdb, 0, &err );
230 0 : ulong last_slot = fd_rocksdb_root_slot( &big_rocksdb, 1, &err );
231 0 : if( start_slot < first_slot ) start_slot = first_slot;
232 0 : if( end_slot > last_slot ) end_slot = last_slot;
233 :
234 0 : FD_LOG_NOTICE(( "copying over rocks db for range [%lu, %lu]", start_slot, end_slot ));
235 :
236 : /* Copy over all slot indexed columns */
237 0 : for( ulong cf_idx=1UL; cf_idx<FD_ROCKSDB_CF_CNT; cf_idx++ ) {
238 0 : fd_rocksdb_copy_over_slot_indexed_range( &big_rocksdb, &mini_rocksdb, cf_idx,
239 0 : start_slot, end_slot );
240 0 : }
241 0 : FD_LOG_NOTICE(( "copied over all slot indexed columns" ));
242 :
243 : /* TODO: Currently, the address signatures column family isn't copied as it
244 : is indexed on the pubkey. */
245 :
246 0 : rocksdb_flushoptions_t * flush_options = rocksdb_flushoptions_create();
247 0 : rocksdb_flushoptions_set_wait( flush_options, 1 );
248 0 : char * flush_err = NULL;
249 0 : rocksdb_flush_cfs( mini_rocksdb.db, flush_options,
250 0 : &mini_rocksdb.cf_handles[ 1 ],
251 0 : FD_ROCKSDB_CF_CNT - 1, &flush_err );
252 0 : if( FD_UNLIKELY( flush_err ) ) {
253 0 : FD_LOG_WARNING(( "minify: flushing minified rocksdb failed: %s", flush_err ));
254 0 : rocksdb_free( flush_err );
255 0 : }
256 0 : rocksdb_flushoptions_destroy( flush_options );
257 :
258 0 : fd_rocksdb_destroy( &big_rocksdb );
259 0 : fd_rocksdb_destroy( &mini_rocksdb );
260 0 : }
261 :
262 : int
263 : main( int argc,
264 : char ** argv ) {
265 : if( FD_UNLIKELY( argc==1 ) ) {
266 : FD_LOG_ERR(( "no command specified" ));
267 : }
268 :
269 : fd_boot( &argc, &argv );
270 :
271 : char const * cmd = fd_env_strip_cmdline_cstr ( &argc, &argv, "--cmd", NULL, NULL );
272 : ulong start_slot = fd_env_strip_cmdline_ulong ( &argc, &argv, "--start-slot", NULL, 0UL );
273 : ulong end_slot = fd_env_strip_cmdline_ulong ( &argc, &argv, "--end-slot", NULL, ULONG_MAX );
274 : char const * mini_db_dir = fd_env_strip_cmdline_cstr ( &argc, &argv, "--minified-rocksdb", NULL, NULL );
275 : char const * rocksdb_path = fd_env_strip_cmdline_cstr ( &argc, &argv, "--rocksdb", NULL, NULL );
276 :
277 : if( rocksdb_path ) {
278 : FD_LOG_NOTICE(( "rocksdb=%s", rocksdb_path ));
279 : }
280 :
281 : if( FD_UNLIKELY( !cmd ) ) {
282 : FD_LOG_ERR(( "no command specified" ));
283 : } else if( strcmp( cmd, "minify" ) == 0 ) {
284 : minify( rocksdb_path, mini_db_dir, start_slot, end_slot );
285 : } else {
286 : FD_LOG_ERR(( "unknown command=%s", cmd ));
287 : }
288 :
289 : return 0;
290 : }
|
