Line data Source code
1 : #ifndef HEADER_fd_src_flamenco_runtime_fd_rocksdb_h
2 : #define HEADER_fd_src_flamenco_runtime_fd_rocksdb_h
3 :
4 : #include "../../ballet/block/fd_microblock.h"
5 : #include "fd_blockstore.h"
6 :
7 : /** allocations made for offline-replay in the blockstore */
8 : struct fd_block {
9 : /* Used only in offline at the moment. Stored in the blockstore
10 : memory and used to iterate the block's contents.
11 :
12 : A block's data region is indexed to support iterating by shred,
13 : microblock/entry batch, microblock/entry, or transaction.
14 : This is done by iterating the headers for each, stored in allocated
15 : memory.
16 : To iterate shred payloads, for example, a caller should iterate the headers in tandem with the data region
17 : (offsetting by the bytes indicated in the shred header).
18 :
19 : Note random access of individual shred indices is not performant, due to the variable-length
20 : nature of shreds. */
21 :
22 : ulong data_gaddr; /* ptr to the beginning of the block's allocated data region */
23 : ulong data_sz; /* block size */
24 : ulong shreds_gaddr; /* ptr to the first fd_block_shred_t */
25 : ulong shreds_cnt;
26 : ulong batch_gaddr; /* list of fd_block_entry_batch_t */
27 : ulong batch_cnt;
28 : ulong micros_gaddr; /* ptr to the list of fd_block_micro_t */
29 : ulong micros_cnt;
30 : ulong txns_gaddr; /* ptr to the list of fd_block_txn_t */
31 : ulong txns_cnt;
32 : ulong txns_meta_gaddr; /* ptr to the allocation for txn meta data */
33 : ulong txns_meta_sz;
34 : };
35 : typedef struct fd_block fd_block_t;
36 :
37 : FD_PROTOTYPES_BEGIN
38 :
39 : /* fd_blockstore_block_data_laddr returns a local pointer to the block's
40 : data. The returned pointer lifetime is until the block is removed. */
41 :
42 : FD_FN_PURE static inline uchar *
43 0 : fd_blockstore_block_data_laddr( fd_blockstore_t * blockstore, fd_block_t * block ) {
44 0 : return fd_wksp_laddr_fast( fd_blockstore_wksp( blockstore ), block->data_gaddr );
45 0 : }
46 :
47 : FD_FN_PURE static inline fd_block_entry_batch_t *
48 0 : fd_blockstore_block_batch_laddr( fd_blockstore_t * blockstore, fd_block_t * block ) {
49 0 : return fd_wksp_laddr_fast( fd_blockstore_wksp( blockstore ), block->batch_gaddr );
50 0 : }
51 :
52 : FD_FN_PURE static inline fd_block_micro_t *
53 0 : fd_blockstore_block_micro_laddr( fd_blockstore_t * blockstore, fd_block_t * block ) {
54 0 : return fd_wksp_laddr_fast( fd_blockstore_wksp( blockstore ), block->micros_gaddr );
55 0 : }
56 :
57 : FD_PROTOTYPES_END
58 :
59 : #if FD_HAS_ROCKSDB
60 :
61 : #include "../../ballet/shred/fd_shred.h"
62 : #include <rocksdb/c.h>
63 :
64 0 : #define FD_ROCKSDB_CF_CNT (21UL)
65 :
66 0 : #define FD_ROCKSDB_CFIDX_DEFAULT (0UL)
67 0 : #define FD_ROCKSDB_CFIDX_META (1UL)
68 0 : #define FD_ROCKSDB_CFIDX_DEAD_SLOTS (2UL)
69 0 : #define FD_ROCKSDB_CFIDX_DUPLICATE_SLOTS (3UL) /* Usually empty */
70 0 : #define FD_ROCKSDB_CFIDX_ERASURE_META (4UL)
71 0 : #define FD_ROCKSDB_CFIDX_ORPHANS (5UL) /* Usually empty */
72 0 : #define FD_ROCKSDB_CFIDX_BANK_HASHES (6UL)
73 0 : #define FD_ROCKSDB_CFIDX_ROOT (7UL)
74 0 : #define FD_ROCKSDB_CFIDX_INDEX (8UL)
75 0 : #define FD_ROCKSDB_CFIDX_DATA_SHRED (9UL)
76 0 : #define FD_ROCKSDB_CFIDX_CODE_SHRED (10UL)
77 0 : #define FD_ROCKSDB_CFIDX_TRANSACTION_STATUS (11UL)
78 0 : #define FD_ROCKSDB_CFIDX_ADDRESS_SIGNATURES (12UL)
79 0 : #define FD_ROCKSDB_CFIDX_TRANSACTION_MEMOS (13UL)
80 0 : #define FD_ROCKSDB_CFIDX_TRANSACTION_STATUS_INDEX (14UL)
81 0 : #define FD_ROCKSDB_CFIDX_REWARDS (15UL)
82 0 : #define FD_ROCKSDB_CFIDX_BLOCKTIME (16UL)
83 0 : #define FD_ROCKSDB_CFIDX_PERF_SAMPLES (17UL)
84 0 : #define FD_ROCKSDB_CFIDX_BLOCK_HEIGHT (18UL)
85 0 : #define FD_ROCKSDB_CFIDX_OPTIMISTIC_SLOTS (19UL)
86 0 : #define FD_ROCKSDB_CFIDX_MERKLE_ROOT_META (20UL) /* Usually empty */
87 :
88 : /* Solana rocksdb client */
89 : struct fd_rocksdb {
90 : rocksdb_t * db;
91 : const char * db_name;
92 : const char * cfgs [ FD_ROCKSDB_CF_CNT ];
93 : rocksdb_column_family_handle_t* cf_handles[ FD_ROCKSDB_CF_CNT ];
94 : rocksdb_options_t * opts;
95 : rocksdb_readoptions_t * ro;
96 : rocksdb_writeoptions_t * wo;
97 : };
98 : typedef struct fd_rocksdb fd_rocksdb_t;
99 : #define FD_ROCKSDB_FOOTPRINT sizeof(fd_rocksdb_t)
100 : #define FD_ROCKSDB_ALIGN (8UL)
101 :
102 : /* root column iterator */
103 : struct fd_rocksdb_root_iter {
104 : fd_rocksdb_t * db;
105 : rocksdb_iterator_t* iter;
106 : };
107 : typedef struct fd_rocksdb_root_iter fd_rocksdb_root_iter_t;
108 : #define FD_ROCKSDB_ROOT_ITER_FOOTPRINT sizeof(fd_rocksdb_root_iter_t)
109 : #define FD_ROCKSDB_ROOT_ITER_ALIGN (8UL)
110 :
111 : FD_PROTOTYPES_BEGIN
112 :
113 : void *
114 : fd_rocksdb_root_iter_new( void * shiter );
115 :
116 : fd_rocksdb_root_iter_t *
117 : fd_rocksdb_root_iter_join( void * iter );
118 :
119 : void *
120 : fd_rocksdb_root_iter_leave( fd_rocksdb_root_iter_t * iter );
121 :
122 : /* fd_rocksdb_root_iter_seek
123 :
124 : 0 = success
125 : -1 = seek for supplied slot failed
126 : -2 = seek succeeded but slot did not match what we seeked for
127 : -3 = seek succeeded but points at an empty slot */
128 :
129 : int
130 : fd_rocksdb_root_iter_seek( fd_rocksdb_root_iter_t * iter,
131 : fd_rocksdb_t * db,
132 : ulong slot,
133 : fd_slot_meta_t * m,
134 : fd_valloc_t valloc );
135 :
136 : /* fd_rocksdb_root_iter_next
137 :
138 : 0 = success
139 : -1 = not properly initialized with a seek
140 : -2 = invalid starting iterator
141 : -3 = next returned an invalid iterator state
142 : -4 = seek succeeded but points at an empty slot */
143 :
144 : int
145 : fd_rocksdb_root_iter_next( fd_rocksdb_root_iter_t * iter,
146 : fd_slot_meta_t * m,
147 : fd_valloc_t valloc );
148 :
149 : int
150 : fd_rocksdb_root_iter_slot( fd_rocksdb_root_iter_t * self,
151 : ulong * slot );
152 :
153 : void
154 : fd_rocksdb_root_iter_destroy( fd_rocksdb_root_iter_t * iter );
155 :
156 : /* fd_rocksdb_init: Returns a pointer to a description of the error on failure
157 :
158 : The provided db_name needs to point at the actual rocksdb directory
159 : as apposed to the directory above (like the solana ledger-tool) */
160 :
161 : char *
162 : fd_rocksdb_init( fd_rocksdb_t * db,
163 : char const * db_name );
164 :
165 : /* fd_rocksdb_new: Creates a new rocksdb
166 :
167 : The provided db_name has to the be the full path where the directory
168 : will be created. The fd_rocksdb_t object will be initialized */
169 :
170 : void
171 : fd_rocksdb_new( fd_rocksdb_t * db,
172 : char const * db_name );
173 :
174 : /* fd_rocksdb_destroy
175 :
176 : Frees up the internal data structures */
177 :
178 : void
179 : fd_rocksdb_destroy( fd_rocksdb_t * db );
180 :
181 : /* fd_rocksdb_last_slot: Returns the last slot in the db
182 :
183 : This uses the root column to discover the slot of the last root in
184 : the db. If there is an error, this sets *err to a constant string
185 : describing the error. There is no need to free that string. */
186 :
187 : ulong
188 : fd_rocksdb_last_slot( fd_rocksdb_t * db,
189 : char ** err );
190 :
191 : /* fd_rocksdb_first_slot: Returns the first slot in the db
192 :
193 : This uses the root column to discover the slot of the first root in
194 : the db. If there is an error, this sets *err to a constant string
195 : describing the error. There is no need to free that string. */
196 :
197 : ulong
198 : fd_rocksdb_first_slot( fd_rocksdb_t * db,
199 : char ** err );
200 :
201 : ulong
202 : fd_rocksdb_find_last_slot( fd_rocksdb_t * db,
203 : char ** err );
204 :
205 : /* fd_rocksdb_get_meta
206 :
207 : Retrieves the meta structure associated with the supplied slot. If
208 : there is an error, *err is set to a string describing the error.
209 : It is expected that you should free() the error once done with it
210 :
211 : returns a 0 if there is no obvious error */
212 : int
213 : fd_rocksdb_get_meta( fd_rocksdb_t * db,
214 : ulong slot,
215 : fd_slot_meta_t * m,
216 : fd_valloc_t valloc );
217 :
218 : /* fd_rocksdb_get_txn_status_raw queries transaction status metadata.
219 : slot is the slot number of the block that contains the txn. sig
220 : points to the first signature of the txn. Returns data==NULL if
221 : record not found. On success, creates a malloc-backed buffer to hold
222 : return value, copies raw serialized status into buffer, sets *psz to
223 : the byte size of the status and returns pointer to buffer. Caller
224 : must free() non-NULL returned region. On failure, returns NULL and
225 : content of *psz is undefined. Value is Protobuf-encoded
226 : TransactionStatusMeta. Use fd_solblock nanopb API to deserialize
227 : value. */
228 :
229 : void *
230 : fd_rocksdb_get_txn_status_raw( fd_rocksdb_t * self,
231 : ulong slot,
232 : void const * sig,
233 : ulong * psz );
234 :
235 : /* fd_rocksdb_copy_over_slot_indexed_range copies over all entries for a
236 : given column family index into another rocksdb assuming that the key
237 : is prefixed with the slot number. This includes column families where
238 : the key is just the slot number but also ones where the key starts with
239 : the slot number. */
240 :
241 : int
242 : fd_rocksdb_copy_over_slot_indexed_range( fd_rocksdb_t * src,
243 : fd_rocksdb_t * dst,
244 : ulong cf_idx,
245 : ulong start_slot,
246 : ulong end_slot );
247 :
248 : /* fd_rocksdb_copy_over_txn_status_range copies over all transaction statuses
249 : within a block range assuming the blockstore contains relevant pointers to
250 : the transactions within the range. The blockstore object must be populated
251 : with the relevant block range. */
252 :
253 : int
254 : fd_rocksdb_copy_over_txn_status_range( fd_rocksdb_t * src,
255 : fd_rocksdb_t * dst,
256 : fd_blockstore_t * blockstore,
257 : ulong start_slot,
258 : ulong end_slot );
259 :
260 : /* fd_rocksdb_copy_over_txn_status constructs a key to query a transaction
261 : status and copies over the entry into another rocksdb. The index is used
262 : to specify which transaction. */
263 :
264 : void
265 : fd_rocksdb_copy_over_txn_status( fd_rocksdb_t * src,
266 : fd_rocksdb_t * dst,
267 : ulong slot,
268 : void const * sig );
269 :
270 : /* fd_rocksdb_insert_entry inserts a key, value pair into a given rocksdb */
271 :
272 : int
273 : fd_rocksdb_insert_entry( fd_rocksdb_t * db,
274 : ulong cf_idx,
275 : const char * key,
276 : ulong key_len,
277 : const char * value,
278 : ulong value_len );
279 :
280 : /* Import from rocksdb into blockstore */
281 :
282 : int
283 : fd_rocksdb_import_block_blockstore( fd_rocksdb_t * db,
284 : fd_slot_meta_t * m,
285 : fd_blockstore_t * blockstore,
286 : int txnstatus,
287 : const uchar * hash_override,
288 : fd_valloc_t valloc );
289 :
290 : int
291 : fd_rocksdb_import_block_shredcap( fd_rocksdb_t * db,
292 : fd_slot_meta_t * metadata,
293 : fd_io_buffered_ostream_t * ostream,
294 : fd_io_buffered_ostream_t * bank_hash_ostream,
295 : fd_valloc_t valloc );
296 :
297 : /* fd_blockstore_block_query queries blockstore for block at slot.
298 : Returns a pointer to the block or NULL if not in blockstore. The
299 : returned pointer lifetime is until the block is removed. Check
300 : return value for error info.
301 :
302 : In theory the caller does not need to wrap this function in a
303 : start/end read. What is being read lives in the block_info object,
304 : and this function does a valid concurrent read for the block_gaddr.
305 : The fd_block_t object itself has no such guarantees, and needs a
306 : read/write lock to modify. */
307 : void
308 : fd_blockstore_block_allocs_remove( fd_blockstore_t * blockstore, ulong slot );
309 :
310 : static inline fd_block_t *
311 0 : fd_blockstore_block_query(fd_blockstore_t *blockstore, ulong slot){
312 0 : int err = FD_MAP_ERR_AGAIN;
313 0 : ulong query_block_gaddr = 0;
314 0 : while( err == FD_MAP_ERR_AGAIN ){
315 0 : fd_block_map_query_t quer[1] = { 0 };
316 0 : err = fd_block_map_query_try( blockstore->block_map, &slot, NULL, quer, 0 );
317 0 : fd_block_info_t * query = fd_block_map_query_ele( quer );
318 0 : if ( err == FD_MAP_ERR_KEY ) return NULL;
319 0 : if ( FD_UNLIKELY( err == FD_MAP_ERR_AGAIN ) ) continue;
320 : /* later change this to all shreds received */
321 0 : if( FD_UNLIKELY( query->block_gaddr == 0 ) ) return NULL;
322 0 : query_block_gaddr = query->block_gaddr;
323 0 : err = fd_block_map_query_test( quer );
324 0 : }
325 0 : return fd_wksp_laddr_fast( fd_blockstore_wksp( blockstore ), query_block_gaddr );
326 0 : }
327 :
328 : FD_PROTOTYPES_END
329 :
330 : #endif
331 :
332 : #endif // HEADER_fd_src_flamenco_runtime_fd_rocksdb_h
|
