Line data Source code
1 : #ifndef HEADER_fd_src_flamenco_runtime_fd_rocksdb_h
2 : #define HEADER_fd_src_flamenco_runtime_fd_rocksdb_h
3 :
4 : #include "../../ballet/block/fd_microblock.h"
5 : #include "fd_blockstore.h"
6 :
7 : /** allocations made for offline-replay in the blockstore */
8 : struct fd_block {
9 : /* Used only in offline at the moment. Stored in the blockstore
10 : memory and used to iterate the block's contents.
11 :
12 : A block's data region is indexed to support iterating by shred,
13 : microblock/entry batch, microblock/entry, or transaction.
14 : This is done by iterating the headers for each, stored in allocated
15 : memory.
16 : To iterate shred payloads, for example, a caller should iterate the headers in tandem with the data region
17 : (offsetting by the bytes indicated in the shred header).
18 :
19 : Note random access of individual shred indices is not performant, due to the variable-length
20 : nature of shreds. */
21 :
22 : ulong data_gaddr; /* ptr to the beginning of the block's allocated data region */
23 : ulong data_sz; /* block size */
24 : ulong shreds_gaddr; /* ptr to the first fd_block_shred_t */
25 : ulong shreds_cnt;
26 : ulong batch_gaddr; /* list of fd_block_entry_batch_t */
27 : ulong batch_cnt;
28 : ulong micros_gaddr; /* ptr to the list of fd_block_micro_t */
29 : ulong micros_cnt;
30 : ulong txns_gaddr; /* ptr to the list of fd_block_txn_t */
31 : ulong txns_cnt;
32 : ulong txns_meta_gaddr; /* ptr to the allocation for txn meta data */
33 : ulong txns_meta_sz;
34 : };
35 : typedef struct fd_block fd_block_t;
36 :
37 : FD_PROTOTYPES_BEGIN
38 :
39 : /* fd_blockstore_block_data_laddr returns a local pointer to the block's
40 : data. The returned pointer lifetime is until the block is removed. */
41 :
42 : FD_FN_PURE static inline uchar *
43 0 : fd_blockstore_block_data_laddr( fd_blockstore_t * blockstore, fd_block_t * block ) {
44 0 : return fd_wksp_laddr_fast( fd_blockstore_wksp( blockstore ), block->data_gaddr );
45 0 : }
46 :
47 : FD_FN_PURE static inline fd_block_entry_batch_t *
48 0 : fd_blockstore_block_batch_laddr( fd_blockstore_t * blockstore, fd_block_t * block ) {
49 0 : return fd_wksp_laddr_fast( fd_blockstore_wksp( blockstore ), block->batch_gaddr );
50 0 : }
51 :
52 : FD_FN_PURE static inline fd_block_micro_t *
53 0 : fd_blockstore_block_micro_laddr( fd_blockstore_t * blockstore, fd_block_t * block ) {
54 0 : return fd_wksp_laddr_fast( fd_blockstore_wksp( blockstore ), block->micros_gaddr );
55 0 : }
56 :
57 : FD_PROTOTYPES_END
58 :
59 : #if FD_HAS_ROCKSDB
60 :
61 : #include "../../ballet/shred/fd_shred.h"
62 : #include <rocksdb/c.h>
63 :
64 0 : #define FD_ROCKSDB_CF_CNT (22UL)
65 :
66 0 : #define FD_ROCKSDB_CFIDX_DEFAULT (0UL)
67 0 : #define FD_ROCKSDB_CFIDX_META (1UL)
68 0 : #define FD_ROCKSDB_CFIDX_DEAD_SLOTS (2UL)
69 0 : #define FD_ROCKSDB_CFIDX_DUPLICATE_SLOTS (3UL) /* Usually empty */
70 0 : #define FD_ROCKSDB_CFIDX_ERASURE_META (4UL)
71 0 : #define FD_ROCKSDB_CFIDX_ORPHANS (5UL) /* Usually empty */
72 0 : #define FD_ROCKSDB_CFIDX_BANK_HASHES (6UL)
73 0 : #define FD_ROCKSDB_CFIDX_ROOT (7UL)
74 0 : #define FD_ROCKSDB_CFIDX_INDEX (8UL)
75 0 : #define FD_ROCKSDB_CFIDX_DATA_SHRED (9UL)
76 0 : #define FD_ROCKSDB_CFIDX_CODE_SHRED (10UL)
77 0 : #define FD_ROCKSDB_CFIDX_TRANSACTION_STATUS (11UL)
78 0 : #define FD_ROCKSDB_CFIDX_ADDRESS_SIGNATURES (12UL)
79 0 : #define FD_ROCKSDB_CFIDX_TRANSACTION_MEMOS (13UL)
80 0 : #define FD_ROCKSDB_CFIDX_TRANSACTION_STATUS_INDEX (14UL)
81 0 : #define FD_ROCKSDB_CFIDX_REWARDS (15UL)
82 0 : #define FD_ROCKSDB_CFIDX_BLOCKTIME (16UL)
83 0 : #define FD_ROCKSDB_CFIDX_PERF_SAMPLES (17UL)
84 0 : #define FD_ROCKSDB_CFIDX_BLOCK_HEIGHT (18UL)
85 0 : #define FD_ROCKSDB_CFIDX_PROGRAM_COSTS (19UL) /* Usually empty */
86 0 : #define FD_ROCKSDB_CFIDX_OPTIMISTIC_SLOTS (20UL)
87 0 : #define FD_ROCKSDB_CFIDX_MERKLE_ROOT_META (21UL) /* Usually empty */
88 :
89 : /* Solana rocksdb client */
90 : struct fd_rocksdb {
91 : rocksdb_t * db;
92 : const char * db_name;
93 : const char * cfgs [ FD_ROCKSDB_CF_CNT ];
94 : rocksdb_column_family_handle_t* cf_handles[ FD_ROCKSDB_CF_CNT ];
95 : rocksdb_options_t * opts;
96 : rocksdb_readoptions_t * ro;
97 : rocksdb_writeoptions_t * wo;
98 : };
99 : typedef struct fd_rocksdb fd_rocksdb_t;
100 : #define FD_ROCKSDB_FOOTPRINT sizeof(fd_rocksdb_t)
101 : #define FD_ROCKSDB_ALIGN (8UL)
102 :
103 : /* root column iterator */
104 : struct fd_rocksdb_root_iter {
105 : fd_rocksdb_t * db;
106 : rocksdb_iterator_t* iter;
107 : };
108 : typedef struct fd_rocksdb_root_iter fd_rocksdb_root_iter_t;
109 : #define FD_ROCKSDB_ROOT_ITER_FOOTPRINT sizeof(fd_rocksdb_root_iter_t)
110 : #define FD_ROCKSDB_ROOT_ITER_ALIGN (8UL)
111 :
112 : FD_PROTOTYPES_BEGIN
113 :
114 : void *
115 : fd_rocksdb_root_iter_new( void * shiter );
116 :
117 : fd_rocksdb_root_iter_t *
118 : fd_rocksdb_root_iter_join( void * iter );
119 :
120 : void *
121 : fd_rocksdb_root_iter_leave( fd_rocksdb_root_iter_t * iter );
122 :
123 : /* fd_rocksdb_root_iter_seek
124 :
125 : 0 = success
126 : -1 = seek for supplied slot failed
127 : -2 = seek succeeded but slot did not match what we seeked for
128 : -3 = seek succeeded but points at an empty slot */
129 :
130 : int
131 : fd_rocksdb_root_iter_seek( fd_rocksdb_root_iter_t * iter,
132 : fd_rocksdb_t * db,
133 : ulong slot,
134 : fd_slot_meta_t * m,
135 : fd_valloc_t valloc );
136 :
137 : /* fd_rocksdb_root_iter_next
138 :
139 : 0 = success
140 : -1 = not properly initialized with a seek
141 : -2 = invalid starting iterator
142 : -3 = next returned an invalid iterator state
143 : -4 = seek succeeded but points at an empty slot */
144 :
145 : int
146 : fd_rocksdb_root_iter_next( fd_rocksdb_root_iter_t * iter,
147 : fd_slot_meta_t * m,
148 : fd_valloc_t valloc );
149 :
150 : int
151 : fd_rocksdb_root_iter_slot( fd_rocksdb_root_iter_t * self,
152 : ulong * slot );
153 :
154 : void
155 : fd_rocksdb_root_iter_destroy( fd_rocksdb_root_iter_t * iter );
156 :
157 : /* fd_rocksdb_init: Returns a pointer to a description of the error on failure
158 :
159 : The provided db_name needs to point at the actual rocksdb directory
160 : as apposed to the directory above (like the solana ledger-tool) */
161 :
162 : char *
163 : fd_rocksdb_init( fd_rocksdb_t * db,
164 : char const * db_name );
165 :
166 : /* fd_rocksdb_new: Creates a new rocksdb
167 :
168 : The provided db_name has to the be the full path where the directory
169 : will be created. The fd_rocksdb_t object will be initialized */
170 :
171 : void
172 : fd_rocksdb_new( fd_rocksdb_t * db,
173 : char const * db_name );
174 :
175 : /* fd_rocksdb_destroy
176 :
177 : Frees up the internal data structures */
178 :
179 : void
180 : fd_rocksdb_destroy( fd_rocksdb_t * db );
181 :
182 : /* fd_rocksdb_last_slot: Returns the last slot in the db
183 :
184 : This uses the root column to discover the slot of the last root in
185 : the db. If there is an error, this sets *err to a constant string
186 : describing the error. There is no need to free that string. */
187 :
188 : ulong
189 : fd_rocksdb_last_slot( fd_rocksdb_t * db,
190 : char ** err );
191 :
192 : /* fd_rocksdb_first_slot: Returns the first slot in the db
193 :
194 : This uses the root column to discover the slot of the first root in
195 : the db. If there is an error, this sets *err to a constant string
196 : describing the error. There is no need to free that string. */
197 :
198 : ulong
199 : fd_rocksdb_first_slot( fd_rocksdb_t * db,
200 : char ** err );
201 :
202 : ulong
203 : fd_rocksdb_find_last_slot( fd_rocksdb_t * db,
204 : char ** err );
205 :
206 : /* fd_rocksdb_get_meta
207 :
208 : Retrieves the meta structure associated with the supplied slot. If
209 : there is an error, *err is set to a string describing the error.
210 : It is expected that you should free() the error once done with it
211 :
212 : returns a 0 if there is no obvious error */
213 : int
214 : fd_rocksdb_get_meta( fd_rocksdb_t * db,
215 : ulong slot,
216 : fd_slot_meta_t * m,
217 : fd_valloc_t valloc );
218 :
219 : /* fd_rocksdb_get_txn_status_raw queries transaction status metadata.
220 : slot is the slot number of the block that contains the txn. sig
221 : points to the first signature of the txn. Returns data==NULL if
222 : record not found. On success, creates a malloc-backed buffer to hold
223 : return value, copies raw serialized status into buffer, sets *psz to
224 : the byte size of the status and returns pointer to buffer. Caller
225 : must free() non-NULL returned region. On failure, returns NULL and
226 : content of *psz is undefined. Value is Protobuf-encoded
227 : TransactionStatusMeta. Use fd_solblock nanopb API to deserialize
228 : value. */
229 :
230 : void *
231 : fd_rocksdb_get_txn_status_raw( fd_rocksdb_t * self,
232 : ulong slot,
233 : void const * sig,
234 : ulong * psz );
235 :
236 : /* fd_rocksdb_copy_over_slot_indexed_range copies over all entries for a
237 : given column family index into another rocksdb assuming that the key
238 : is prefixed with the slot number. This includes column families where
239 : the key is just the slot number but also ones where the key starts with
240 : the slot number. */
241 :
242 : int
243 : fd_rocksdb_copy_over_slot_indexed_range( fd_rocksdb_t * src,
244 : fd_rocksdb_t * dst,
245 : ulong cf_idx,
246 : ulong start_slot,
247 : ulong end_slot );
248 :
249 : /* fd_rocksdb_copy_over_txn_status_range copies over all transaction statuses
250 : within a block range assuming the blockstore contains relevant pointers to
251 : the transactions within the range. The blockstore object must be populated
252 : with the relevant block range. */
253 :
254 : int
255 : fd_rocksdb_copy_over_txn_status_range( fd_rocksdb_t * src,
256 : fd_rocksdb_t * dst,
257 : fd_blockstore_t * blockstore,
258 : ulong start_slot,
259 : ulong end_slot );
260 :
261 : /* fd_rocksdb_copy_over_txn_status constructs a key to query a transaction
262 : status and copies over the entry into another rocksdb. The index is used
263 : to specify which transaction. */
264 :
265 : void
266 : fd_rocksdb_copy_over_txn_status( fd_rocksdb_t * src,
267 : fd_rocksdb_t * dst,
268 : ulong slot,
269 : void const * sig );
270 :
271 : /* fd_rocksdb_insert_entry inserts a key, value pair into a given rocksdb */
272 :
273 : int
274 : fd_rocksdb_insert_entry( fd_rocksdb_t * db,
275 : ulong cf_idx,
276 : const char * key,
277 : ulong key_len,
278 : const char * value,
279 : ulong value_len );
280 :
281 : /* Import from rocksdb into blockstore */
282 :
283 : int
284 : fd_rocksdb_import_block_blockstore( fd_rocksdb_t * db,
285 : fd_slot_meta_t * m,
286 : fd_blockstore_t * blockstore,
287 : int txnstatus,
288 : const uchar * hash_override,
289 : fd_valloc_t valloc );
290 :
291 : int
292 : fd_rocksdb_import_block_shredcap( fd_rocksdb_t * db,
293 : fd_slot_meta_t * metadata,
294 : fd_io_buffered_ostream_t * ostream,
295 : fd_io_buffered_ostream_t * bank_hash_ostream,
296 : fd_valloc_t valloc );
297 :
298 : /* fd_blockstore_block_query queries blockstore for block at slot.
299 : Returns a pointer to the block or NULL if not in blockstore. The
300 : returned pointer lifetime is until the block is removed. Check
301 : return value for error info.
302 :
303 : In theory the caller does not need to wrap this function in a
304 : start/end read. What is being read lives in the block_info object,
305 : and this function does a valid concurrent read for the block_gaddr.
306 : The fd_block_t object itself has no such guarantees, and needs a
307 : read/write lock to modify. */
308 : void
309 : fd_blockstore_block_allocs_remove( fd_blockstore_t * blockstore, ulong slot );
310 :
311 : static inline fd_block_t *
312 0 : fd_blockstore_block_query(fd_blockstore_t *blockstore, ulong slot){
313 0 : int err = FD_MAP_ERR_AGAIN;
314 0 : ulong query_block_gaddr = 0;
315 0 : while( err == FD_MAP_ERR_AGAIN ){
316 0 : fd_block_map_query_t quer[1] = { 0 };
317 0 : err = fd_block_map_query_try( blockstore->block_map, &slot, NULL, quer, 0 );
318 0 : fd_block_info_t * query = fd_block_map_query_ele( quer );
319 0 : if ( err == FD_MAP_ERR_KEY ) return NULL;
320 0 : if ( FD_UNLIKELY( err == FD_MAP_ERR_AGAIN ) ) continue;
321 : /* later change this to all shreds recieved */
322 0 : if( FD_UNLIKELY( query->block_gaddr == 0 ) ) return NULL;
323 0 : query_block_gaddr = query->block_gaddr;
324 0 : err = fd_block_map_query_test( quer );
325 0 : }
326 0 : return fd_wksp_laddr_fast( fd_blockstore_wksp( blockstore ), query_block_gaddr );
327 0 : }
328 :
329 : FD_PROTOTYPES_END
330 :
331 : #endif
332 :
333 : #endif // HEADER_fd_src_flamenco_runtime_fd_rocksdb_h
|
