Line data Source code
1 : #ifndef HEADER_fd_src_flamenco_runtime_fd_rocksdb_h
2 : #define HEADER_fd_src_flamenco_runtime_fd_rocksdb_h
3 :
4 : #include "../../ballet/block/fd_microblock.h"
5 : #include "fd_blockstore.h"
6 :
7 : /** allocations made for offline-replay in the blockstore */
8 : struct fd_block {
9 : /* Used only in offline at the moment. Stored in the blockstore
10 : memory and used to iterate the block's contents.
11 :
12 : A block's data region is indexed to support iterating by shred,
13 : microblock/entry batch, microblock/entry, or transaction.
14 : This is done by iterating the headers for each, stored in allocated
15 : memory.
16 : To iterate shred payloads, for example, a caller should iterate the headers in tandem with the data region
17 : (offsetting by the bytes indicated in the shred header).
18 :
19 : Note random access of individual shred indices is not performant, due to the variable-length
20 : nature of shreds. */
21 :
22 : ulong data_gaddr; /* ptr to the beginning of the block's allocated data region */
23 : ulong data_sz; /* block size */
24 : ulong shreds_gaddr; /* ptr to the first fd_block_shred_t */
25 : ulong shreds_cnt;
26 : ulong batch_gaddr; /* list of fd_block_entry_batch_t */
27 : ulong batch_cnt;
28 : ulong micros_gaddr; /* ptr to the list of fd_block_micro_t */
29 : ulong micros_cnt;
30 : };
31 : typedef struct fd_block fd_block_t;
32 :
33 : FD_PROTOTYPES_BEGIN
34 :
35 : /* fd_blockstore_block_data_laddr returns a local pointer to the block's
36 : data. The returned pointer lifetime is until the block is removed. */
37 :
38 : FD_FN_PURE static inline uchar *
39 0 : fd_blockstore_block_data_laddr( fd_blockstore_t * blockstore, fd_block_t * block ) {
40 0 : return fd_wksp_laddr_fast( fd_blockstore_wksp( blockstore ), block->data_gaddr );
41 0 : }
42 :
43 : FD_FN_PURE static inline fd_block_entry_batch_t *
44 0 : fd_blockstore_block_batch_laddr( fd_blockstore_t * blockstore, fd_block_t * block ) {
45 0 : return fd_wksp_laddr_fast( fd_blockstore_wksp( blockstore ), block->batch_gaddr );
46 0 : }
47 :
48 : FD_FN_PURE static inline fd_block_micro_t *
49 0 : fd_blockstore_block_micro_laddr( fd_blockstore_t * blockstore, fd_block_t * block ) {
50 0 : return fd_wksp_laddr_fast( fd_blockstore_wksp( blockstore ), block->micros_gaddr );
51 0 : }
52 :
53 : FD_PROTOTYPES_END
54 :
55 : #if FD_HAS_ROCKSDB
56 :
57 : #include "../../ballet/shred/fd_shred.h"
58 : #include <rocksdb/c.h>
59 :
60 0 : #define FD_ROCKSDB_CF_CNT (21UL)
61 :
62 0 : #define FD_ROCKSDB_CFIDX_DEFAULT (0UL)
63 0 : #define FD_ROCKSDB_CFIDX_META (1UL)
64 0 : #define FD_ROCKSDB_CFIDX_DEAD_SLOTS (2UL)
65 0 : #define FD_ROCKSDB_CFIDX_DUPLICATE_SLOTS (3UL) /* Usually empty */
66 0 : #define FD_ROCKSDB_CFIDX_ERASURE_META (4UL)
67 0 : #define FD_ROCKSDB_CFIDX_ORPHANS (5UL) /* Usually empty */
68 0 : #define FD_ROCKSDB_CFIDX_BANK_HASHES (6UL)
69 0 : #define FD_ROCKSDB_CFIDX_ROOT (7UL)
70 0 : #define FD_ROCKSDB_CFIDX_INDEX (8UL)
71 0 : #define FD_ROCKSDB_CFIDX_DATA_SHRED (9UL)
72 0 : #define FD_ROCKSDB_CFIDX_CODE_SHRED (10UL)
73 0 : #define FD_ROCKSDB_CFIDX_TRANSACTION_STATUS (11UL)
74 0 : #define FD_ROCKSDB_CFIDX_ADDRESS_SIGNATURES (12UL)
75 0 : #define FD_ROCKSDB_CFIDX_TRANSACTION_MEMOS (13UL)
76 0 : #define FD_ROCKSDB_CFIDX_TRANSACTION_STATUS_INDEX (14UL)
77 0 : #define FD_ROCKSDB_CFIDX_REWARDS (15UL)
78 0 : #define FD_ROCKSDB_CFIDX_BLOCKTIME (16UL)
79 0 : #define FD_ROCKSDB_CFIDX_PERF_SAMPLES (17UL)
80 0 : #define FD_ROCKSDB_CFIDX_BLOCK_HEIGHT (18UL)
81 0 : #define FD_ROCKSDB_CFIDX_OPTIMISTIC_SLOTS (19UL)
82 0 : #define FD_ROCKSDB_CFIDX_MERKLE_ROOT_META (20UL) /* Usually empty */
83 :
84 : /* Solana rocksdb client */
85 : struct fd_rocksdb {
86 : rocksdb_t * db;
87 : const char * db_name;
88 : const char * cfgs [ FD_ROCKSDB_CF_CNT ];
89 : rocksdb_column_family_handle_t* cf_handles[ FD_ROCKSDB_CF_CNT ];
90 : rocksdb_options_t * opts;
91 : rocksdb_readoptions_t * ro;
92 : rocksdb_writeoptions_t * wo;
93 : };
94 : typedef struct fd_rocksdb fd_rocksdb_t;
95 : #define FD_ROCKSDB_FOOTPRINT sizeof(fd_rocksdb_t)
96 : #define FD_ROCKSDB_ALIGN (8UL)
97 :
98 : /* root column iterator */
99 : struct fd_rocksdb_root_iter {
100 : fd_rocksdb_t * db;
101 : rocksdb_iterator_t* iter;
102 : };
103 : typedef struct fd_rocksdb_root_iter fd_rocksdb_root_iter_t;
104 : #define FD_ROCKSDB_ROOT_ITER_FOOTPRINT sizeof(fd_rocksdb_root_iter_t)
105 : #define FD_ROCKSDB_ROOT_ITER_ALIGN (8UL)
106 :
107 : FD_PROTOTYPES_BEGIN
108 :
109 : void *
110 : fd_rocksdb_root_iter_new( void * shiter );
111 :
112 : fd_rocksdb_root_iter_t *
113 : fd_rocksdb_root_iter_join( void * iter );
114 :
115 : void *
116 : fd_rocksdb_root_iter_leave( fd_rocksdb_root_iter_t * iter );
117 :
118 : /* fd_rocksdb_root_iter_seek
119 :
120 : 0 = success
121 : -1 = seek for supplied slot failed
122 : -2 = seek succeeded but slot did not match what we seeked for
123 : -3 = seek succeeded but points at an empty slot */
124 :
125 : int
126 : fd_rocksdb_root_iter_seek( fd_rocksdb_root_iter_t * iter,
127 : fd_rocksdb_t * db,
128 : ulong slot,
129 : fd_slot_meta_t * m,
130 : fd_valloc_t valloc );
131 :
132 : /* fd_rocksdb_root_iter_next
133 :
134 : 0 = success
135 : -1 = not properly initialized with a seek
136 : -2 = invalid starting iterator
137 : -3 = next returned an invalid iterator state
138 : -4 = seek succeeded but points at an empty slot */
139 :
140 : int
141 : fd_rocksdb_root_iter_next( fd_rocksdb_root_iter_t * iter,
142 : fd_slot_meta_t * m,
143 : fd_valloc_t valloc );
144 :
145 : int
146 : fd_rocksdb_root_iter_slot( fd_rocksdb_root_iter_t * self,
147 : ulong * slot );
148 :
149 : void
150 : fd_rocksdb_root_iter_destroy( fd_rocksdb_root_iter_t * iter );
151 :
152 : /* fd_rocksdb_init: Returns a pointer to a description of the error on failure
153 :
154 : The provided db_name needs to point at the actual rocksdb directory
155 : as apposed to the directory above (like the solana ledger-tool) */
156 :
157 : char *
158 : fd_rocksdb_init( fd_rocksdb_t * db,
159 : char const * db_name );
160 :
161 : /* fd_rocksdb_new: Creates a new rocksdb
162 :
163 : The provided db_name has to the be the full path where the directory
164 : will be created. The fd_rocksdb_t object will be initialized */
165 :
166 : void
167 : fd_rocksdb_new( fd_rocksdb_t * db,
168 : char const * db_name );
169 :
170 : /* fd_rocksdb_destroy
171 :
172 : Frees up the internal data structures */
173 :
174 : void
175 : fd_rocksdb_destroy( fd_rocksdb_t * db );
176 :
177 : /* fd_rocksdb_last_slot: Returns the last slot in the db
178 :
179 : This uses the root column to discover the slot of the last root in
180 : the db. If there is an error, this sets *err to a constant string
181 : describing the error. There is no need to free that string. */
182 :
183 : ulong
184 : fd_rocksdb_last_slot( fd_rocksdb_t * db,
185 : char ** err );
186 :
187 : /* fd_rocksdb_first_slot: Returns the first slot in the db
188 :
189 : This uses the root column to discover the slot of the first root in
190 : the db. If there is an error, this sets *err to a constant string
191 : describing the error. There is no need to free that string. */
192 :
193 : ulong
194 : fd_rocksdb_first_slot( fd_rocksdb_t * db,
195 : char ** err );
196 :
197 : ulong
198 : fd_rocksdb_find_last_slot( fd_rocksdb_t * db,
199 : char ** err );
200 :
201 : /* fd_rocksdb_get_meta
202 :
203 : Retrieves the meta structure associated with the supplied slot. If
204 : there is an error, *err is set to a string describing the error.
205 : It is expected that you should free() the error once done with it
206 :
207 : returns a 0 if there is no obvious error */
208 : int
209 : fd_rocksdb_get_meta( fd_rocksdb_t * db,
210 : ulong slot,
211 : fd_slot_meta_t * m,
212 : fd_valloc_t valloc );
213 :
214 : /* fd_rocksdb_get_txn_status_raw queries transaction status metadata.
215 : slot is the slot number of the block that contains the txn. sig
216 : points to the first signature of the txn. Returns data==NULL if
217 : record not found. On success, creates a malloc-backed buffer to hold
218 : return value, copies raw serialized status into buffer, sets *psz to
219 : the byte size of the status and returns pointer to buffer. Caller
220 : must free() non-NULL returned region. On failure, returns NULL and
221 : content of *psz is undefined. Value is Protobuf-encoded
222 : TransactionStatusMeta. Use fd_solblock nanopb API to deserialize
223 : value. */
224 :
225 : void *
226 : fd_rocksdb_get_txn_status_raw( fd_rocksdb_t * self,
227 : ulong slot,
228 : void const * sig,
229 : ulong * psz );
230 :
231 : /* fd_rocksdb_copy_over_slot_indexed_range copies over all entries for a
232 : given column family index into another rocksdb assuming that the key
233 : is prefixed with the slot number. This includes column families where
234 : the key is just the slot number but also ones where the key starts with
235 : the slot number. */
236 :
237 : int
238 : fd_rocksdb_copy_over_slot_indexed_range( fd_rocksdb_t * src,
239 : fd_rocksdb_t * dst,
240 : ulong cf_idx,
241 : ulong start_slot,
242 : ulong end_slot );
243 :
244 : /* fd_rocksdb_copy_over_txn_status constructs a key to query a transaction
245 : status and copies over the entry into another rocksdb. The index is used
246 : to specify which transaction. */
247 :
248 : void
249 : fd_rocksdb_copy_over_txn_status( fd_rocksdb_t * src,
250 : fd_rocksdb_t * dst,
251 : ulong slot,
252 : void const * sig );
253 :
254 : /* fd_rocksdb_insert_entry inserts a key, value pair into a given rocksdb */
255 :
256 : int
257 : fd_rocksdb_insert_entry( fd_rocksdb_t * db,
258 : ulong cf_idx,
259 : const char * key,
260 : ulong key_len,
261 : const char * value,
262 : ulong value_len );
263 :
264 : /* Import from rocksdb into blockstore */
265 :
266 : int
267 : fd_rocksdb_import_block_blockstore( fd_rocksdb_t * db,
268 : fd_slot_meta_t * m,
269 : fd_blockstore_t * blockstore,
270 : const uchar * hash_override,
271 : fd_valloc_t valloc );
272 :
273 : int
274 : fd_rocksdb_import_block_shredcap( fd_rocksdb_t * db,
275 : fd_slot_meta_t * metadata,
276 : fd_io_buffered_ostream_t * ostream,
277 : fd_io_buffered_ostream_t * bank_hash_ostream,
278 : fd_valloc_t valloc );
279 :
280 : /* fd_blockstore_block_query queries blockstore for block at slot.
281 : Returns a pointer to the block or NULL if not in blockstore. The
282 : returned pointer lifetime is until the block is removed. Check
283 : return value for error info.
284 :
285 : In theory the caller does not need to wrap this function in a
286 : start/end read. What is being read lives in the block_info object,
287 : and this function does a valid concurrent read for the block_gaddr.
288 : The fd_block_t object itself has no such guarantees, and needs a
289 : read/write lock to modify. */
290 : void
291 : fd_blockstore_block_allocs_remove( fd_blockstore_t * blockstore, ulong slot );
292 :
293 : static inline fd_block_t *
294 0 : fd_blockstore_block_query(fd_blockstore_t *blockstore, ulong slot){
295 0 : int err = FD_MAP_ERR_AGAIN;
296 0 : ulong query_block_gaddr = 0;
297 0 : while( err == FD_MAP_ERR_AGAIN ){
298 0 : fd_block_map_query_t quer[1] = { 0 };
299 0 : err = fd_block_map_query_try( blockstore->block_map, &slot, NULL, quer, 0 );
300 0 : fd_block_info_t * query = fd_block_map_query_ele( quer );
301 0 : if ( err == FD_MAP_ERR_KEY ) return NULL;
302 0 : if ( FD_UNLIKELY( err == FD_MAP_ERR_AGAIN ) ) continue;
303 : /* later change this to all shreds received */
304 0 : if( FD_UNLIKELY( query->block_gaddr == 0 ) ) return NULL;
305 0 : query_block_gaddr = query->block_gaddr;
306 0 : err = fd_block_map_query_test( quer );
307 0 : }
308 0 : return fd_wksp_laddr_fast( fd_blockstore_wksp( blockstore ), query_block_gaddr );
309 0 : }
310 :
311 : FD_PROTOTYPES_END
312 :
313 : #endif
314 :
315 : #endif // HEADER_fd_src_flamenco_runtime_fd_rocksdb_h
|
