Line data Source code
1 : #include "fd_bpf_program_util.h"
2 : #include "fd_bpf_loader_program.h"
3 : #include "../fd_acc_mgr.h"
4 : #include "../context/fd_exec_slot_ctx.h"
5 : #include "../../vm/syscall/fd_vm_syscall.h"
6 :
7 : #include <assert.h>
8 :
9 : fd_sbpf_validated_program_t *
10 2334 : fd_sbpf_validated_program_new( void * mem, fd_sbpf_elf_info_t const * elf_info ) {
11 2334 : fd_sbpf_validated_program_t * validated_prog = (fd_sbpf_validated_program_t *)mem;
12 :
13 2334 : ulong l = FD_LAYOUT_INIT;
14 :
15 : /* calldests backing memory */
16 2334 : l = FD_LAYOUT_APPEND( l, alignof(fd_sbpf_validated_program_t), sizeof(fd_sbpf_validated_program_t) );
17 2334 : validated_prog->calldests_shmem = (uchar *)mem + l;
18 :
19 : /* rodata backing memory */
20 2334 : l = FD_LAYOUT_APPEND( l, fd_sbpf_calldests_align(), fd_sbpf_calldests_footprint(elf_info->rodata_sz/8UL) );
21 2334 : validated_prog->rodata = (uchar *)mem + l;
22 :
23 2334 : return (fd_sbpf_validated_program_t *)mem;
24 2334 : }
25 :
26 : ulong
27 0 : fd_sbpf_validated_program_align( void ) {
28 0 : return alignof(fd_sbpf_validated_program_t);
29 0 : }
30 :
31 : ulong
32 2334 : fd_sbpf_validated_program_footprint( fd_sbpf_elf_info_t const * elf_info ) {
33 2334 : ulong l = FD_LAYOUT_INIT;
34 2334 : l = FD_LAYOUT_APPEND( l, alignof(fd_sbpf_validated_program_t), sizeof(fd_sbpf_validated_program_t) );
35 2334 : l = FD_LAYOUT_APPEND( l, fd_sbpf_calldests_align(), fd_sbpf_calldests_footprint(elf_info->rodata_sz/8UL) );
36 2334 : l = FD_LAYOUT_APPEND( l, 8UL, elf_info->rodata_footprint );
37 2334 : l = FD_LAYOUT_FINI( l, 128UL );
38 2334 : return l;
39 2334 : }
40 :
41 : static inline fd_funk_rec_key_t
42 189048 : fd_acc_mgr_cache_key( fd_pubkey_t const * pubkey ) {
43 189048 : fd_funk_rec_key_t id;
44 189048 : memcpy( id.uc, pubkey, sizeof(fd_pubkey_t) );
45 189048 : memset( id.uc + sizeof(fd_pubkey_t), 0, sizeof(fd_funk_rec_key_t) - sizeof(fd_pubkey_t) );
46 :
47 189048 : id.c[ FD_FUNK_REC_KEY_FOOTPRINT - 1 ] = FD_FUNK_KEY_TYPE_ELF_CACHE;
48 :
49 189048 : return id;
50 189048 : }
51 :
52 : int
53 : fd_bpf_get_executable_program_content_for_upgradeable_loader( fd_exec_slot_ctx_t * slot_ctx,
54 : fd_borrowed_account_t * program_acc,
55 : uchar const ** program_data,
56 88260 : ulong * program_data_len ) {
57 88260 : FD_BORROWED_ACCOUNT_DECL( programdata_acc );
58 :
59 88260 : fd_bincode_decode_ctx_t ctx = {
60 88260 : .data = program_acc->const_data,
61 88260 : .dataend = program_acc->const_data + program_acc->const_meta->dlen,
62 88260 : .valloc = fd_scratch_virtual(),
63 88260 : };
64 :
65 88260 : fd_bpf_upgradeable_loader_state_t program_account_state = {0};
66 88260 : if( FD_UNLIKELY( fd_bpf_upgradeable_loader_state_decode( &program_account_state, &ctx ) ) ) {
67 54414 : return -1;
68 54414 : }
69 :
70 33846 : if( !fd_bpf_upgradeable_loader_state_is_program( &program_account_state ) ) {
71 11427 : return -1;
72 11427 : }
73 :
74 22419 : fd_pubkey_t * programdata_address = &program_account_state.inner.program.programdata_address;
75 :
76 22419 : if( fd_acc_mgr_view( slot_ctx->acc_mgr, slot_ctx->funk_txn, programdata_address, programdata_acc ) != FD_ACC_MGR_SUCCESS ) {
77 2463 : return -1;
78 2463 : }
79 :
80 19956 : fd_bincode_decode_ctx_t ctx_programdata = {
81 19956 : .data = programdata_acc->const_data,
82 19956 : .dataend = programdata_acc->const_data + programdata_acc->const_meta->dlen,
83 19956 : .valloc = fd_scratch_virtual(),
84 19956 : };
85 :
86 19956 : fd_bpf_upgradeable_loader_state_t program_data_account_state = {0};
87 19956 : if( FD_UNLIKELY( fd_bpf_upgradeable_loader_state_decode( &program_data_account_state, &ctx_programdata ) ) ) {
88 13482 : return -1;
89 13482 : }
90 :
91 6474 : *program_data = programdata_acc->const_data + PROGRAMDATA_METADATA_SIZE;
92 6474 : *program_data_len = programdata_acc->const_meta->dlen - PROGRAMDATA_METADATA_SIZE;
93 6474 : return 0;
94 19956 : }
95 :
96 : int
97 : fd_bpf_get_executable_program_content_for_v1_v2_loaders( fd_borrowed_account_t * program_acc,
98 : uchar const ** program_data,
99 98742 : ulong * program_data_len ) {
100 98742 : *program_data = program_acc->const_data;
101 98742 : *program_data_len = program_acc->const_meta->dlen;
102 98742 : return 0;
103 98742 : }
104 :
105 : int
106 : fd_bpf_create_bpf_program_cache_entry( fd_exec_slot_ctx_t * slot_ctx,
107 187002 : fd_borrowed_account_t * program_acc ) {
108 187002 : FD_SCRATCH_SCOPE_BEGIN {
109 :
110 187002 : fd_pubkey_t * program_pubkey = program_acc->pubkey;
111 :
112 187002 : fd_funk_t * funk = slot_ctx->acc_mgr->funk;
113 187002 : fd_funk_txn_t * funk_txn = slot_ctx->funk_txn;
114 187002 : fd_funk_rec_key_t id = fd_acc_mgr_cache_key( program_pubkey );
115 :
116 187002 : uchar const * program_data = NULL;
117 187002 : ulong program_data_len = 0UL;
118 :
119 : /* For v3 loaders, deserialize the program account and lookup the
120 : programdata account. Deserialize the programdata account. */
121 :
122 187002 : int res;
123 187002 : if( !memcmp( program_acc->const_meta->info.owner, fd_solana_bpf_loader_upgradeable_program_id.key, sizeof(fd_pubkey_t) ) ) {
124 88260 : res = fd_bpf_get_executable_program_content_for_upgradeable_loader( slot_ctx, program_acc, &program_data, &program_data_len );
125 98742 : } else {
126 98742 : res = fd_bpf_get_executable_program_content_for_v1_v2_loaders( program_acc, &program_data, &program_data_len );
127 98742 : }
128 :
129 187002 : if( res ) {
130 81786 : return -1;
131 81786 : }
132 :
133 105216 : fd_sbpf_elf_info_t elf_info = {0};
134 105216 : if( fd_sbpf_elf_peek( &elf_info, program_data, program_data_len, /* deploy checks */ 0 ) == NULL ) {
135 102882 : FD_LOG_DEBUG(( "fd_sbpf_elf_peek() failed: %s", fd_sbpf_strerror() ));
136 102882 : return FD_EXECUTOR_INSTR_ERR_INVALID_ACC_DATA;
137 102882 : }
138 :
139 2334 : int funk_err = FD_FUNK_SUCCESS;
140 2334 : fd_funk_rec_t const * existing_rec = fd_funk_rec_query_global( funk, funk_txn, &id, NULL );
141 2334 : fd_funk_rec_t * rec = fd_funk_rec_write_prepare( funk, funk_txn, &id, fd_sbpf_validated_program_footprint( &elf_info ), 1, existing_rec, &funk_err );
142 2334 : if( rec == NULL || funk_err != FD_FUNK_SUCCESS ) {
143 0 : return -1;
144 0 : }
145 :
146 2334 : void * val = fd_funk_val( rec, fd_funk_wksp( funk ) );
147 2334 : fd_sbpf_validated_program_t * validated_prog = fd_sbpf_validated_program_new( val, &elf_info );
148 :
149 2334 : ulong prog_align = fd_sbpf_program_align();
150 2334 : ulong prog_footprint = fd_sbpf_program_footprint( &elf_info );
151 2334 : fd_sbpf_program_t * prog = fd_sbpf_program_new( fd_scratch_alloc( prog_align, prog_footprint ), &elf_info, validated_prog->rodata );
152 2334 : if( FD_UNLIKELY( !prog ) ) {
153 0 : return -1;
154 0 : }
155 :
156 : /* Allocate syscalls */
157 :
158 2334 : fd_sbpf_syscalls_t * syscalls = fd_sbpf_syscalls_new( fd_scratch_alloc( fd_sbpf_syscalls_align(), fd_sbpf_syscalls_footprint() ) );
159 2334 : if( FD_UNLIKELY( !syscalls ) ) {
160 0 : FD_LOG_ERR(( "Call to fd_sbpf_syscalls_new() failed" ));
161 0 : }
162 :
163 2334 : fd_vm_syscall_register_slot( syscalls, slot_ctx, 0 );
164 :
165 : /* Load program. */
166 :
167 2334 : if( FD_UNLIKELY( 0!=fd_sbpf_program_load( prog, program_data, program_data_len, syscalls, false ) ) ) {
168 : /* Remove pending funk record */
169 339 : FD_LOG_DEBUG(( "fd_sbpf_program_load() failed: %s", fd_sbpf_strerror() ));
170 339 : fd_funk_rec_remove( funk, rec, 0 );
171 339 : return -1;
172 339 : }
173 :
174 : /* Validate the program. */
175 :
176 1995 : fd_vm_t _vm[ 1UL ];
177 1995 : fd_vm_t * vm = fd_vm_join( fd_vm_new( _vm ) );
178 1995 : if( FD_UNLIKELY( !vm ) ) {
179 0 : FD_LOG_ERR(( "fd_vm_new() or fd_vm_join() failed" ));
180 0 : }
181 1995 : fd_exec_instr_ctx_t dummy_instr_ctx = {0};
182 1995 : dummy_instr_ctx.slot_ctx = slot_ctx;
183 1995 : vm = fd_vm_init( vm,
184 1995 : &dummy_instr_ctx,
185 1995 : 0UL,
186 1995 : 0UL,
187 1995 : prog->rodata,
188 1995 : prog->rodata_sz,
189 1995 : prog->text,
190 1995 : prog->text_cnt,
191 1995 : prog->text_off,
192 1995 : prog->text_sz,
193 1995 : prog->entry_pc,
194 1995 : prog->calldests,
195 1995 : NULL,
196 1995 : NULL,
197 1995 : NULL,
198 1995 : NULL,
199 1995 : 0U,
200 1995 : NULL,
201 1995 : 0,
202 1995 : FD_FEATURE_ACTIVE( slot_ctx, bpf_account_data_direct_mapping ) );
203 :
204 1995 : if( FD_UNLIKELY( !vm ) ) {
205 0 : FD_LOG_ERR(( "fd_vm_init() failed" ));
206 0 : }
207 :
208 1995 : res = fd_vm_validate( vm );
209 1995 : if( FD_UNLIKELY( res ) ) {
210 : /* Remove pending funk record */
211 228 : FD_LOG_DEBUG(( "fd_vm_validate() failed" ));
212 228 : fd_funk_rec_remove( funk, rec, 0 );
213 228 : return -1;
214 228 : }
215 :
216 1767 : fd_memcpy( validated_prog->calldests_shmem, prog->calldests_shmem, fd_sbpf_calldests_footprint(prog->rodata_sz/8UL) );
217 1767 : validated_prog->calldests = fd_sbpf_calldests_join( validated_prog->calldests_shmem );
218 :
219 1767 : validated_prog->entry_pc = prog->entry_pc;
220 1767 : validated_prog->last_updated_slot = slot_ctx->slot_bank.slot;
221 1767 : validated_prog->text_off = prog->text_off;
222 1767 : validated_prog->text_cnt = prog->text_cnt;
223 1767 : validated_prog->text_sz = prog->text_sz;
224 1767 : validated_prog->rodata_sz = prog->rodata_sz;
225 :
226 1767 : return 0;
227 187002 : } FD_SCRATCH_SCOPE_END;
228 187002 : }
229 :
230 : static void FD_FN_UNUSED
231 : fd_bpf_scan_task( void * tpool,
232 : ulong t0 FD_PARAM_UNUSED, ulong t1 FD_PARAM_UNUSED,
233 : void * args FD_PARAM_UNUSED,
234 : void * reduce FD_PARAM_UNUSED, ulong stride FD_PARAM_UNUSED,
235 : ulong l0 FD_PARAM_UNUSED, ulong l1 FD_PARAM_UNUSED,
236 : ulong m0, ulong m1 FD_PARAM_UNUSED,
237 0 : ulong n0 FD_PARAM_UNUSED, ulong n1 FD_PARAM_UNUSED ) {
238 0 : fd_funk_rec_t const * recs = ((fd_funk_rec_t const **)tpool)[m0];
239 0 : fd_exec_slot_ctx_t * slot_ctx = (fd_exec_slot_ctx_t *)args;
240 0 : uchar * is_bpf_program = (uchar *)reduce + m0;
241 :
242 0 : if( !fd_funk_key_is_acc( recs->pair.key ) ) {
243 0 : *is_bpf_program = 0;
244 0 : return;
245 0 : }
246 :
247 0 : fd_pubkey_t const * pubkey = fd_type_pun_const( recs->pair.key[0].uc );
248 :
249 0 : FD_BORROWED_ACCOUNT_DECL( exec_rec );
250 0 : if( fd_acc_mgr_view( slot_ctx->acc_mgr, slot_ctx->funk_txn, pubkey, exec_rec ) != FD_ACC_MGR_SUCCESS ) {
251 0 : return;
252 0 : }
253 :
254 0 : if( memcmp( exec_rec->const_meta->info.owner, fd_solana_bpf_loader_deprecated_program_id.key, sizeof(fd_pubkey_t) ) &&
255 0 : memcmp( exec_rec->const_meta->info.owner, fd_solana_bpf_loader_program_id.key, sizeof(fd_pubkey_t) ) &&
256 0 : memcmp( exec_rec->const_meta->info.owner, fd_solana_bpf_loader_upgradeable_program_id.key, sizeof(fd_pubkey_t) ) &&
257 0 : memcmp( exec_rec->const_meta->info.owner, fd_solana_bpf_loader_v4_program_id.key, sizeof(fd_pubkey_t) ) ) {
258 0 : *is_bpf_program = 0;
259 0 : } else {
260 0 : *is_bpf_program = 1;
261 0 : }
262 0 : }
263 :
264 : int
265 : fd_bpf_scan_and_create_bpf_program_cache_entry_tpool( fd_exec_slot_ctx_t * slot_ctx,
266 : fd_funk_txn_t * funk_txn,
267 0 : fd_tpool_t * tpool ) {
268 0 : long elapsed_ns = -fd_log_wallclock();
269 0 : fd_funk_t * funk = slot_ctx->acc_mgr->funk;
270 0 : ulong cached_cnt = 0;
271 :
272 : /* Use random-ish xid to avoid concurrency issues */
273 0 : fd_funk_txn_xid_t cache_xid = fd_funk_generate_xid();
274 :
275 0 : fd_funk_txn_t * cache_txn = fd_funk_txn_prepare( funk, slot_ctx->funk_txn, &cache_xid, 1 );
276 0 : if( !cache_txn ) {
277 0 : FD_LOG_ERR(( "fd_funk_txn_prepare() failed" ));
278 0 : return -1;
279 0 : }
280 :
281 0 : fd_funk_txn_t * parent_txn = slot_ctx->funk_txn;
282 0 : slot_ctx->funk_txn = cache_txn;
283 :
284 0 : fd_funk_rec_t const * rec = fd_funk_txn_first_rec( funk, funk_txn );
285 0 : while( rec!=NULL ) {
286 0 : FD_SCRATCH_SCOPE_BEGIN {
287 0 : fd_funk_rec_t const * * recs = fd_scratch_alloc( alignof(fd_funk_rec_t const *), 65536UL * sizeof(fd_funk_rec_t const *) );
288 0 : uchar * is_bpf_program = fd_scratch_alloc( 8UL, 65536UL * sizeof(uchar) );
289 :
290 : /* Make a list of rec ptrs to process */
291 0 : ulong rec_cnt = 0;
292 0 : for( ; NULL != rec; rec = fd_funk_txn_next_rec( funk, rec ) ) {
293 0 : recs[ rec_cnt ] = rec;
294 :
295 0 : if( rec_cnt==65536UL ) {
296 0 : break;
297 0 : }
298 :
299 0 : rec_cnt++;
300 0 : }
301 :
302 0 : fd_tpool_exec_all_block( tpool, 0, fd_tpool_worker_cnt( tpool ), fd_bpf_scan_task, recs, slot_ctx, is_bpf_program, 1, 0, rec_cnt );
303 :
304 0 : for( ulong i = 0; i<rec_cnt; i++ ) {
305 0 : if( !is_bpf_program[ i ] ) {
306 0 : continue;
307 0 : }
308 :
309 0 : fd_pubkey_t const * pubkey = fd_type_pun_const( recs[i]->pair.key[0].uc );
310 0 : int res = fd_bpf_check_and_create_bpf_program_cache_entry( slot_ctx, funk_txn, pubkey );
311 0 : if( res==0 ) {
312 0 : cached_cnt++;
313 0 : }
314 0 : }
315 :
316 0 : } FD_SCRATCH_SCOPE_END;
317 0 : }
318 :
319 0 : if( fd_funk_txn_publish_into_parent( funk, cache_txn, 1 ) != FD_FUNK_SUCCESS ) {
320 0 : FD_LOG_ERR(( "fd_funk_txn_publish_into_parent() failed" ));
321 0 : return -1;
322 0 : }
323 :
324 0 : slot_ctx->funk_txn = parent_txn;
325 :
326 0 : elapsed_ns += fd_log_wallclock();
327 :
328 0 : FD_LOG_NOTICE(( "loaded program cache - entries: %lu, elapsed_seconds: %ld", cached_cnt, elapsed_ns/(long)1e9 ));
329 :
330 0 : return 0;
331 0 : }
332 :
333 : int
334 : fd_bpf_scan_and_create_bpf_program_cache_entry( fd_exec_slot_ctx_t * slot_ctx,
335 108786 : fd_funk_txn_t * funk_txn ) {
336 108786 : fd_funk_t * funk = slot_ctx->acc_mgr->funk;
337 108786 : ulong cnt = 0;
338 :
339 : /* Use random-ish xid to avoid concurrency issues */
340 108786 : fd_funk_txn_xid_t cache_xid = fd_funk_generate_xid();
341 :
342 108786 : fd_funk_txn_t * cache_txn = fd_funk_txn_prepare( funk, slot_ctx->funk_txn, &cache_xid, 1 );
343 108786 : if( !cache_txn ) {
344 0 : FD_LOG_ERR(( "fd_funk_txn_prepare() failed" ));
345 0 : return -1;
346 0 : }
347 :
348 108786 : fd_funk_txn_t * parent_txn = slot_ctx->funk_txn;
349 108786 : slot_ctx->funk_txn = cache_txn;
350 :
351 108786 : for (fd_funk_rec_t const *rec = fd_funk_txn_first_rec( funk, funk_txn );
352 1347885 : NULL != rec;
353 1239099 : rec = fd_funk_txn_next_rec( funk, rec )) {
354 1239099 : if( !fd_funk_key_is_acc( rec->pair.key ) ) {
355 0 : continue;
356 0 : }
357 :
358 1239099 : fd_pubkey_t const * program_pubkey = fd_type_pun_const( rec->pair.key[0].uc );
359 :
360 1239099 : int res = fd_bpf_check_and_create_bpf_program_cache_entry( slot_ctx,
361 1239099 : funk_txn,
362 1239099 : program_pubkey );
363 :
364 1239099 : if( res==0 ) {
365 1767 : cnt++;
366 1767 : }
367 1239099 : }
368 :
369 108786 : FD_LOG_DEBUG(( "loaded program cache: %lu", cnt));
370 :
371 108786 : if( fd_funk_txn_publish_into_parent( funk, cache_txn, 1 ) != FD_FUNK_SUCCESS ) {
372 0 : FD_LOG_ERR(( "fd_funk_txn_publish_into_parent() failed" ));
373 0 : return -1;
374 0 : }
375 :
376 108786 : slot_ctx->funk_txn = parent_txn;
377 108786 : return 0;
378 108786 : }
379 :
380 : int
381 : fd_bpf_check_and_create_bpf_program_cache_entry( fd_exec_slot_ctx_t * slot_ctx,
382 : fd_funk_txn_t * funk_txn,
383 1239099 : fd_pubkey_t const * pubkey ) {
384 1239099 : FD_BORROWED_ACCOUNT_DECL( exec_rec );
385 1239099 : if( fd_acc_mgr_view( slot_ctx->acc_mgr, funk_txn, pubkey, exec_rec ) != FD_ACC_MGR_SUCCESS ) {
386 18543 : return -1;
387 18543 : }
388 :
389 1220556 : if( memcmp( exec_rec->const_meta->info.owner, fd_solana_bpf_loader_deprecated_program_id.key, sizeof(fd_pubkey_t) ) &&
390 1220556 : memcmp( exec_rec->const_meta->info.owner, fd_solana_bpf_loader_program_id.key, sizeof(fd_pubkey_t) ) &&
391 1220556 : memcmp( exec_rec->const_meta->info.owner, fd_solana_bpf_loader_upgradeable_program_id.key, sizeof(fd_pubkey_t) ) &&
392 1220556 : memcmp( exec_rec->const_meta->info.owner, fd_solana_bpf_loader_v4_program_id.key, sizeof(fd_pubkey_t) ) ) {
393 1033554 : return -1;
394 1033554 : }
395 :
396 187002 : if( fd_bpf_create_bpf_program_cache_entry( slot_ctx, exec_rec ) != 0 ) {
397 185235 : return -1;
398 185235 : }
399 :
400 1767 : return 0;
401 187002 : }
402 :
403 : int
404 : fd_bpf_load_cache_entry( fd_exec_slot_ctx_t const * slot_ctx,
405 : fd_pubkey_t const * program_pubkey,
406 2046 : fd_sbpf_validated_program_t ** valid_prog ) {
407 2046 : fd_funk_t * funk = slot_ctx->acc_mgr->funk;
408 2046 : fd_funk_txn_t * funk_txn = slot_ctx->funk_txn;
409 2046 : fd_funk_rec_key_t id = fd_acc_mgr_cache_key( program_pubkey );
410 :
411 2046 : fd_funk_rec_t const * rec = fd_funk_rec_query_global(funk, funk_txn, &id, NULL);
412 :
413 2046 : if( FD_UNLIKELY( !rec || !!( rec->flags & FD_FUNK_REC_FLAG_ERASE ) ) ) {
414 1029 : return -1;
415 1029 : }
416 :
417 1017 : void const * data = fd_funk_val_const( rec, fd_funk_wksp(funk) );
418 :
419 : /* TODO: magic check */
420 :
421 1017 : *valid_prog = (fd_sbpf_validated_program_t *)data;
422 :
423 1017 : return 0;
424 2046 : }
|
