Line data Source code
1 : #include "fd_progcache.h"
2 : #include "../vm/fd_vm.h" /* fd_vm_syscall_register_slot, fd_vm_validate */
3 : #include "../../util/alloc/fd_alloc.h"
4 :
5 : #include <stdlib.h>
6 :
7 : /* Can be overridden by test executables */
8 : __attribute__((weak)) int const fd_progcache_use_malloc = 0;
9 : static inline _Bool
10 5121 : use_malloc( void ) {
11 5121 : _Bool use_malloc = !!fd_progcache_use_malloc;
12 5121 : FD_COMPILER_FORGET( use_malloc ); /* prevent constant propagation */
13 5121 : return use_malloc;
14 5121 : }
15 :
16 : void *
17 : fd_progcache_val_alloc( fd_progcache_rec_t * rec,
18 : fd_progcache_join_t * join,
19 : ulong val_align,
20 5016 : ulong val_footprint ) {
21 5016 : if( rec->data_gaddr ) fd_progcache_val_free( rec, join );
22 5016 : ulong val_max = 0UL;
23 5016 : void * mem;
24 5016 : ulong gaddr;
25 5016 : if( FD_UNLIKELY( use_malloc() ) ) { /* test only */
26 0 : mem = aligned_alloc( val_align, val_footprint );
27 0 : if( FD_UNLIKELY( !mem ) ) return NULL;
28 0 : val_max = val_footprint;
29 0 : gaddr = (ulong)mem;
30 5016 : } else {
31 5016 : mem = fd_alloc_malloc_at_least( join->alloc, val_align, val_footprint, &val_max );
32 5016 : if( FD_UNLIKELY( !mem ) ) return NULL;
33 108 : FD_CHECK_CRIT( val_max<=UINT_MAX, "massive" ); /* unreachable */
34 108 : gaddr = fd_wksp_gaddr_fast( join->data_base, mem );
35 108 : }
36 108 : rec->data_gaddr = gaddr;
37 108 : rec->data_max = (uint)val_max;
38 108 : return mem;
39 5016 : }
40 :
41 : void
42 : fd_progcache_val_free1( fd_progcache_rec_t * rec,
43 : void * val,
44 105 : fd_alloc_t * alloc ) {
45 105 : if( FD_UNLIKELY( use_malloc() ) ) { /* test only */
46 0 : free( val );
47 105 : } else {
48 105 : fd_alloc_free( alloc, val );
49 105 : }
50 105 : rec->data_gaddr = 0UL;
51 105 : rec->data_max = 0U;
52 105 : rec->rodata_off = 0U;
53 105 : rec->rodata_sz = 0U;
54 105 : }
55 :
56 : void
57 : fd_progcache_val_free( fd_progcache_rec_t * rec,
58 99 : fd_progcache_join_t * join ) {
59 99 : if( !rec->data_gaddr ) return;
60 96 : void * mem = fd_wksp_laddr_fast( join->data_base, rec->data_gaddr );
61 :
62 : /* Illegal to call val_free on a spill-allocated buffer */
63 96 : FD_TEST( !( (ulong)mem >= (ulong)join->shmem->spill.spad &&
64 96 : (ulong)mem < (ulong)join->shmem->spill.spad+FD_PROGCACHE_SPAD_MAX ) );
65 :
66 96 : fd_progcache_val_free1( rec, mem, join->alloc );
67 96 : }
68 :
69 : FD_FN_PURE ulong
70 7578 : fd_progcache_val_footprint( fd_sbpf_elf_info_t const * elf_info ) {
71 7578 : int has_calldests = !fd_sbpf_enable_stricter_elf_headers_enabled( elf_info->sbpf_version );
72 7578 : ulong pc_max = fd_ulong_max( 1UL, elf_info->text_cnt );
73 :
74 : /* load_buf_sz is the exact buffer the loader needs (peek-computed):
75 : text_off+text_sz for strict, the rodata image for lenient-fast, or bin_sz
76 : for legacy lenient. */
77 7578 : ulong l = FD_LAYOUT_INIT;
78 7578 : if( has_calldests ) {
79 7578 : l = FD_LAYOUT_APPEND( l, fd_sbpf_calldests_align(), fd_sbpf_calldests_footprint( pc_max ) );
80 7578 : }
81 7578 : l = FD_LAYOUT_APPEND( l, 8UL, elf_info->load_buf_sz );
82 7578 : return FD_LAYOUT_FINI( l, fd_progcache_val_align() );
83 7578 : }
84 :
85 : /* Program loader wrapper */
86 :
87 : fd_progcache_rec_t *
88 : fd_progcache_rec_load( fd_progcache_rec_t * rec,
89 : fd_wksp_t * wksp,
90 : fd_sbpf_elf_info_t const * elf_info,
91 : fd_sbpf_loader_config_t const * config,
92 : ulong load_slot,
93 : fd_features_t const * features,
94 : void const * progdata,
95 : ulong progdata_sz,
96 : void * scratch,
97 2562 : ulong scratch_sz ) {
98 :
99 : /* Format object */
100 :
101 2562 : int has_calldests = !fd_sbpf_enable_stricter_elf_headers_enabled( elf_info->sbpf_version );
102 :
103 2562 : void * val = fd_wksp_laddr_fast( wksp, rec->data_gaddr );
104 2562 : void * calldests_mem = NULL;
105 2562 : void * rodata_mem;
106 2562 : if( has_calldests ) {
107 : /* Lenient (v0-v2): [ calldests | rodata ] laid out inside val. The rodata
108 : buffer is load_buf_sz (rodata image on the fast path, bin_sz on the
109 : legacy path); must match fd_progcache_val_footprint. */
110 2562 : FD_SCRATCH_ALLOC_INIT( l, val );
111 2562 : calldests_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_sbpf_calldests_align(), fd_sbpf_calldests_footprint( fd_ulong_max( 1UL, elf_info->text_cnt ) ) );
112 2562 : rodata_mem = FD_SCRATCH_ALLOC_APPEND( l, 8UL, elf_info->load_buf_sz );
113 2562 : FD_SCRATCH_ALLOC_FINI( l, fd_progcache_val_align() );
114 2562 : FD_TEST( _l-(ulong)val == fd_progcache_val_footprint( elf_info ) );
115 2562 : } else {
116 : /* Strict (v3+): no calldests, so rodata is just the start of val
117 : (val is fd_progcache_val_align()-aligned, which is >= 8). */
118 0 : rodata_mem = val;
119 0 : }
120 :
121 2562 : rec->calldests_off = has_calldests ? (uint)( (ulong)calldests_mem - (ulong)val ) : UINT_MAX;
122 2562 : rec->rodata_off = (uint)( (ulong)rodata_mem - (ulong)val );
123 2562 : rec->entry_pc = 0;
124 2562 : rec->rodata_sz = 0;
125 :
126 2562 : rec->text_cnt = elf_info->text_cnt;
127 2562 : rec->text_off = elf_info->text_off;
128 2562 : rec->text_sz = (uint)elf_info->text_sz;
129 2562 : rec->sbpf_version = (uchar)elf_info->sbpf_version;
130 :
131 : /* Set up sbpf_loader (redirect writes into progcache_rec object) */
132 :
133 2562 : fd_sbpf_program_t prog[1] = {{
134 2562 : .info = *elf_info,
135 2562 : .rodata = rodata_mem,
136 2562 : .text = (ulong *)((ulong)rodata_mem + elf_info->text_off), /* FIXME: WHAT IF MISALIGNED */
137 2562 : .entry_pc = ULONG_MAX
138 2562 : }};
139 2562 : if( has_calldests && elf_info->text_cnt>0UL ) {
140 2562 : prog->calldests_shmem = calldests_mem;
141 2562 : prog->calldests = fd_sbpf_calldests_join( fd_sbpf_calldests_new( calldests_mem, elf_info->text_cnt ) );
142 2562 : }
143 :
144 : /* Loader requires syscall table */
145 :
146 2562 : fd_sbpf_syscalls_t _syscalls[ FD_SBPF_SYSCALLS_SLOT_CNT ];
147 2562 : fd_sbpf_syscalls_t * syscalls = fd_sbpf_syscalls_join( fd_sbpf_syscalls_new( _syscalls ) );
148 2562 : int syscalls_err = fd_vm_syscall_register_slot( syscalls, load_slot, features, /* is_deploy */ 0 );
149 2562 : if( FD_UNLIKELY( syscalls_err!=FD_VM_SUCCESS ) ) FD_LOG_CRIT(( "fd_vm_syscall_register_slot failed" ));
150 :
151 : /* Run ELF loader.
152 :
153 : Scratch is needed only by the lenient (v0-v2) fallback path, which
154 : assembles the rodata segment via a scratch buffer. The lenient fast
155 : path and strict (v3+) loads write directly into the destination buffer;
156 : passing NULL both selects the loader's fast/no-scratch path and faults
157 : loudly if it ever starts relying on scratch. */
158 :
159 2562 : int use_scratch = fd_sbpf_loader_is_legacy_lenient( elf_info );
160 2562 : void * load_scratch = use_scratch ? scratch : NULL;
161 2562 : ulong load_scratch_sz = use_scratch ? scratch_sz : 0UL;
162 :
163 2562 : if( FD_UNLIKELY( 0!=fd_sbpf_program_load( prog, progdata, progdata_sz, syscalls, config, load_scratch, load_scratch_sz ) ) ) {
164 0 : return NULL;
165 0 : }
166 :
167 2562 : rec->entry_pc = (uint)prog->entry_pc;
168 2562 : rec->rodata_sz = (uint)prog->rodata_sz;
169 :
170 : /* Run bytecode validator */
171 :
172 2562 : fd_vm_t _vm[1];
173 2562 : fd_vm_t * vm = fd_vm_join( fd_vm_new( _vm ) );
174 2562 : if( FD_UNLIKELY( !vm ) ) FD_LOG_CRIT(( "fd_vm_new failed" ));
175 2562 : vm = fd_vm_init( vm,
176 2562 : NULL, /* OK since unused in `fd_vm_validate()` */
177 2562 : 0UL,
178 2562 : 0UL,
179 2562 : prog->rodata,
180 2562 : prog->rodata_sz,
181 2562 : prog->text,
182 2562 : prog->info.text_cnt,
183 2562 : prog->info.text_off,
184 2562 : prog->info.text_sz,
185 2562 : prog->entry_pc,
186 2562 : prog->calldests,
187 2562 : elf_info->sbpf_version,
188 2562 : syscalls,
189 2562 : NULL,
190 2562 : NULL,
191 2562 : NULL,
192 2562 : 0U,
193 2562 : NULL,
194 2562 : 0,
195 2562 : FD_FEATURE_ACTIVE( load_slot, features, account_data_direct_mapping ),
196 2562 : FD_FEATURE_ACTIVE( load_slot, features, syscall_parameter_address_restrictions ),
197 2562 : FD_FEATURE_ACTIVE( load_slot, features, virtual_address_space_adjustments ),
198 2562 : 0,
199 2562 : 0UL );
200 2562 : if( FD_UNLIKELY( !vm ) ) FD_LOG_CRIT(( "fd_vm_init failed" ));
201 :
202 2562 : if( FD_UNLIKELY( fd_vm_validate( vm )!=FD_VM_SUCCESS ) ) return NULL;
203 :
204 2559 : return rec;
205 2562 : }
206 :
207 : fd_progcache_rec_t *
208 3 : fd_progcache_rec_nx( fd_progcache_rec_t * rec ) {
209 3 : rec->data_gaddr = 0UL;
210 3 : rec->data_max = 0U;
211 3 : rec->entry_pc = 0;
212 3 : rec->text_cnt = 0;
213 3 : rec->text_off = 0;
214 3 : rec->text_sz = 0;
215 3 : rec->rodata_sz = 0;
216 : rec->calldests_off = UINT_MAX;
217 3 : rec->rodata_off = 0;
218 3 : rec->sbpf_version = 0;
219 3 : return rec;
220 3 : }
|
