Line data Source code
1 : #include "fd_sbpf_loader.h"
2 : #include "fd_sbpf_opcodes.h"
3 : #include "../../util/fd_util.h"
4 : #include "../../util/bits/fd_sat.h"
5 : #include "../murmur3/fd_murmur3.h"
6 :
7 : #include <assert.h>
8 : #include <stdio.h>
9 :
10 : /* Error handling *****************************************************/
11 :
12 : /* Thread local storage last error value */
13 :
14 : static FD_TL int ldr_errno = 0;
15 : static FD_TL int ldr_err_srcln = -1;
16 : #define FD_SBPF_ERRBUF_SZ (128UL)
17 : static FD_TL char fd_sbpf_errbuf[ FD_SBPF_ERRBUF_SZ ] = {0};
18 :
19 : /* fd_sbpf_loader_seterr remembers the error ID and line number of the
20 : current file at which the last error occurred. */
21 :
22 : __attribute__((cold,noinline)) int
23 : fd_sbpf_loader_seterr( int err,
24 107775 : int srcln ) {
25 107775 : ldr_errno = err;
26 107775 : ldr_err_srcln = srcln;
27 107775 : return err;
28 107775 : }
29 :
30 : /* Macros for returning an error from the current function while also
31 : remembering the error code. */
32 :
33 104127 : #define ERR( err ) return fd_sbpf_loader_seterr( (err), __LINE__ )
34 104070 : #define FAIL() ERR( FD_SBPF_ERR_INVALID_ELF )
35 16462296 : #define REQUIRE(x) do { if ( FD_UNLIKELY( !(x) ) ) FAIL(); } while (0)
36 :
37 : char const *
38 103842 : fd_sbpf_strerror( void ) {
39 103842 : if( FD_UNLIKELY( ldr_errno==0 ) )
40 0 : strcpy( fd_sbpf_errbuf, "ok" );
41 103842 : else
42 103842 : snprintf( fd_sbpf_errbuf, FD_SBPF_ERRBUF_SZ,
43 103842 : "code %d at %s(%d)", ldr_errno, __FILE__, ldr_err_srcln );
44 103842 : return fd_sbpf_errbuf;
45 103842 : }
46 :
47 : /* ELF loader, part 1 **************************************************
48 :
49 : Start with a static piece of scratch memory and do basic validation
50 : of the file content. Walk the section table once and remember
51 : sections of interest.
52 :
53 : ### Terminology
54 :
55 : This source follows common ELF naming practices.
56 :
57 : section: a named data region present in the ELF file
58 : segment: a contiguous memory region containing sections
59 : (not necessarily contiguous in the ELF file)
60 :
61 : physical address (paddr): Byte offset into ELF file (uchar * bin)
62 : virtual address (vaddr): VM memory address */
63 :
64 : /* Provide convenient access to file header and ELF content */
65 :
66 : __extension__ union fd_sbpf_elf {
67 : fd_elf64_ehdr ehdr;
68 : uchar bin[0];
69 : };
70 : typedef union fd_sbpf_elf fd_sbpf_elf_t;
71 :
72 : /* FD_SBPF_MM_{...}_ADDR are hardcoded virtual addresses of segments
73 : in the sBPF virtual machine.
74 :
75 : FIXME: These should be defined elsewhere */
76 :
77 2581044 : #define FD_SBPF_MM_PROGRAM_ADDR (0x100000000UL) /* readonly program data */
78 : #define FD_SBPF_MM_STACK_ADDR (0x200000000UL) /* stack (with gaps) */
79 :
80 : /* _fd_int_store_if_negative stores x to *p if *p is negative (branchless) */
81 :
82 : static inline int
83 : _fd_int_store_if_negative( int * p,
84 8043 : int x ) {
85 8043 : return (*p = fd_int_if( (*p)<0, x, *p ));
86 8043 : }
87 :
88 : /* fd_sbpf_check_ehdr verifies the ELF file header. */
89 :
90 : static int
91 : fd_sbpf_check_ehdr( fd_elf64_ehdr const * ehdr,
92 22281 : ulong elf_sz ) {
93 :
94 : /* Validate ELF magic */
95 22281 : REQUIRE( ( fd_uint_load_4( ehdr->e_ident )==0x464c457fU )
96 : /* Validate file type/target identification
97 : Solana/Agave performs header checks across two places:
98 : - Elf64::parse https://github.com/solana-labs/rbpf/blob/v0.8.0/src/elf_parser/mod.rs#L108
99 : - Executable::validate https://github.com/solana-labs/rbpf/blob/v0.8.0/src/elf.rs#L518
100 : These two sections are executed in close proximity, with no modifications to the header in between.
101 : We can therefore consolidate the checks in one place.
102 : */
103 22281 : & ( ehdr->e_ident[ FD_ELF_EI_CLASS ]==FD_ELF_CLASS_64 )
104 22281 : & ( ehdr->e_ident[ FD_ELF_EI_DATA ]==FD_ELF_DATA_LE )
105 22281 : & ( ehdr->e_ident[ FD_ELF_EI_VERSION ]==1 )
106 22281 : & ( ehdr->e_ident[ FD_ELF_EI_OSABI ]==FD_ELF_OSABI_NONE )
107 22281 : & ( ehdr->e_type ==FD_ELF_ET_DYN )
108 22281 : & ( ( ehdr->e_machine ==FD_ELF_EM_BPF )
109 22281 : | ( ehdr->e_machine ==FD_ELF_EM_SBPF ) )
110 22281 : & ( ehdr->e_version ==1 )
111 : /* Coherence checks */
112 22281 : & ( ehdr->e_ehsize ==sizeof(fd_elf64_ehdr) )
113 22281 : & ( ehdr->e_phentsize==sizeof(fd_elf64_phdr) )
114 22281 : & ( ehdr->e_shentsize==sizeof(fd_elf64_shdr) )
115 22281 : & ( ehdr->e_shstrndx < ehdr->e_shnum )
116 22281 : & ( ehdr->e_flags !=FD_ELF_EF_SBPF_V2 ) );
117 :
118 : /* Bounds check program header table */
119 :
120 4326 : ulong const phoff = ehdr->e_phoff;
121 4326 : ulong const phnum = ehdr->e_phnum;
122 4326 : REQUIRE( ( fd_ulong_is_aligned( phoff, 8UL ) )
123 4326 : & ( phoff<=elf_sz ) ); /* out of bounds */
124 :
125 4287 : REQUIRE( phnum<=(ULONG_MAX/sizeof(fd_elf64_phdr)) ); /* overflow */
126 4287 : ulong const phsz = phnum*sizeof(fd_elf64_phdr);
127 :
128 4287 : ulong const phoff_end = phoff+phsz;
129 4287 : REQUIRE( ( phoff_end>=phoff ) /* overflow */
130 4287 : & ( phoff_end<=elf_sz ) /* out of bounds */
131 4287 : & ( (phoff_end==0UL) /* overlaps file header */
132 4287 : | (phoff>=sizeof(fd_elf64_ehdr)) ) );
133 :
134 : /* Bounds check section header table */
135 :
136 4287 : ulong const shoff = ehdr->e_shoff;
137 4287 : ulong const shnum = ehdr->e_shnum;
138 4287 : REQUIRE( ( fd_ulong_is_aligned( shoff, 8UL ) )
139 4287 : & ( shoff>=sizeof(fd_elf64_ehdr) ) /* overlaps file header */
140 4287 : & ( shoff< elf_sz ) /* out of bounds */
141 4287 : & ( shnum> 0UL ) ); /* not enough sections */
142 :
143 4284 : REQUIRE( shoff<=(ULONG_MAX/sizeof(fd_elf64_shdr)) ); /* overflow */
144 4284 : ulong const shsz = shnum*sizeof(fd_elf64_shdr);
145 :
146 4284 : ulong const shoff_end = shoff+shsz;
147 4284 : REQUIRE( ( shoff_end>=shoff ) /* overflow */
148 4284 : & ( shoff_end<=elf_sz ) ); /* out of bounds */
149 :
150 : /* Overlap checks */
151 :
152 4281 : REQUIRE( (phoff>=shoff_end) | (shoff>=phoff_end) ); /* overlap shdrs<>phdrs */
153 :
154 4245 : return 0;
155 4281 : }
156 :
157 : /* shdr_get_loaded_size returns the loaded size of a section, i.e. the
158 : number of bytes loaded into the rodata segment. sBPF ELFs grossly
159 : misuse the sh_size parameter. When SHT_NOBITS is set, the actual
160 : section size is zero, and the section size is ignored. */
161 :
162 : static ulong
163 1547868 : shdr_get_loaded_size( fd_elf64_shdr const * shdr ) {
164 1547868 : return fd_ulong_if( shdr->sh_type==FD_ELF_SHT_NOBITS, 0UL, shdr->sh_size );
165 1547868 : }
166 :
167 : /* check_cstr verifies a string in a string table. Returns non-NULL if
168 : the string is null terminated and contains at most max non-NULL
169 : characters. Returns NULL if the off is out of bounds, or if the max
170 : or EOF are reached before the null terminator. */
171 :
172 : static char const *
173 : check_cstr( uchar const * bin,
174 : ulong bin_sz,
175 : ulong off,
176 : ulong max,
177 487893 : ulong * opt_sz ) {
178 487893 : if( FD_UNLIKELY( off>=bin_sz ) ) return NULL;
179 487893 : max += 1UL; /* include NULL terminator */
180 487893 : max = fd_ulong_min( max, bin_sz-off ); /* truncate to available size */
181 487893 : char const * cstr = (char const *)( bin+off );
182 487893 : ulong len = strnlen( cstr, max );
183 487893 : if( opt_sz ) *opt_sz = len;
184 487893 : return len<max ? cstr : NULL;
185 487893 : }
186 :
187 : /* fd_sbpf_load_phdrs walks the program header table. Remembers info
188 : along the way, and performs various validations.
189 :
190 : Assumes that ...
191 : - table does not overlap with file header or section header table and
192 : is within bounds
193 : - offset of program header table is 8 byte aligned */
194 :
195 : static int
196 : fd_sbpf_load_phdrs( fd_sbpf_elf_info_t * info,
197 : fd_sbpf_elf_t const * elf,
198 4245 : ulong elf_sz ) {
199 :
200 4245 : ulong const pht_offset = elf->ehdr.e_phoff;
201 4245 : ulong const pht_cnt = elf->ehdr.e_phnum;
202 :
203 : /* Virtual address of last seen program header */
204 4245 : ulong p_load_vaddr = 0UL;
205 :
206 : /* Read program header table */
207 4245 : fd_elf64_phdr const * phdr = (fd_elf64_phdr const *)( elf->bin + pht_offset );
208 20553 : for( ulong i=0; i<pht_cnt; i++ ) {
209 16332 : switch( phdr[i].p_type ) {
210 4206 : case FD_ELF_PT_DYNAMIC:
211 : /* Remember first PT_DYNAMIC segment */
212 4206 : _fd_int_store_if_negative( &info->phndx_dyn, (int)i );
213 4206 : break;
214 9060 : case FD_ELF_PT_LOAD:
215 : /* LOAD segments must be ordered */
216 9060 : REQUIRE( phdr[ i ].p_vaddr >= p_load_vaddr );
217 9060 : p_load_vaddr = phdr[ i ].p_vaddr;
218 : /* Segment must be within bounds */
219 9060 : REQUIRE( ( phdr[ i ].p_offset + phdr[ i ].p_filesz >= phdr[ i ].p_offset )
220 9060 : & ( phdr[ i ].p_offset + phdr[ i ].p_filesz <= elf_sz ) );
221 : /* No overlap checks */
222 9036 : break;
223 9036 : default:
224 : /* Ignore other segment types */
225 3066 : break;
226 16332 : }
227 16332 : }
228 :
229 4221 : return 0;
230 4245 : }
231 :
232 : /* FD_SBPF_SECTION_NAME_SZ_MAX is the maximum length of a symbol name cstr
233 : including zero terminator.
234 : https://github.com/solana-labs/rbpf/blob/c168a8715da668a71584ea46696d85f25c8918f6/src/elf_parser/mod.rs#L12 */
235 96009 : #define FD_SBPF_SECTION_NAME_SZ_MAX (16UL)
236 :
237 : /* fd_sbpf_load_shdrs walks the section header table. Remembers info
238 : along the way, and performs various validations.
239 :
240 : Assumes that ...
241 : - table does not overlap with file header or program header table and
242 : is within bounds
243 : - offset of section header table is 8 byte aligned
244 : - section header table has at least one entry */
245 :
246 : static int
247 : fd_sbpf_load_shdrs( fd_sbpf_elf_info_t * info,
248 : fd_sbpf_elf_t const * elf,
249 : ulong elf_sz,
250 4221 : int elf_deploy_checks ) {
251 :
252 : /* File Header */
253 4221 : ulong const eh_offset = 0UL;
254 4221 : ulong const eh_offend = sizeof(fd_elf64_ehdr);
255 :
256 : /* Section Header Table */
257 4221 : ulong const sht_offset = elf->ehdr.e_shoff;
258 4221 : ulong const sht_cnt = elf->ehdr.e_shnum;
259 4221 : ulong const sht_sz = sht_cnt*sizeof(fd_elf64_shdr);
260 4221 : ulong const sht_offend = sht_offset + sht_sz;
261 :
262 4221 : fd_elf64_shdr const * shdr = (fd_elf64_shdr const *)( elf->bin + sht_offset );
263 :
264 : /* Program Header Table */
265 4221 : ulong const pht_offset = elf->ehdr.e_phoff;
266 4221 : ulong const pht_cnt = elf->ehdr.e_phnum;
267 4221 : ulong const pht_offend = pht_offset + (pht_cnt*sizeof(fd_elf64_phdr));
268 :
269 : /* Overlap checks */
270 4221 : REQUIRE( (sht_offset>=eh_offend ) | (sht_offend<=eh_offset ) ); /* overlaps ELF file header */
271 4221 : REQUIRE( (sht_offset>=pht_offend) | (sht_offend<=pht_offset) ); /* overlaps program header table */
272 :
273 : /* Require SHT_STRTAB for section name table */
274 :
275 4221 : REQUIRE( elf->ehdr.e_shstrndx < sht_cnt ); /* out of bounds */
276 4221 : REQUIRE( shdr[ elf->ehdr.e_shstrndx ].sh_type==FD_ELF_SHT_STRTAB );
277 :
278 4170 : ulong shstr_off = shdr[ elf->ehdr.e_shstrndx ].sh_offset;
279 4170 : ulong shstr_sz = shdr[ elf->ehdr.e_shstrndx ].sh_size;
280 4170 : REQUIRE( shstr_off<elf_sz );
281 4170 : shstr_sz = fd_ulong_min( shstr_sz, elf_sz-shstr_off );
282 :
283 : /* Clear the "loaded sections" bitmap */
284 :
285 4170 : fd_memset( info->loaded_sections, 0, sizeof(info->loaded_sections) );
286 :
287 : /* Validate section header table.
288 : Check that all sections are in bounds, ordered, and don't overlap. */
289 :
290 4170 : ulong min_sh_offset = 0UL; /* lowest permitted section offset */
291 :
292 : /* Keep track of the physical (file address) end of all relevant
293 : sections to determine rodata_sz */
294 4170 : ulong psegment_end = 0UL; /* Upper bound of physical (file) addressing */
295 :
296 : /* While validating section header table, also figure out size
297 : of the rodata segment. This is the minimal virtual address range
298 : that spans all sections. */
299 4170 : ulong vsegment_start = FD_SBPF_MM_PROGRAM_ADDR; /* Lower bound of segment virtual address */
300 4170 : ulong vsegment_end = 0UL; /* Upper bound of segment virtual address */
301 :
302 4170 : ulong tot_section_sz = 0UL; /* Size of all sections */
303 :
304 52017 : for( ulong i=0UL; i<sht_cnt; i++ ) {
305 48195 : uint sh_type = shdr[ i ].sh_type;
306 48195 : uint sh_name = shdr[ i ].sh_name;
307 48195 : ulong sh_addr = shdr[ i ].sh_addr;
308 48195 : ulong sh_offset = shdr[ i ].sh_offset;
309 48195 : ulong sh_size = shdr[ i ].sh_size;
310 48195 : ulong sh_offend = sh_offset + sh_size;
311 :
312 : /* First section must be SHT_NULL */
313 48195 : REQUIRE( i>0UL || sh_type==FD_ELF_SHT_NULL );
314 :
315 : /* check that physical range has no overflow and is within bounds */
316 48192 : REQUIRE( sh_offend >= sh_offset );
317 48147 : REQUIRE( sh_offend <= elf_sz ); // https://github.com/solana-labs/rbpf/blob/v0.8.0/src/elf_parser/mod.rs#L180
318 :
319 48141 : if( sh_type!=FD_ELF_SHT_NOBITS ) {
320 : /* Overlap checks */
321 47481 : REQUIRE( (sh_offset>=eh_offend ) | (sh_offend<=eh_offset ) ); /* overlaps ELF file header */
322 47481 : REQUIRE( (sh_offset>=pht_offend) | (sh_offend<=pht_offset) ); /* overlaps program header table */
323 47481 : REQUIRE( (sh_offset>=sht_offend) | (sh_offend<=sht_offset) ); /* overlaps section header table */
324 :
325 : /* Ordering and overlap check
326 : https://github.com/solana-labs/rbpf/blob/v0.8.0/src/elf_parser/mod.rs#L177
327 : */
328 47481 : REQUIRE( sh_offset >= min_sh_offset );
329 47481 : min_sh_offset = sh_offend;
330 47481 : }
331 :
332 48141 : if( sh_type==FD_ELF_SHT_DYNAMIC ) {
333 : /* Remember first SHT_DYNAMIC segment */
334 3837 : _fd_int_store_if_negative( &info->shndx_dyn, (int)i );
335 3837 : }
336 :
337 48141 : ulong name_off = shstr_off + (ulong)sh_name;
338 48141 : REQUIRE( ( name_off<elf_sz ) /* out of bounds */
339 48141 : & ( sh_name <shstr_sz ) );
340 :
341 : /* Create name cstr */
342 :
343 48141 : char const * name_ptr = check_cstr( elf->bin + shstr_off, shstr_sz, sh_name, FD_SBPF_SECTION_NAME_SZ_MAX-1UL, NULL );
344 48141 : REQUIRE( name_ptr );
345 47868 : char __attribute__((aligned(16UL))) name[ FD_SBPF_SECTION_NAME_SZ_MAX ] = {0};
346 47868 : strncpy( name, name_ptr, FD_SBPF_SECTION_NAME_SZ_MAX-1UL );
347 :
348 : /* Check name */
349 : /* TODO switch table for this? */
350 : /* TODO reject duplicate sections */
351 :
352 47868 : int load = 0; /* should section be loaded? */
353 :
354 47868 : /**/ if( 0==memcmp( name, ".text", 6UL /* equals */ ) ) {
355 3906 : REQUIRE( (info->shndx_text)<0 ); /* check for duplicate */
356 3906 : info->shndx_text = (int)i;
357 3906 : load = 1;
358 3906 : }
359 43962 : else if( (0==memcmp( name, ".rodata", 8UL /* equals */ ) )
360 43962 : | (0==memcmp( name, ".data.rel.ro", 13UL /* equals */ ) )
361 43962 : | (0==memcmp( name, ".eh_frame", 10UL /* equals */ ) ) ) {
362 8985 : load = 1;
363 8985 : }
364 34977 : else if( 0==memcmp( name, ".symtab", 8UL /* equals */ ) ) {
365 1170 : REQUIRE( (info->shndx_symtab)<0 );
366 1170 : info->shndx_symtab = (int)i;
367 1170 : }
368 33807 : else if( 0==memcmp( name, ".strtab", 8UL /* equals */ ) ) {
369 1197 : REQUIRE( (info->shndx_strtab)<0 );
370 1176 : info->shndx_strtab = (int)i;
371 1176 : }
372 32610 : else if( 0==memcmp( name, ".dynstr", 8UL /* equals */ ) ) {
373 3789 : REQUIRE( (info->shndx_dynstr)<0 );
374 3789 : info->shndx_dynstr = (int)i;
375 3789 : }
376 28821 : else if( 0==memcmp( name, ".bss", 4UL /* has prefix */ ) ) {
377 0 : FAIL();
378 0 : }
379 28821 : else if( 0==memcmp( name, ".data.rel", 9UL /* has prefix */ ) ) {} /* ignore */
380 28512 : else if( (0==memcmp( name, ".data", 5UL /* has prefix */ ) )
381 28512 : & ( ( shdr[ i ].sh_flags & (FD_ELF_SHF_ALLOC|FD_ELF_SHF_WRITE) )
382 28512 : ==(FD_ELF_SHF_ALLOC|FD_ELF_SHF_WRITE) ) ) {
383 0 : FAIL();
384 0 : }
385 28512 : else {} /* ignore */
386 : /* else ignore */
387 :
388 47847 : if( load ) {
389 : /* Remember that section should be loaded */
390 :
391 12891 : info->loaded_sections[ i>>6UL ] |= (1UL)<<(i&63UL);
392 :
393 : /* Check that virtual address range is in MM_PROGRAM bounds */
394 :
395 12891 : ulong sh_actual_size = shdr_get_loaded_size( &shdr[ i ] );
396 12891 : ulong sh_virtual_end = sh_addr + sh_actual_size;
397 :
398 : /* https://github.com/solana-labs/rbpf/blob/v0.8.0/src/elf.rs#L426 */
399 12891 : if ( FD_UNLIKELY( elf_deploy_checks ) ){
400 2499 : REQUIRE( sh_addr == sh_offset );
401 2499 : }
402 12891 : REQUIRE( sh_addr < FD_SBPF_MM_PROGRAM_ADDR ); /* overflow check */
403 12891 : REQUIRE( sh_actual_size < FD_SBPF_MM_PROGRAM_ADDR ); /* overflow check */
404 12891 : REQUIRE( sh_virtual_end <= FD_SBPF_MM_STACK_ADDR-FD_SBPF_MM_PROGRAM_ADDR ); /* check overlap with stack */
405 :
406 : /* Check that physical address range is in bounds
407 : (Seems redundant?) */
408 12891 : ulong paddr_end = sh_offset + sh_actual_size;
409 12891 : REQUIRE( paddr_end >= sh_offset );
410 12891 : REQUIRE( paddr_end <= elf_sz );
411 :
412 12891 : vsegment_start = fd_ulong_min( vsegment_start, sh_addr );
413 : /* Expand range to fit section */
414 12891 : psegment_end = fd_ulong_max( psegment_end, paddr_end );
415 12891 : vsegment_end = fd_ulong_max( vsegment_end, sh_virtual_end );
416 :
417 : /* Coherence check sum of section sizes */
418 12891 : REQUIRE( tot_section_sz + sh_actual_size >= tot_section_sz ); /* overflow check */
419 12891 : tot_section_sz += sh_actual_size;
420 12891 : }
421 47847 : }
422 :
423 : /* More coherence checks */
424 3822 : REQUIRE( psegment_end <= elf_sz ); // https://github.com/solana-labs/rbpf/blob/v0.8.0/src/elf.rs#L782
425 :
426 :
427 : /* Check that the rodata segment is within bounds
428 : https://github.com/solana-labs/rbpf/blob/v0.8.0/src/elf.rs#L725 */
429 3822 : if ( FD_UNLIKELY( elf_deploy_checks ) ){
430 702 : REQUIRE( fd_ulong_sat_add( vsegment_start, tot_section_sz) <= vsegment_end );
431 702 : }
432 :
433 : /* Require .text section */
434 :
435 3813 : REQUIRE( (info->shndx_text)>=0 );
436 3648 : fd_elf64_shdr const * shdr_text = &shdr[ info->shndx_text ];
437 3648 : REQUIRE( (shdr_text->sh_addr <= elf->ehdr.e_entry)
438 : /* check that entrypoint is in text VM range */
439 3648 : & (elf->ehdr.e_entry < fd_ulong_sat_add( shdr_text->sh_addr, shdr_text->sh_size ) ) );
440 : /* NOTE: Does NOT check that the entrypoint is in text section file
441 : range (which may be 0 sz if SHT_NOBITS). This check is
442 : separately done in the sBPF verifier. */
443 :
444 3648 : info->text_off = (uint)shdr_text->sh_offset;
445 3648 : ulong text_size = shdr_get_loaded_size( shdr_text );
446 3648 : info->text_sz = text_size;
447 3648 : info->text_cnt = (uint) text_size / 8U;
448 :
449 :
450 : /* Convert entrypoint offset to program counter */
451 :
452 3648 : info->rodata_sz = (uint)psegment_end;
453 3648 : info->rodata_footprint = (uint)elf_sz;
454 :
455 3648 : ulong entry_off = fd_ulong_sat_sub( elf->ehdr.e_entry, shdr_text->sh_addr );
456 3648 : ulong entry_pc = entry_off / 8UL;
457 :
458 : /* Follows https://github.com/solana-labs/rbpf/blob/v0.8.0/src/elf.rs#L443 */
459 3648 : REQUIRE( fd_ulong_is_aligned( entry_off, 8UL ) );
460 3648 : REQUIRE( entry_pc < ( info->rodata_sz / 8UL ) );
461 3648 : info->entry_pc = (uint)entry_pc;
462 :
463 3648 : if( (info->shndx_dynstr)>=0 ) {
464 3573 : fd_elf64_shdr const * shdr_dynstr = &shdr[ info->shndx_dynstr ];
465 3573 : ulong sh_offset = shdr_dynstr->sh_offset;
466 3573 : ulong sh_size = shdr_dynstr->sh_size;
467 3573 : REQUIRE( (sh_offset+sh_size>=sh_offset) & (sh_offset+sh_size<=info->rodata_footprint) );
468 3573 : info->dynstr_off = (uint)sh_offset;
469 3573 : info->dynstr_sz = (uint)sh_size;
470 3573 : }
471 :
472 3648 : return 0;
473 3648 : }
474 :
475 : static int
476 : _fd_sbpf_elf_peek( fd_sbpf_elf_info_t * info,
477 : void const * bin,
478 : ulong elf_sz,
479 106812 : int elf_deploy_checks ) {
480 :
481 : /* ELFs must have a file header */
482 106812 : REQUIRE( elf_sz>sizeof(fd_elf64_ehdr) );
483 :
484 : /* Reject overlong ELFs (using uint addressing internally).
485 : This is well beyond Solana's max account size of 10 MB. */
486 24399 : REQUIRE( elf_sz<=UINT_MAX );
487 :
488 : /* Initialize info struct */
489 22281 : *info = (fd_sbpf_elf_info_t) {
490 22281 : .text_off = 0U,
491 22281 : .text_cnt = 0U,
492 22281 : .dynstr_off = 0U,
493 22281 : .dynstr_sz = 0U,
494 22281 : .rodata_footprint = 0U,
495 22281 : .rodata_sz = 0U,
496 22281 : .shndx_text = -1,
497 22281 : .shndx_symtab = -1,
498 22281 : .shndx_strtab = -1,
499 22281 : .shndx_dyn = -1,
500 22281 : .shndx_dynstr = -1,
501 22281 : .phndx_dyn = -1,
502 : /* !!! Keep this in sync with -Werror=missing-field-initializers */
503 22281 : };
504 :
505 22281 : fd_sbpf_elf_t const * elf = (fd_sbpf_elf_t const *)bin;
506 22281 : int err;
507 :
508 : /* Validate file header */
509 22281 : if( FD_UNLIKELY( (err=fd_sbpf_check_ehdr( &elf->ehdr, elf_sz ))!=0 ) )
510 18036 : return err;
511 :
512 : /* Program headers */
513 4245 : if( FD_UNLIKELY( (err=fd_sbpf_load_phdrs( info, elf, elf_sz ))!=0 ) )
514 24 : return err;
515 :
516 : /* Section headers */
517 4221 : if( FD_UNLIKELY( (err=fd_sbpf_load_shdrs( info, elf, elf_sz, elf_deploy_checks ))!=0 ) )
518 573 : return err;
519 :
520 3648 : return 0;
521 4221 : }
522 :
523 : fd_sbpf_elf_info_t *
524 : fd_sbpf_elf_peek( fd_sbpf_elf_info_t * info,
525 : void const * bin,
526 : ulong elf_sz,
527 106812 : int elf_deploy_checks ) {
528 106812 : return (_fd_sbpf_elf_peek( info, bin, elf_sz, elf_deploy_checks ) == 0) ? info : NULL;
529 106812 : }
530 :
531 : /* ELF loader, part 2 **************************************************
532 :
533 : Prepare a copy of a subrange of the ELF content: The rodata segment.
534 : Mangle the copy by applying dynamic relocations. Then, zero out
535 : parts of the segment that are not interesting to the loader.
536 :
537 : ### Terminology
538 :
539 : Shorthands for relocation handling:
540 :
541 : S: Symbol value (typically an ELF physical address)
542 : A: Implicit addend, i.e. the original value of the field that the
543 : relocation handler is about to write to
544 : V: Virtual address, i.e. the target value that the relocation
545 : handler is about to write into where the implicit addend was
546 : previously stored */
547 :
548 : ulong
549 3648 : fd_sbpf_program_align( void ) {
550 3648 : return alignof( fd_sbpf_program_t );
551 3648 : }
552 :
553 : ulong
554 3648 : fd_sbpf_program_footprint( fd_sbpf_elf_info_t const * info ) {
555 3648 : FD_COMPILER_UNPREDICTABLE( info ); /* Make this appear as FD_FN_PURE (e.g. footprint might depened on info contents in future) */
556 3648 : return FD_LAYOUT_FINI( FD_LAYOUT_APPEND( FD_LAYOUT_APPEND( FD_LAYOUT_INIT,
557 3648 : alignof(fd_sbpf_program_t), sizeof(fd_sbpf_program_t) ),
558 3648 : fd_sbpf_calldests_align(), fd_sbpf_calldests_footprint( info->rodata_sz / 8UL ) ), /* calldests bitmap */
559 3648 : alignof(fd_sbpf_program_t) );
560 3648 : }
561 :
562 : fd_sbpf_program_t *
563 : fd_sbpf_program_new( void * prog_mem,
564 : fd_sbpf_elf_info_t const * elf_info,
565 3648 : void * rodata ) {
566 :
567 3648 : if( FD_UNLIKELY( !prog_mem ) ) {
568 0 : FD_LOG_WARNING(( "NULL prog_mem" ));
569 0 : return NULL;
570 0 : }
571 :
572 3648 : if( FD_UNLIKELY( !elf_info ) ) {
573 0 : FD_LOG_WARNING(( "NULL elf_info" ));
574 0 : return NULL;
575 0 : }
576 :
577 3648 : if( FD_UNLIKELY( ((elf_info->rodata_footprint)>0U) & (!rodata)) ) {
578 0 : FD_LOG_WARNING(( "NULL rodata" ));
579 0 : return NULL;
580 0 : }
581 :
582 : /* https://github.com/solana-labs/rbpf/blob/v0.8.0/src/elf_parser/mod.rs#L99 */
583 3648 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong) rodata, FD_SBPF_PROG_RODATA_ALIGN ) ) ){
584 0 : FD_LOG_WARNING(( "rodata is not 8-byte aligned" ));
585 0 : return NULL;
586 0 : }
587 :
588 : /* Initialize program struct */
589 :
590 3648 : FD_SCRATCH_ALLOC_INIT( laddr, prog_mem );
591 3648 : fd_sbpf_program_t * prog = FD_SCRATCH_ALLOC_APPEND( laddr, alignof(fd_sbpf_program_t), sizeof(fd_sbpf_program_t) );
592 :
593 0 : *prog = (fd_sbpf_program_t) {
594 3648 : .info = *elf_info,
595 3648 : .rodata = rodata,
596 3648 : .rodata_sz = elf_info->rodata_sz,
597 3648 : .text = (ulong *)((ulong)rodata + elf_info->text_off), /* FIXME: WHAT IF MISALIGNED */
598 3648 : .text_off = elf_info->text_off,
599 3648 : .text_cnt = elf_info->text_cnt,
600 3648 : .text_sz = elf_info->text_sz,
601 3648 : .entry_pc = elf_info->entry_pc
602 3648 : };
603 :
604 : /* Initialize calldests map */
605 :
606 3648 : ulong pc_max = elf_info->rodata_sz / 8UL;
607 3648 : prog->calldests_shmem = fd_sbpf_calldests_new(
608 3648 : FD_SCRATCH_ALLOC_APPEND( laddr, fd_sbpf_calldests_align(),
609 3648 : fd_sbpf_calldests_footprint( pc_max ) ),
610 0 : pc_max );
611 0 : prog->calldests = fd_sbpf_calldests_join( prog->calldests_shmem );
612 :
613 3648 : return prog;
614 3648 : }
615 :
616 : void *
617 30 : fd_sbpf_program_delete( fd_sbpf_program_t * mem ) {
618 :
619 30 : fd_sbpf_calldests_delete( fd_sbpf_calldests_leave( mem->calldests ) );
620 30 : fd_memset( mem, 0, sizeof(fd_sbpf_program_t) );
621 :
622 30 : return (void *)mem;
623 30 : }
624 :
625 : /* fd_sbpf_loader_t contains various temporary state during loading */
626 :
627 : struct fd_sbpf_loader {
628 : /* External objects */
629 : ulong * calldests; /* owned by program */
630 : fd_sbpf_syscalls_t * syscalls; /* owned by caller */
631 :
632 : /* Dynamic table */
633 : uint dyn_off; /* File offset of dynamic table (UINT_MAX=missing) */
634 : uint dyn_cnt; /* Number of dynamic table entries */
635 :
636 : /* Dynamic table entries */
637 : ulong dt_rel;
638 : ulong dt_relent;
639 : ulong dt_relsz;
640 : ulong dt_symtab;
641 :
642 : /* Dynamic symbols */
643 : uint dynsym_off; /* File offset of .dynsym section (0=missing) */
644 : uint dynsym_cnt; /* Symbol count */
645 :
646 : int elf_deploy_checks;
647 : };
648 : typedef struct fd_sbpf_loader fd_sbpf_loader_t;
649 :
650 : /* FD_SBPF_SYM_NAME_SZ_MAX is the maximum length of a symbol name cstr
651 : including zero terminator.
652 : https://github.com/solana-labs/rbpf/blob/c168a8715da668a71584ea46696d85f25c8918f6/src/elf_parser/mod.rs#L13 */
653 439752 : #define FD_SBPF_SYM_NAME_SZ_MAX (64UL)
654 :
655 :
656 : static int
657 : fd_sbpf_find_dynamic( fd_sbpf_loader_t * loader,
658 : fd_sbpf_elf_t const * elf,
659 : ulong elf_sz,
660 3648 : fd_sbpf_elf_info_t const * info ) {
661 :
662 3648 : fd_elf64_shdr const * shdrs = (fd_elf64_shdr const *)( elf->bin + elf->ehdr.e_shoff );
663 3648 : fd_elf64_phdr const * phdrs = (fd_elf64_phdr const *)( elf->bin + elf->ehdr.e_phoff );
664 :
665 : /* Try first PT_DYNAMIC in program header table */
666 :
667 3648 : if( (info->phndx_dyn)>=0 ) {
668 3555 : ulong dyn_off = phdrs[ info->phndx_dyn ].p_offset;
669 3555 : ulong dyn_sz = phdrs[ info->phndx_dyn ].p_filesz;
670 3555 : ulong dyn_end = dyn_off+dyn_sz;
671 :
672 : /* Fall through to SHT_DYNAMIC if invalid */
673 :
674 3555 : if( FD_LIKELY( ( dyn_end>=dyn_off ) /* overflow */
675 3555 : & ( dyn_end<=elf_sz ) /* out of bounds */
676 3555 : & fd_ulong_is_aligned( dyn_off, 8UL ) /* misaligned */
677 3555 : & fd_ulong_is_aligned( dyn_sz, sizeof(fd_elf64_dyn) ) /* misaligned sz */ ) ) {
678 3306 : loader->dyn_off = (uint)dyn_off;
679 3306 : loader->dyn_cnt = (uint)(dyn_sz / sizeof(fd_elf64_dyn));
680 3306 : return 0;
681 3306 : }
682 3555 : }
683 :
684 : /* Try first SHT_DYNAMIC in section header table */
685 :
686 342 : if( (info->shndx_dyn)>0 ) {
687 258 : ulong dyn_off = shdrs[ info->shndx_dyn ].sh_offset;
688 258 : ulong dyn_sz = shdrs[ info->shndx_dyn ].sh_size;
689 258 : ulong dyn_end = dyn_off+dyn_sz;
690 :
691 : /* This time, don't tolerate errors */
692 :
693 258 : REQUIRE( ( dyn_end>=dyn_off ) /* overflow */
694 258 : & ( dyn_end<=elf_sz ) /* out of bounds */
695 258 : & fd_ulong_is_aligned( dyn_off, 8UL ) /* misaligned */
696 258 : & fd_ulong_is_aligned( dyn_sz, sizeof(fd_elf64_dyn) ) /* misaligned sz */ );
697 :
698 258 : loader->dyn_off = (uint)dyn_off;
699 258 : loader->dyn_cnt = (uint)(dyn_sz / sizeof(fd_elf64_dyn));
700 258 : return 0;
701 258 : }
702 :
703 : /* Missing or invalid PT_DYNAMIC and missing SHT_DYNAMIC, skip. */
704 84 : return 0;
705 342 : }
706 :
707 : static int
708 : fd_sbpf_load_dynamic( fd_sbpf_loader_t * loader,
709 : fd_sbpf_elf_t const * elf,
710 3648 : ulong elf_sz ) {
711 :
712 3648 : fd_elf64_shdr const * shdrs = (fd_elf64_shdr const *)( elf->bin + elf->ehdr.e_shoff );
713 :
714 : /* Skip if no dynamic table was found */
715 :
716 3648 : if( !loader->dyn_cnt ) return 0;
717 :
718 : /* Walk dynamic table */
719 :
720 3555 : fd_elf64_dyn const * dyn = (fd_elf64_dyn const *)( elf->bin + loader->dyn_off );
721 3555 : ulong const dyn_cnt = loader->dyn_cnt;
722 :
723 108897 : for( ulong i=0; i<dyn_cnt; i++ ) {
724 108810 : if( FD_UNLIKELY( dyn[i].d_tag==FD_ELF_DT_NULL ) ) break;
725 :
726 105342 : ulong d_val = dyn[i].d_un.d_val;
727 105342 : switch( dyn[i].d_tag ) {
728 3207 : case FD_ELF_DT_REL: loader->dt_rel =d_val; break;
729 3261 : case FD_ELF_DT_RELENT: loader->dt_relent=d_val; break;
730 3261 : case FD_ELF_DT_RELSZ: loader->dt_relsz =d_val; break;
731 3378 : case FD_ELF_DT_SYMTAB: loader->dt_symtab=d_val; break;
732 105342 : }
733 105342 : }
734 :
735 : /* Load dynamic symbol table */
736 :
737 3555 : if( loader->dt_symtab ) {
738 : /* Search for dynamic symbol table
739 : FIXME unfortunate bounded O(n^2) -- could convert to binary search */
740 :
741 : /* FIXME this could be clobbered by relocations, causing strict
742 : aliasing violations */
743 :
744 3378 : fd_elf64_shdr const * shdr_dynsym = NULL;
745 :
746 14580 : for( ulong i=0; i<elf->ehdr.e_shnum; i++ ) {
747 14580 : if( shdrs[ i ].sh_addr == loader->dt_symtab ) {
748 : /* TODO: verify this ... */
749 : /* Check section type */
750 3378 : uint sh_type = shdrs[ i ].sh_type;
751 : // https://github.com/solana-labs/rbpf/blob/v0.8.5/src/elf_parser/mod.rs#L500
752 3378 : REQUIRE( (sh_type==FD_ELF_SHT_SYMTAB) | (sh_type==FD_ELF_SHT_DYNSYM) );
753 :
754 2802 : shdr_dynsym = &shdrs[ i ];
755 2802 : break;
756 3378 : }
757 14580 : }
758 2802 : REQUIRE( shdr_dynsym );
759 :
760 : /* Check if out of bounds or misaligned */
761 :
762 2802 : ulong sh_offset = shdr_dynsym->sh_offset;
763 2802 : ulong sh_size = shdr_dynsym->sh_size;
764 :
765 2802 : REQUIRE( ( sh_offset+sh_size>=sh_offset )
766 2802 : & ( sh_offset+sh_size<=elf_sz )
767 2802 : & ( fd_ulong_is_aligned( sh_offset, 8UL ) )
768 2802 : & ( sh_size % sizeof (fd_elf64_sym) == 0UL ) );
769 :
770 2802 : loader->dynsym_off = (uint)sh_offset;
771 2802 : loader->dynsym_cnt = (uint)(sh_size/sizeof(fd_elf64_sym));
772 2802 : }
773 :
774 2979 : return 0;
775 3555 : }
776 :
777 : /* ELF Dynamic Relocations *********************************************
778 :
779 : ### Summary
780 :
781 : The sBPF ELF loader provides a limited dynamic relocation mechanism
782 : to fix up Clang-generated shared objects for execution in an sBPF VM.
783 :
784 : The relocation types themselves violate the eBPF and ELF specs in
785 : various ways. In short, the relocation table (via DT_REL) is used to
786 : shift program code from zero-based addressing to the MM_PROGRAM
787 : segment in the VM memory map (at 0x1_0000_0000).
788 :
789 : As part of the Solana VM protocol it abides by strict determinism
790 : requirements. This sadly means that we will have to replicate all
791 : edge cases and bugs in the Solana Labs ELF loader.
792 :
793 : Three relocation types are currently supported:
794 : - R_BPF_64_64: Sets an absolute address of a symbol as the
795 : 64-bit immediate field of an lddw instruction
796 : - R_BPF_64_RELATIVE: Adds MM_PROGRAM_START (0x1_0000_0000) to ...
797 : a) ... the 64-bit imm field of an lddw instruction (if in text)
798 : b) ... a 64-bit integer (if not in text section)
799 : - R_BPF_64_32: Sets the 32-bit immediate field of a call
800 : instruction to ...
801 : a) the ID of a local function (Murmur3 hash of function PC address)
802 : b) the ID of a syscall
803 :
804 : Obviously invalid relocations (e.g. out-of-bounds of ELF file or
805 : unsupported reloc type) raise an error.
806 : Relocations that would corrupt ELF data structures are silently
807 : ignored (using the fd_sbpf_reloc_mask mechanism).
808 :
809 : ### History
810 :
811 : The use of relocations is technically redundant, as the Solana VM
812 : memory map has been hardcoded in program runtime v1 (so far the only
813 : runtime). However, virtually all deployed programs as of April 2023
814 : are position-independent shared objects and make heavy use of such
815 : relocations.
816 :
817 : Relocations in the Solana VM have a complicated history. Over the
818 : course of years, multiple protocol bugs have been added and fixed.
819 : The ELF loader needs to handle all these edge cases to avoid breaking
820 : "userspace". I.e. any deployed programs which might be immutable
821 : must continue to function.
822 :
823 : While this complex logic will probably stick around for the next few
824 : years, the Solana protocol is getting increasingly restrictive for
825 : newly deployed ELFs. Another proposed change is upgrading to
826 : position-dependent binaries without any dynamic relocations. */
827 :
828 : /* R_BPF_64_64 relocates an absolute address into the extended imm field
829 : of an lddw-form instruction. (Two instruction slots, low 32 bits in
830 : first immediate field, high 32 bits in second immediate field)
831 :
832 : Bits 0..32 32..64 64..96 96..128
833 : [ ... ] [ IMM_LO ] [ ... ] [ IMM_HI ] */
834 :
835 : static int
836 : fd_sbpf_r_bpf_64_64( fd_sbpf_loader_t const * loader,
837 : fd_sbpf_elf_t const * elf,
838 : ulong elf_sz,
839 : uchar * rodata,
840 : fd_sbpf_elf_info_t const * info,
841 123 : fd_elf64_rel const * rel ) {
842 :
843 123 : (void)info;
844 :
845 123 : uint r_sym = FD_ELF64_R_SYM( rel->r_info );
846 123 : ulong r_offset = rel->r_offset;
847 :
848 : /* Bounds check */
849 123 : REQUIRE( ( r_offset+16UL> r_offset )
850 123 : & ( r_offset+16UL<=elf_sz ) );
851 :
852 : /* Offsets of implicit addend (immediate fields) */
853 123 : ulong A_off_lo = r_offset+ 4UL;
854 123 : ulong A_off_hi = r_offset+12UL;
855 :
856 : /* Read implicit addend (imm field of first insn slot) */
857 : // SBF_V2: ulong A_off = is_text ? r_offset+4UL : r_offset;
858 123 : REQUIRE( A_off_lo+4UL<elf_sz );
859 :
860 : /* Lookup symbol */
861 123 : REQUIRE( r_sym < loader->dynsym_cnt );
862 72 : fd_elf64_sym const * dynsyms = (fd_elf64_sym const *)( elf->bin + loader->dynsym_off );
863 72 : fd_elf64_sym const * sym = &dynsyms[ r_sym ];
864 72 : ulong S = sym->st_value;
865 :
866 : /* Relocate */
867 72 : ulong A = FD_LOAD( uint, &rodata[ A_off_lo ] );
868 72 : ulong V = fd_ulong_sat_add( S, A );
869 72 : if( V<FD_SBPF_MM_PROGRAM_ADDR ) V+=FD_SBPF_MM_PROGRAM_ADDR;
870 :
871 : /* Write back */
872 72 : FD_STORE( uint, &rodata[ A_off_lo ], (uint)(V ) );
873 72 : FD_STORE( uint, &rodata[ A_off_hi ], (uint)(V>>32UL) );
874 :
875 72 : return 0;
876 123 : }
877 :
878 : /* R_BPF_64_RELATIVE is almost entirely Solana specific. */
879 :
880 : static int
881 : fd_sbpf_r_bpf_64_relative( fd_sbpf_elf_t const * elf,
882 : ulong elf_sz,
883 : uchar * rodata,
884 : fd_sbpf_elf_info_t const * info,
885 1528257 : fd_elf64_rel const * rel ) {
886 :
887 1528257 : ulong r_offset = rel->r_offset;
888 :
889 : /* Is reloc target in .text section? */
890 1528257 : fd_elf64_shdr const * shdrs = (fd_elf64_shdr const *)( elf->bin + elf->ehdr.e_shoff );
891 1528257 : fd_elf64_shdr const * shdr_text = &shdrs[ info->shndx_text ];
892 1528257 : int is_text = ( ( r_offset >= shdr_text->sh_offset ) &
893 1528257 : ( r_offset < shdr_text->sh_offset +
894 1528257 : shdr_get_loaded_size( shdr_text ) ) );
895 :
896 1528257 : if( is_text ) {
897 : /* If reloc target is in .text, behave like R_BPF_64_64, except:
898 : - R_SYM(r_info) is ignored
899 : - If implicit addend looks like a physical address, make it
900 : a virtual address (by adding a constant offset)
901 :
902 : This relocation type seems to make little sense but is required
903 : for most programs. */
904 :
905 1085241 : REQUIRE( (r_offset+16UL>r_offset) & (r_offset+16UL<=elf_sz) );
906 1085238 : ulong imm_lo_off = r_offset+ 4UL;
907 1085238 : ulong imm_hi_off = r_offset+12UL;
908 :
909 : /* Read implicit addend */
910 1085238 : uint va_lo = FD_LOAD( uint, rodata+imm_lo_off );
911 1085238 : uint va_hi = FD_LOAD( uint, rodata+imm_hi_off );
912 1085238 : ulong va = ( (ulong)va_hi<<32UL ) | va_lo;
913 :
914 1085238 : REQUIRE( va!=0UL );
915 1085238 : va = va<FD_SBPF_MM_PROGRAM_ADDR ? va+FD_SBPF_MM_PROGRAM_ADDR : va;
916 :
917 : /* Write back
918 : Skip bounds check as .text is guaranteed to be writable */
919 1085238 : FD_STORE( uint, rodata+imm_lo_off, (uint)( va ) );
920 1085238 : FD_STORE( uint, rodata+imm_hi_off, (uint)( va>>32UL ) );
921 1085238 : } else {
922 : /* Outside .text do a 64-bit write */
923 :
924 : /* Bounds checks */
925 443016 : REQUIRE( (r_offset+8UL>r_offset) & (r_offset+8UL<=elf_sz) );
926 :
927 : /* Read implicit addend */
928 443016 : ulong va = FD_LOAD( uint, rodata+r_offset+4UL );
929 :
930 : /* Relocate */
931 443016 : va = fd_ulong_sat_add( va, FD_SBPF_MM_PROGRAM_ADDR );
932 :
933 : /* Write back */
934 443016 : FD_STORE( ulong, rodata+r_offset, va );
935 443016 : }
936 :
937 1528254 : return 0;
938 1528257 : }
939 :
940 : static int
941 : fd_sbpf_r_bpf_64_32( fd_sbpf_loader_t const * loader,
942 : fd_sbpf_elf_t const * elf,
943 : ulong elf_sz,
944 : uchar * rodata,
945 : fd_sbpf_elf_info_t const * info,
946 439752 : fd_elf64_rel const * rel ) {
947 :
948 439752 : uint r_sym = FD_ELF64_R_SYM( rel->r_info );
949 439752 : ulong r_offset = rel->r_offset;
950 :
951 : /* Lookup symbol */
952 439752 : REQUIRE( r_sym < loader->dynsym_cnt );
953 439752 : fd_elf64_shdr const * shdrs = (fd_elf64_shdr const *)( elf->bin + elf->ehdr.e_shoff );
954 439752 : fd_elf64_sym const * dynsyms = (fd_elf64_sym const *)( elf->bin + loader->dynsym_off );
955 439752 : fd_elf64_sym const * sym = &dynsyms[ r_sym ];
956 439752 : ulong S = sym->st_value;
957 :
958 : /* Verify .dynstr (TODO can we lift this out of the reloc handler?) */
959 439752 : REQUIRE( info->shndx_dynstr > 0 );
960 439752 : REQUIRE( shdrs[ info->shndx_dynstr ].sh_type == FD_ELF_SHT_STRTAB );
961 :
962 : /* Verify symbol name */
963 439752 : ulong name_len;
964 439752 : char const * name = check_cstr( elf->bin + info->dynstr_off, info->dynstr_sz, sym->st_name, FD_SBPF_SYM_NAME_SZ_MAX-1UL, &name_len );
965 439752 : REQUIRE( name );
966 :
967 : /* Value to write into relocated field */
968 439752 : uint V;
969 :
970 439752 : int is_func_call = ( FD_ELF64_ST_TYPE( sym->st_info ) == FD_ELF_STT_FUNC )
971 439752 : & ( S!=0UL );
972 439752 : if( is_func_call ) {
973 : /* Check whether function call is in virtual memory range of text section. */
974 3801 : fd_elf64_shdr const * shdr_text = &shdrs[ info->shndx_text ];
975 3801 : ulong sh_addr = shdr_text->sh_addr;
976 3801 : ulong sh_size = shdr_text->sh_size;
977 3801 : REQUIRE( (S>=sh_addr) & (S<sh_addr+sh_size) );
978 :
979 : /* Note: The above check is broken, as sh_size is interpreted as 0
980 : for SHT_NOBITS section. Yet, this is the "correct" loading
981 : behavior according to protocol rules. */
982 :
983 : /* Register function call */
984 3801 : ulong target_pc = (S-sh_addr) / 8UL;
985 :
986 : /* TODO bounds check the target? */
987 :
988 : /* Register new entry */
989 3801 : uint hash;
990 3801 : if( name_len >= 10UL && 0==strncmp( name, "entrypoint", name_len ) ) {
991 : /* Skip insertion of "entrypoint" relocation entries to calldests. This
992 : emulates Solana/Agave's behavior of unregistering these entries before
993 : registering the entrypoint manually.
994 : Entrypoint is registered in fd_sbpf_program_load.
995 : Hash is still applied. */
996 111 : hash = 0x71e3cf81;
997 3690 : } else {
998 3690 : hash = fd_pchash( (uint)target_pc );
999 3690 : if( FD_LIKELY( target_pc < (info->rodata_sz / 8UL ) ) )
1000 3690 : fd_sbpf_calldests_insert( loader->calldests, target_pc );
1001 3690 : }
1002 :
1003 : /* Check for collision with syscall ID
1004 : https://github.com/solana-labs/rbpf/blob/57139e9e1fca4f01155f7d99bc55cdcc25b0bc04/src/program.rs#L142-L146 */
1005 3801 : REQUIRE( !fd_sbpf_syscalls_query( loader->syscalls, hash, NULL ) );
1006 3801 : V = (uint)hash;
1007 435951 : } else {
1008 : /* FIXME Should cache Murmur hashes.
1009 : If max ELF size is 10MB, can fit about 640k relocs.
1010 : Each reloc could point to a symbol with the same st_name,
1011 : which results in 640MB hash input data without caching. */
1012 435951 : uint hash = fd_murmur3_32( name, name_len, 0UL );
1013 : /* Ensure that requested syscall ID exists only when deploying
1014 : https://github.com/solana-labs/rbpf/blob/v0.8.0/src/elf.rs#L1097 */
1015 435951 : if ( FD_UNLIKELY( loader->elf_deploy_checks ) ) {
1016 9207 : REQUIRE( fd_sbpf_syscalls_query( loader->syscalls, hash, NULL ) );
1017 9207 : }
1018 :
1019 435852 : V = hash;
1020 435852 : }
1021 :
1022 : /* Bounds checks */
1023 439653 : REQUIRE( (r_offset+8UL>r_offset) & (r_offset+8UL<=elf_sz) );
1024 439593 : ulong A_off = r_offset+4UL;
1025 :
1026 : /* Apply relocation */
1027 439593 : FD_STORE( uint, rodata+A_off, V );
1028 :
1029 439593 : return 0;
1030 439653 : }
1031 :
1032 : static int
1033 : fd_sbpf_apply_reloc( fd_sbpf_loader_t const * loader,
1034 : fd_sbpf_elf_t const * elf,
1035 : ulong elf_sz,
1036 : uchar * rodata,
1037 : fd_sbpf_elf_info_t const * info,
1038 1968189 : fd_elf64_rel const * rel ) {
1039 1968189 : switch( FD_ELF64_R_TYPE( rel->r_info ) ) {
1040 123 : case FD_ELF_R_BPF_64_64:
1041 123 : return fd_sbpf_r_bpf_64_64 ( loader, elf, elf_sz, rodata, info, rel );
1042 1528257 : case FD_ELF_R_BPF_64_RELATIVE:
1043 1528257 : return fd_sbpf_r_bpf_64_relative( elf, elf_sz, rodata, info, rel );
1044 439752 : case FD_ELF_R_BPF_64_32:
1045 439752 : return fd_sbpf_r_bpf_64_32 ( loader, elf, elf_sz, rodata, info, rel );
1046 57 : default:
1047 57 : ERR( FD_SBPF_ERR_INVALID_ELF );
1048 1968189 : }
1049 1968189 : }
1050 :
1051 : /* fd_sbpf_hash_calls converts local call instructions in the "LLVM
1052 : form" (immediate is a program counter offset) to eBPF form (immediate
1053 : is a hash of the target program counter). Corresponds to
1054 : fixup_relative calls in solana-labs/rbpf.
1055 :
1056 : Assumes that the text section range exists, is within bounds, and
1057 : does not overlap with the ELF file header, program header table, or
1058 : section header table. */
1059 :
1060 : static int
1061 : fd_sbpf_hash_calls( fd_sbpf_loader_t * loader,
1062 : fd_sbpf_program_t * prog,
1063 3072 : fd_sbpf_elf_t const * elf ) {
1064 :
1065 3072 : fd_elf64_shdr const * shdrs = (fd_elf64_shdr const *)( elf->bin + elf->ehdr.e_shoff );
1066 3072 : fd_sbpf_elf_info_t * info = &prog->info;
1067 3072 : uchar * rodata = prog->rodata;
1068 :
1069 3072 : fd_elf64_shdr const * shtext = &shdrs[ info->shndx_text ];
1070 3072 : fd_sbpf_calldests_t * calldests = loader->calldests;
1071 :
1072 3072 : uchar * ptr = rodata + shtext->sh_offset;
1073 3072 : ulong insn_cnt = shdr_get_loaded_size( shtext ) / 8UL;
1074 :
1075 55387617 : for( ulong i=0; i<insn_cnt; i++, ptr+=8UL ) {
1076 55384656 : ulong insn = *((ulong *) ptr);
1077 :
1078 : /* Check for call instruction. If immediate is UINT_MAX, assume
1079 : that compiler generated a relocation instead. */
1080 55384656 : ulong opc = insn & 0xFF;
1081 55384656 : int imm = (int)(insn >> 32UL);
1082 55384656 : if( (opc!=0x85) | (imm==-1) )
1083 51772047 : continue;
1084 :
1085 : /* Mark function call destination */
1086 3612609 : long target_pc_s;
1087 3612609 : REQUIRE( 0==__builtin_saddl_overflow( (long)i+1L, imm, &target_pc_s ) );
1088 3612609 : ulong target_pc = (ulong)target_pc_s;
1089 3612609 : REQUIRE( target_pc<insn_cnt ); /* bounds check target */
1090 :
1091 3612498 : fd_sbpf_calldests_insert( calldests, target_pc );
1092 :
1093 : /* Replace immediate with hash */
1094 3612498 : uint pc_hash = fd_pchash( (uint)target_pc );
1095 : /* Check for collision with syscall ID
1096 : https://github.com/solana-labs/rbpf/blob/57139e9e1fca4f01155f7d99bc55cdcc25b0bc04/src/program.rs#L142-L146 */
1097 3612498 : REQUIRE( !fd_sbpf_syscalls_query( loader->syscalls, pc_hash, NULL ) );
1098 :
1099 3612498 : FD_STORE( uint, ptr+4UL, pc_hash );
1100 3612498 : }
1101 :
1102 2961 : return 0;
1103 3072 : }
1104 :
1105 : static int
1106 : fd_sbpf_relocate( fd_sbpf_loader_t const * loader,
1107 : fd_sbpf_elf_t const * elf,
1108 : ulong elf_sz,
1109 : uchar * rodata,
1110 2961 : fd_sbpf_elf_info_t const * info ) {
1111 :
1112 2961 : ulong const dt_rel = loader->dt_rel;
1113 2961 : ulong const dt_relent = loader->dt_relent;
1114 2961 : ulong const dt_relsz = loader->dt_relsz;
1115 :
1116 : /* Skip relocation if DT_REL is missing */
1117 :
1118 2961 : if( dt_rel == 0UL ) return 0;
1119 :
1120 : /* Validate reloc table params */
1121 :
1122 2583 : REQUIRE( dt_relent==sizeof(fd_elf64_rel) );
1123 2577 : REQUIRE( dt_relsz !=0UL );
1124 2577 : REQUIRE( (dt_relsz % sizeof(fd_elf64_rel))==0UL );
1125 :
1126 : /* Resolve DT_REL virtual address to file offset
1127 : First, attempt to find segment containing DT_REL */
1128 :
1129 2577 : ulong rel_off = ULONG_MAX;
1130 :
1131 2577 : fd_elf64_phdr const * phdrs = (fd_elf64_phdr const *)( elf->bin + elf->ehdr.e_phoff );
1132 2577 : ulong rel_phnum;
1133 4395 : for( rel_phnum=0; rel_phnum < elf->ehdr.e_phnum; rel_phnum++ ) {
1134 4335 : ulong va_lo = phdrs[ rel_phnum ].p_vaddr;
1135 4335 : ulong va_hi = phdrs[ rel_phnum ].p_memsz + va_lo;
1136 4335 : REQUIRE( va_hi>=va_lo );
1137 4335 : if( (dt_rel>=va_lo) & (dt_rel<va_hi) ) {
1138 : /* Found */
1139 2517 : ulong va_off = dt_rel - va_lo;
1140 2517 : ulong pa_lo = phdrs[ rel_phnum ].p_offset + va_off;
1141 : /* Overflow checks */
1142 2517 : REQUIRE( (va_off<=dt_rel)
1143 2517 : & (pa_lo >=va_off)
1144 2517 : & (pa_lo < elf_sz) );
1145 2517 : rel_off = pa_lo;
1146 2517 : break;
1147 2517 : }
1148 4335 : }
1149 :
1150 : /* DT_REL not contained in any segment. Fallback to section header
1151 : table for finding first dynamic reloc section. */
1152 :
1153 2577 : if( rel_phnum == elf->ehdr.e_phnum ) {
1154 60 : fd_elf64_shdr const * shdrs = (fd_elf64_shdr const *)( elf->bin + elf->ehdr.e_shoff );
1155 60 : ulong rel_shnum;
1156 462 : for( rel_shnum=0; rel_shnum < elf->ehdr.e_shnum; rel_shnum++ )
1157 462 : if( shdrs[ rel_shnum ].sh_addr==dt_rel )
1158 60 : break;
1159 60 : REQUIRE( rel_shnum < elf->ehdr.e_shnum );
1160 60 : rel_off = shdrs[ rel_shnum ].sh_offset;
1161 60 : }
1162 :
1163 2577 : REQUIRE( fd_ulong_is_aligned( rel_off, 8UL ) );
1164 2577 : REQUIRE( (rel_off < elf_sz)
1165 2577 : & (dt_relsz <= elf_sz)
1166 2577 : & ((rel_off+dt_relsz) <= elf_sz) );
1167 :
1168 : /* Load section and reloc tables
1169 : Assume section header already validated at this point */
1170 :
1171 2577 : fd_elf64_rel const * rel = (fd_elf64_rel const *)( elf->bin + rel_off );
1172 2577 : ulong rel_cnt = dt_relsz/sizeof(fd_elf64_rel);
1173 :
1174 : /* Apply each reloc */
1175 :
1176 1970496 : for( ulong i=0; i<rel_cnt; i++ ) {
1177 1968189 : int res = fd_sbpf_apply_reloc( loader, elf, elf_sz, rodata, info, &rel[ i ] );
1178 1968189 : if( res!=0 ) return res;
1179 1968189 : }
1180 :
1181 2307 : return 0;
1182 2577 : }
1183 :
1184 : static int
1185 : fd_sbpf_zero_rodata( fd_sbpf_elf_t * elf,
1186 : uchar * rodata,
1187 2685 : fd_sbpf_elf_info_t const * info ) {
1188 :
1189 2685 : fd_elf64_shdr const * shdrs = (fd_elf64_shdr const *)( elf->bin + elf->ehdr.e_shoff );
1190 :
1191 : /* memset gaps between sections to zero.
1192 : Assume section sh_addrs are monotonically increasing.
1193 : Assume section virtual address ranges equal physical address ranges.
1194 : Assume ranges are not overflowing. */
1195 : /* FIXME match Solana more closely here */
1196 :
1197 2685 : ulong cursor = 0UL;
1198 33141 : for( ulong i=0; i<elf->ehdr.e_shnum; i++ ) {
1199 30456 : if( !( info->loaded_sections[ i>>6UL ] & (1UL<<(i&63UL)) ) ) continue;
1200 :
1201 9564 : fd_elf64_shdr const * shdr = &shdrs[ i ];
1202 :
1203 : /* NOBITS sections are included in rodata, but may have invalid
1204 : offsets, thus we can't trust the shdr->sh_offset field. */
1205 9564 : if( FD_UNLIKELY( shdr->sh_type==FD_ELF_SHT_NOBITS ) ) continue;
1206 :
1207 9525 : ulong off = shdr->sh_offset;
1208 9525 : ulong sz = shdr->sh_size;
1209 9525 : assert( cursor<=off ); /* Invariant: Monotonically increasing offsets */
1210 9525 : assert( off+sz>=off ); /* Invariant: No integer overflow */
1211 9525 : assert( off+sz<=info->rodata_sz ); /* Invariant: No buffer overflow */
1212 :
1213 : /* Fill gap with zeros */
1214 9525 : ulong gap = off - cursor;
1215 9525 : fd_memset( rodata+cursor, 0, gap );
1216 :
1217 9525 : cursor = off+sz;
1218 9525 : }
1219 :
1220 2685 : fd_memset( rodata+cursor, 0, info->rodata_sz - cursor );
1221 :
1222 2685 : return 0;
1223 2685 : }
1224 :
1225 : int
1226 : fd_sbpf_program_load( fd_sbpf_program_t * prog,
1227 : void const * _bin,
1228 : ulong elf_sz,
1229 : fd_sbpf_syscalls_t * syscalls,
1230 3648 : int elf_deploy_checks ) {
1231 3648 : fd_sbpf_loader_seterr( 0, 0 );
1232 :
1233 3648 : int err;
1234 3648 : fd_sbpf_elf_t * elf = (fd_sbpf_elf_t *)_bin;
1235 :
1236 3648 : fd_sbpf_loader_t loader = {
1237 3648 : .calldests = prog->calldests,
1238 3648 : .syscalls = syscalls,
1239 :
1240 3648 : .dyn_off = 0U,
1241 3648 : .dyn_cnt = 0U,
1242 :
1243 3648 : .dt_rel = 0UL,
1244 3648 : .dt_relent = 0UL,
1245 3648 : .dt_relsz = 0UL,
1246 3648 : .dt_symtab = 0UL,
1247 :
1248 3648 : .dynsym_off = 0U,
1249 3648 : .dynsym_cnt = 0U,
1250 3648 : .elf_deploy_checks = elf_deploy_checks
1251 3648 : };
1252 :
1253 : /* Find dynamic section */
1254 3648 : if( FD_UNLIKELY( (err=fd_sbpf_find_dynamic( &loader, elf, elf_sz, &prog->info ))!=0 ) )
1255 0 : return err;
1256 :
1257 : /* Load dynamic section */
1258 3648 : if( FD_UNLIKELY( (err=fd_sbpf_load_dynamic( &loader, elf, elf_sz ))!=0 ) )
1259 576 : return err;
1260 :
1261 : /* Register entrypoint to calldests. */
1262 3072 : fd_sbpf_calldests_insert( prog->calldests, prog->entry_pc );
1263 :
1264 : /* Copy rodata segment */
1265 3072 : fd_memcpy( prog->rodata, elf->bin, prog->info.rodata_footprint );
1266 :
1267 : /* Convert calls with PC relative immediate to hashes */
1268 3072 : if( FD_UNLIKELY( (err=fd_sbpf_hash_calls ( &loader, prog, elf ))!=0 ) )
1269 111 : return err;
1270 :
1271 : /* Apply relocations */
1272 2961 : if( FD_UNLIKELY( (err=fd_sbpf_relocate ( &loader, elf, elf_sz, prog->rodata, &prog->info ))!=0 ) )
1273 276 : return err;
1274 :
1275 : /* Create read-only segment */
1276 2685 : if( FD_UNLIKELY( (err=fd_sbpf_zero_rodata( elf, prog->rodata, &prog->info ))!=0 ) )
1277 0 : return err;
1278 :
1279 2685 : return 0;
1280 2685 : }
1281 :
1282 : #undef ERR
1283 : #undef FAIL
1284 : #undef REQUIRE
|
