Line data Source code
1 : #include "fd_sbpf_loader.h"
2 : #include "fd_sbpf_opcodes.h"
3 : #include "../../util/fd_util.h"
4 : #include "../../util/bits/fd_sat.h"
5 : #include "../murmur3/fd_murmur3.h"
6 :
7 : #include <assert.h>
8 : #include <stdio.h>
9 :
10 : /* Error handling *****************************************************/
11 :
12 : /* Thread local storage last error value */
13 :
14 : static FD_TL int ldr_errno = 0;
15 : static FD_TL int ldr_err_srcln = -1;
16 : #define FD_SBPF_ERRBUF_SZ (128UL)
17 : static FD_TL char fd_sbpf_errbuf[ FD_SBPF_ERRBUF_SZ ] = {0};
18 :
19 : /* fd_sbpf_loader_seterr remembers the error ID and line number of the
20 : current file at which the last error occurred. */
21 :
22 : __attribute__((cold,noinline)) static int
23 : fd_sbpf_loader_seterr( int err,
24 78 : int srcln ) {
25 78 : ldr_errno = err;
26 78 : ldr_err_srcln = srcln;
27 78 : return err;
28 78 : }
29 :
30 : /* Macros for returning an error from the current function while also
31 : remembering the error code. */
32 :
33 45 : #define ERR( err ) return fd_sbpf_loader_seterr( (err), __LINE__ )
34 45 : #define FAIL() ERR( FD_SBPF_ERR_INVALID_ELF )
35 9450 : #define REQUIRE(x) do { if ( FD_UNLIKELY( !(x) ) ) FAIL(); } while (0)
36 :
37 : char const *
38 0 : fd_sbpf_strerror( void ) {
39 0 : if( FD_UNLIKELY( ldr_errno==0 ) )
40 0 : strcpy( fd_sbpf_errbuf, "ok" );
41 0 : else
42 0 : snprintf( fd_sbpf_errbuf, FD_SBPF_ERRBUF_SZ,
43 0 : "code %d at %s(%d)", ldr_errno, __FILE__, ldr_err_srcln );
44 0 : return fd_sbpf_errbuf;
45 0 : }
46 :
47 : /* ELF loader, part 1 **************************************************
48 :
49 : Start with a static piece of scratch memory and do basic validation
50 : of the file content. Walk the section table once and remember
51 : sections of interest.
52 :
53 : ### Terminology
54 :
55 : This source follows common ELF naming practices.
56 :
57 : section: a named data region present in the ELF file
58 : segment: a contiguous memory region containing sections
59 : (not necessarily contiguous in the ELF file)
60 :
61 : physical address (paddr): Byte offset into ELF file (uchar * bin)
62 : virtual address (vaddr): VM memory address */
63 :
64 : /* Provide convenient access to file header and ELF content */
65 :
66 : __extension__ union fd_sbpf_elf {
67 : fd_elf64_ehdr ehdr;
68 : uchar bin[0];
69 : };
70 : typedef union fd_sbpf_elf fd_sbpf_elf_t;
71 :
72 : /* FD_SBPF_MM_{...}_ADDR are hardcoded virtual addresses of segments
73 : in the sBPF virtual machine.
74 :
75 : FIXME: These should be defined elsewhere */
76 :
77 381 : #define FD_SBPF_MM_PROGRAM_ADDR (0x100000000UL) /* readonly program data */
78 : #define FD_SBPF_MM_STACK_ADDR (0x200000000UL) /* stack (with gaps) */
79 :
80 : /* _fd_int_store_if_negative stores x to *p if *p is negative (branchless) */
81 :
82 : static inline int
83 : _fd_int_store_if_negative( int * p,
84 102 : int x ) {
85 102 : return (*p = fd_int_if( (*p)<0, x, *p ));
86 102 : }
87 :
88 : /* fd_sbpf_check_ehdr verifies the ELF file header. */
89 :
90 : static int
91 : fd_sbpf_check_ehdr( fd_elf64_ehdr const * ehdr,
92 : ulong elf_sz,
93 : uint min_version,
94 93 : uint max_version ) {
95 :
96 : /* Validate ELF magic */
97 93 : REQUIRE( ( fd_uint_load_4( ehdr->e_ident )==0x464c457fU )
98 : /* Validate file type/target identification
99 : Solana/Agave performs header checks across two places:
100 : - Elf64::parse https://github.com/solana-labs/rbpf/blob/v0.8.0/src/elf_parser/mod.rs#L108
101 : - Executable::validate https://github.com/solana-labs/rbpf/blob/v0.8.0/src/elf.rs#L518
102 : These two sections are executed in close proximity, with no modifications to the header in between.
103 : We can therefore consolidate the checks in one place.
104 : */
105 93 : & ( ehdr->e_ident[ FD_ELF_EI_CLASS ]==FD_ELF_CLASS_64 )
106 93 : & ( ehdr->e_ident[ FD_ELF_EI_DATA ]==FD_ELF_DATA_LE )
107 93 : & ( ehdr->e_ident[ FD_ELF_EI_VERSION ]==1 )
108 93 : & ( ehdr->e_ident[ FD_ELF_EI_OSABI ]==FD_ELF_OSABI_NONE )
109 93 : & ( ehdr->e_type ==FD_ELF_ET_DYN )
110 93 : & ( ( ehdr->e_machine ==FD_ELF_EM_BPF )
111 93 : | ( ehdr->e_machine ==FD_ELF_EM_SBPF ) )
112 93 : & ( ehdr->e_version ==1 )
113 : /* Coherence checks */
114 93 : & ( ehdr->e_ehsize ==sizeof(fd_elf64_ehdr) )
115 93 : & ( ehdr->e_phentsize==sizeof(fd_elf64_phdr) )
116 93 : & ( ehdr->e_shentsize==sizeof(fd_elf64_shdr) )
117 93 : & ( ehdr->e_shstrndx < ehdr->e_shnum )
118 93 : & ( ehdr->e_flags >= min_version )
119 93 : & ( max_version
120 93 : ? ( ehdr->e_flags <= max_version )
121 93 : : ( ehdr->e_flags != FD_ELF_EF_SBPF_V2 )
122 93 : )
123 93 : );
124 :
125 : /* Bounds check program header table */
126 :
127 51 : ulong const phoff = ehdr->e_phoff;
128 51 : ulong const phnum = ehdr->e_phnum;
129 51 : REQUIRE( ( fd_ulong_is_aligned( phoff, 8UL ) )
130 51 : & ( phoff<=elf_sz ) ); /* out of bounds */
131 :
132 51 : REQUIRE( phnum<=(ULONG_MAX/sizeof(fd_elf64_phdr)) ); /* overflow */
133 51 : ulong const phsz = phnum*sizeof(fd_elf64_phdr);
134 :
135 51 : ulong const phoff_end = phoff+phsz;
136 51 : REQUIRE( ( phoff_end>=phoff ) /* overflow */
137 51 : & ( phoff_end<=elf_sz ) /* out of bounds */
138 51 : & ( (phoff_end==0UL) /* overlaps file header */
139 51 : | (phoff>=sizeof(fd_elf64_ehdr)) ) );
140 :
141 : /* Bounds check section header table */
142 :
143 51 : ulong const shoff = ehdr->e_shoff;
144 51 : ulong const shnum = ehdr->e_shnum;
145 51 : REQUIRE( ( fd_ulong_is_aligned( shoff, 8UL ) )
146 51 : & ( shoff>=sizeof(fd_elf64_ehdr) ) /* overlaps file header */
147 51 : & ( shoff< elf_sz ) /* out of bounds */
148 51 : & ( shnum> 0UL ) ); /* not enough sections */
149 :
150 51 : REQUIRE( shoff<=(ULONG_MAX/sizeof(fd_elf64_shdr)) ); /* overflow */
151 51 : ulong const shsz = shnum*sizeof(fd_elf64_shdr);
152 :
153 51 : ulong const shoff_end = shoff+shsz;
154 51 : REQUIRE( ( shoff_end>=shoff ) /* overflow */
155 51 : & ( shoff_end<=elf_sz ) ); /* out of bounds */
156 :
157 : /* Overlap checks */
158 :
159 51 : REQUIRE( (phoff>=shoff_end) | (shoff>=phoff_end) ); /* overlap shdrs<>phdrs */
160 :
161 51 : return 0;
162 51 : }
163 :
164 : /* shdr_get_loaded_size returns the loaded size of a section, i.e. the
165 : number of bytes loaded into the rodata segment. sBPF ELFs grossly
166 : misuse the sh_size parameter. When SHT_NOBITS is set, the actual
167 : section size is zero, and the section size is ignored. */
168 :
169 : static ulong
170 351 : shdr_get_loaded_size( fd_elf64_shdr const * shdr ) {
171 351 : return fd_ulong_if( shdr->sh_type==FD_ELF_SHT_NOBITS, 0UL, shdr->sh_size );
172 351 : }
173 :
174 : /* check_cstr verifies a string in a string table. Returns non-NULL if
175 : the string is null terminated and contains at most max non-NULL
176 : characters. Returns NULL if the off is out of bounds, or if the max
177 : or EOF are reached before the null terminator. */
178 :
179 : static char const *
180 : check_cstr( uchar const * bin,
181 : ulong bin_sz,
182 : ulong off,
183 : ulong max,
184 681 : ulong * opt_sz ) {
185 681 : if( FD_UNLIKELY( off>=bin_sz ) ) return NULL;
186 681 : max += 1UL; /* include NULL terminator */
187 681 : max = fd_ulong_min( max, bin_sz-off ); /* truncate to available size */
188 681 : char const * cstr = (char const *)( bin+off );
189 681 : ulong len = strnlen( cstr, max );
190 681 : if( opt_sz ) *opt_sz = len;
191 681 : return len<max ? cstr : NULL;
192 681 : }
193 :
194 : /* fd_sbpf_load_phdrs walks the program header table. Remembers info
195 : along the way, and performs various validations.
196 :
197 : Assumes that ...
198 : - table does not overlap with file header or section header table and
199 : is within bounds
200 : - offset of program header table is 8 byte aligned */
201 :
202 : static int
203 : fd_sbpf_load_phdrs( fd_sbpf_elf_info_t * info,
204 : fd_sbpf_elf_t const * elf,
205 51 : ulong elf_sz ) {
206 :
207 51 : ulong const pht_offset = elf->ehdr.e_phoff;
208 51 : ulong const pht_cnt = elf->ehdr.e_phnum;
209 :
210 : /* Virtual address of last seen program header */
211 51 : ulong p_load_vaddr = 0UL;
212 :
213 : /* Read program header table */
214 51 : fd_elf64_phdr const * phdr = (fd_elf64_phdr const *)( elf->bin + pht_offset );
215 228 : for( ulong i=0; i<pht_cnt; i++ ) {
216 177 : switch( phdr[i].p_type ) {
217 51 : case FD_ELF_PT_DYNAMIC:
218 : /* Remember first PT_DYNAMIC segment */
219 51 : _fd_int_store_if_negative( &info->phndx_dyn, (int)i );
220 51 : break;
221 126 : case FD_ELF_PT_LOAD:
222 : /* LOAD segments must be ordered */
223 126 : REQUIRE( phdr[ i ].p_vaddr >= p_load_vaddr );
224 126 : p_load_vaddr = fd_ulong_sat_add( phdr[ i ].p_offset, phdr[ i ].p_filesz );
225 : /* Segment must be within bounds */
226 126 : REQUIRE( ( phdr[ i ].p_offset + phdr[ i ].p_filesz >= phdr[ i ].p_offset )
227 126 : & ( phdr[ i ].p_offset + phdr[ i ].p_filesz <= elf_sz ) );
228 : /* No overlap checks */
229 126 : break;
230 126 : default:
231 : /* Ignore other segment types */
232 0 : break;
233 177 : }
234 177 : }
235 :
236 51 : return 0;
237 51 : }
238 :
239 : /* FD_SBPF_SECTION_NAME_SZ_MAX is the maximum length of a symbol name cstr
240 : including zero terminator.
241 : https://github.com/solana-labs/rbpf/blob/c168a8715da668a71584ea46696d85f25c8918f6/src/elf_parser/mod.rs#L12 */
242 1011 : #define FD_SBPF_SECTION_NAME_SZ_MAX (16UL)
243 :
244 : /* fd_sbpf_load_shdrs walks the section header table. Remembers info
245 : along the way, and performs various validations.
246 :
247 : Assumes that ...
248 : - table does not overlap with file header or program header table and
249 : is within bounds
250 : - offset of section header table is 8 byte aligned
251 : - section header table has at least one entry */
252 :
253 : static int
254 : fd_sbpf_load_shdrs( fd_sbpf_elf_info_t * info,
255 : fd_sbpf_elf_t const * elf,
256 : ulong elf_sz,
257 51 : int elf_deploy_checks ) {
258 :
259 : /* File Header */
260 51 : ulong const eh_offset = 0UL;
261 51 : ulong const eh_offend = sizeof(fd_elf64_ehdr);
262 :
263 : /* Section Header Table */
264 51 : ulong const sht_offset = elf->ehdr.e_shoff;
265 51 : ulong const sht_cnt = elf->ehdr.e_shnum;
266 51 : ulong const sht_sz = sht_cnt*sizeof(fd_elf64_shdr);
267 51 : ulong const sht_offend = sht_offset + sht_sz;
268 :
269 51 : fd_elf64_shdr const * shdr = (fd_elf64_shdr const *)( elf->bin + sht_offset );
270 :
271 : /* Program Header Table */
272 51 : ulong const pht_offset = elf->ehdr.e_phoff;
273 51 : ulong const pht_cnt = elf->ehdr.e_phnum;
274 51 : ulong const pht_offend = pht_offset + (pht_cnt*sizeof(fd_elf64_phdr));
275 :
276 : /* Overlap checks */
277 51 : REQUIRE( (sht_offset>=eh_offend ) | (sht_offend<=eh_offset ) ); /* overlaps ELF file header */
278 51 : REQUIRE( (sht_offset>=pht_offend) | (sht_offend<=pht_offset) ); /* overlaps program header table */
279 :
280 : /* Require SHT_STRTAB for section name table */
281 :
282 51 : REQUIRE( elf->ehdr.e_shstrndx < sht_cnt ); /* out of bounds */
283 51 : REQUIRE( shdr[ elf->ehdr.e_shstrndx ].sh_type==FD_ELF_SHT_STRTAB );
284 :
285 51 : ulong shstr_off = shdr[ elf->ehdr.e_shstrndx ].sh_offset;
286 51 : ulong shstr_sz = shdr[ elf->ehdr.e_shstrndx ].sh_size;
287 51 : REQUIRE( shstr_off<elf_sz );
288 51 : shstr_sz = fd_ulong_min( shstr_sz, elf_sz-shstr_off );
289 :
290 : /* Clear the "loaded sections" bitmap */
291 :
292 51 : fd_memset( info->loaded_sections, 0, sizeof(info->loaded_sections) );
293 :
294 : /* Validate section header table.
295 : Check that all sections are in bounds, ordered, and don't overlap. */
296 :
297 51 : ulong min_sh_offset = 0UL; /* lowest permitted section offset */
298 :
299 : /* Keep track of the physical (file address) end of all relevant
300 : sections to determine rodata_sz */
301 51 : ulong psegment_end = 0UL; /* Upper bound of physical (file) addressing */
302 :
303 : /* While validating section header table, also figure out size
304 : of the rodata segment. This is the minimal virtual address range
305 : that spans all sections. */
306 51 : ulong vsegment_start = FD_SBPF_MM_PROGRAM_ADDR; /* Lower bound of segment virtual address */
307 51 : ulong vsegment_end = 0UL; /* Upper bound of segment virtual address */
308 :
309 51 : ulong tot_section_sz = 0UL; /* Size of all sections */
310 :
311 555 : for( ulong i=0UL; i<sht_cnt; i++ ) {
312 507 : uint sh_type = shdr[ i ].sh_type;
313 507 : uint sh_name = shdr[ i ].sh_name;
314 507 : ulong sh_addr = shdr[ i ].sh_addr;
315 507 : ulong sh_offset = shdr[ i ].sh_offset;
316 507 : ulong sh_size = shdr[ i ].sh_size;
317 507 : ulong sh_offend = sh_offset + sh_size;
318 :
319 : /* First section must be SHT_NULL */
320 507 : REQUIRE( i>0UL || sh_type==FD_ELF_SHT_NULL );
321 :
322 : /* check that physical range has no overflow and is within bounds */
323 507 : REQUIRE( sh_offend >= sh_offset );
324 507 : REQUIRE( sh_offend <= elf_sz ); // https://github.com/solana-labs/rbpf/blob/v0.8.0/src/elf_parser/mod.rs#L180
325 :
326 507 : if( sh_type!=FD_ELF_SHT_NOBITS ) {
327 : /* Overlap checks */
328 504 : REQUIRE( (sh_offset>=eh_offend ) | (sh_offend<=eh_offset ) ); /* overlaps ELF file header */
329 504 : REQUIRE( (sh_offset>=pht_offend) | (sh_offend<=pht_offset) ); /* overlaps program header table */
330 504 : REQUIRE( (sh_offset>=sht_offend) | (sh_offend<=sht_offset) ); /* overlaps section header table */
331 :
332 : /* Ordering and overlap check
333 : https://github.com/solana-labs/rbpf/blob/v0.8.0/src/elf_parser/mod.rs#L177
334 : */
335 504 : REQUIRE( sh_offset >= min_sh_offset );
336 504 : min_sh_offset = sh_offend;
337 504 : }
338 :
339 507 : if( sh_type==FD_ELF_SHT_DYNAMIC ) {
340 : /* Remember first SHT_DYNAMIC segment */
341 51 : _fd_int_store_if_negative( &info->shndx_dyn, (int)i );
342 51 : }
343 :
344 507 : ulong name_off = shstr_off + (ulong)sh_name;
345 507 : REQUIRE( ( name_off<elf_sz ) /* out of bounds */
346 507 : & ( sh_name <shstr_sz ) );
347 :
348 : /* Create name cstr */
349 :
350 507 : char const * name_ptr = check_cstr( elf->bin + shstr_off, shstr_sz, sh_name, FD_SBPF_SECTION_NAME_SZ_MAX-1UL, NULL );
351 507 : REQUIRE( name_ptr );
352 504 : char __attribute__((aligned(16UL))) name[ FD_SBPF_SECTION_NAME_SZ_MAX ] = {0};
353 504 : strncpy( name, name_ptr, FD_SBPF_SECTION_NAME_SZ_MAX-1UL );
354 :
355 : /* Check name */
356 : /* TODO switch table for this? */
357 : /* TODO reject duplicate sections */
358 :
359 504 : int load = 0; /* should section be loaded? */
360 :
361 504 : /**/ if( 0==memcmp( name, ".text", 6UL /* equals */ ) ) {
362 51 : REQUIRE( (info->shndx_text)<0 ); /* check for duplicate */
363 51 : info->shndx_text = (int)i;
364 51 : load = 1;
365 51 : }
366 453 : else if( (0==memcmp( name, ".rodata", 8UL /* equals */ ) )
367 453 : | (0==memcmp( name, ".data.rel.ro", 13UL /* equals */ ) )
368 453 : | (0==memcmp( name, ".eh_frame", 10UL /* equals */ ) ) ) {
369 57 : load = 1;
370 57 : }
371 396 : else if( 0==memcmp( name, ".symtab", 8UL /* equals */ ) ) {
372 33 : REQUIRE( (info->shndx_symtab)<0 );
373 33 : info->shndx_symtab = (int)i;
374 33 : }
375 363 : else if( 0==memcmp( name, ".strtab", 8UL /* equals */ ) ) {
376 33 : REQUIRE( (info->shndx_strtab)<0 );
377 33 : info->shndx_strtab = (int)i;
378 33 : }
379 330 : else if( 0==memcmp( name, ".dynstr", 8UL /* equals */ ) ) {
380 48 : REQUIRE( (info->shndx_dynstr)<0 );
381 48 : info->shndx_dynstr = (int)i;
382 48 : }
383 282 : else if( 0==memcmp( name, ".bss", 4UL /* has prefix */ ) ) {
384 0 : FAIL();
385 0 : }
386 282 : else if( 0==memcmp( name, ".data.rel", 9UL /* has prefix */ ) ) {} /* ignore */
387 282 : else if( (0==memcmp( name, ".data", 5UL /* has prefix */ ) )
388 282 : & ( ( shdr[ i ].sh_flags & (FD_ELF_SHF_ALLOC|FD_ELF_SHF_WRITE) )
389 282 : ==(FD_ELF_SHF_ALLOC|FD_ELF_SHF_WRITE) ) ) {
390 0 : FAIL();
391 0 : }
392 282 : else {} /* ignore */
393 : /* else ignore */
394 :
395 504 : if( load ) {
396 : /* Remember that section should be loaded */
397 :
398 108 : info->loaded_sections[ i>>6UL ] |= (1UL)<<(i&63UL);
399 :
400 : /* Check that virtual address range is in MM_PROGRAM bounds */
401 :
402 108 : ulong sh_actual_size = shdr_get_loaded_size( &shdr[ i ] );
403 108 : ulong sh_virtual_end = sh_addr + sh_actual_size;
404 :
405 : /* https://github.com/solana-labs/rbpf/blob/v0.8.0/src/elf.rs#L426 */
406 108 : if ( FD_UNLIKELY( elf_deploy_checks ) ){
407 54 : REQUIRE( sh_addr == sh_offset );
408 54 : }
409 108 : REQUIRE( sh_addr < FD_SBPF_MM_PROGRAM_ADDR ); /* overflow check */
410 108 : REQUIRE( sh_actual_size < FD_SBPF_MM_PROGRAM_ADDR ); /* overflow check */
411 108 : REQUIRE( sh_virtual_end <= FD_SBPF_MM_STACK_ADDR-FD_SBPF_MM_PROGRAM_ADDR ); /* check overlap with stack */
412 :
413 : /* Check that physical address range is in bounds
414 : (Seems redundant?) */
415 108 : ulong paddr_end = sh_offset + sh_actual_size;
416 108 : REQUIRE( paddr_end >= sh_offset );
417 108 : REQUIRE( paddr_end <= elf_sz );
418 :
419 108 : vsegment_start = fd_ulong_min( vsegment_start, sh_addr );
420 : /* Expand range to fit section */
421 108 : psegment_end = fd_ulong_max( psegment_end, paddr_end );
422 108 : vsegment_end = fd_ulong_max( vsegment_end, sh_virtual_end );
423 :
424 : /* Coherence check sum of section sizes */
425 108 : REQUIRE( tot_section_sz + sh_actual_size >= tot_section_sz ); /* overflow check */
426 108 : tot_section_sz += sh_actual_size;
427 108 : }
428 504 : }
429 :
430 : /* More coherence checks */
431 48 : REQUIRE( psegment_end <= elf_sz ); // https://github.com/solana-labs/rbpf/blob/v0.8.0/src/elf.rs#L782
432 :
433 :
434 : /* Check that the rodata segment is within bounds
435 : https://github.com/solana-labs/rbpf/blob/v0.8.0/src/elf.rs#L725 */
436 48 : if ( FD_UNLIKELY( elf_deploy_checks ) ){
437 18 : REQUIRE( fd_ulong_sat_add( vsegment_start, tot_section_sz) <= vsegment_end );
438 18 : }
439 :
440 : /* Require .text section */
441 :
442 48 : REQUIRE( (info->shndx_text)>=0 );
443 48 : fd_elf64_shdr const * shdr_text = &shdr[ info->shndx_text ];
444 48 : REQUIRE( (shdr_text->sh_addr <= elf->ehdr.e_entry)
445 : /* check that entrypoint is in text VM range */
446 48 : & (elf->ehdr.e_entry < fd_ulong_sat_add( shdr_text->sh_addr, shdr_text->sh_size ) ) );
447 : /* NOTE: Does NOT check that the entrypoint is in text section file
448 : range (which may be 0 sz if SHT_NOBITS). This check is
449 : separately done in the sBPF verifier. */
450 :
451 48 : info->text_off = (uint)shdr_text->sh_offset;
452 48 : ulong text_size = shdr_get_loaded_size( shdr_text );
453 48 : info->text_sz = text_size;
454 48 : info->text_cnt = (uint) text_size / 8U;
455 :
456 :
457 : /* Convert entrypoint offset to program counter */
458 :
459 48 : info->rodata_sz = (uint)psegment_end;
460 48 : info->rodata_footprint = (uint)elf_sz;
461 :
462 48 : ulong entry_off = fd_ulong_sat_sub( elf->ehdr.e_entry, shdr_text->sh_addr );
463 48 : ulong entry_pc = entry_off / 8UL;
464 :
465 : /* Follows https://github.com/solana-labs/rbpf/blob/v0.8.0/src/elf.rs#L443 */
466 48 : REQUIRE( fd_ulong_is_aligned( entry_off, 8UL ) );
467 48 : REQUIRE( entry_pc < ( info->rodata_sz / 8UL ) );
468 48 : info->entry_pc = (uint)entry_pc;
469 :
470 48 : if( (info->shndx_dynstr)>=0 ) {
471 48 : fd_elf64_shdr const * shdr_dynstr = &shdr[ info->shndx_dynstr ];
472 48 : ulong sh_offset = shdr_dynstr->sh_offset;
473 48 : ulong sh_size = shdr_dynstr->sh_size;
474 48 : REQUIRE( (sh_offset+sh_size>=sh_offset) & (sh_offset+sh_size<=info->rodata_footprint) );
475 48 : info->dynstr_off = (uint)sh_offset;
476 48 : info->dynstr_sz = (uint)sh_size;
477 48 : }
478 :
479 48 : return 0;
480 48 : }
481 :
482 : fd_sbpf_elf_info_t *
483 : fd_sbpf_elf_peek( fd_sbpf_elf_info_t * info,
484 : void const * bin,
485 : ulong elf_sz,
486 : int elf_deploy_checks,
487 : uint sbpf_min_version,
488 93 : uint sbpf_max_version ) {
489 :
490 : /* ELFs must have a file header */
491 93 : if( FD_UNLIKELY( elf_sz<=sizeof(fd_elf64_ehdr) ) )
492 0 : return NULL;
493 :
494 : /* Reject overlong ELFs (using uint addressing internally).
495 : This is well beyond Solana's max account size of 10 MB. */
496 93 : if( FD_UNLIKELY( elf_sz>UINT_MAX ) )
497 0 : return NULL;
498 :
499 : /* Initialize info struct */
500 93 : *info = (fd_sbpf_elf_info_t) {
501 93 : .text_off = 0U,
502 93 : .text_cnt = 0U,
503 93 : .dynstr_off = 0U,
504 93 : .dynstr_sz = 0U,
505 93 : .rodata_footprint = 0U,
506 93 : .rodata_sz = 0U,
507 93 : .shndx_text = -1,
508 93 : .shndx_symtab = -1,
509 93 : .shndx_strtab = -1,
510 93 : .shndx_dyn = -1,
511 93 : .shndx_dynstr = -1,
512 93 : .phndx_dyn = -1,
513 93 : .sbpf_version = 0U,
514 : /* !!! Keep this in sync with -Werror=missing-field-initializers */
515 93 : };
516 :
517 93 : fd_sbpf_elf_t const * elf = (fd_sbpf_elf_t const *)bin;
518 93 : int err;
519 :
520 : /* Validate file header */
521 93 : if( FD_UNLIKELY( (err=fd_sbpf_check_ehdr( &elf->ehdr, elf_sz, sbpf_min_version, sbpf_max_version ))!=0 ) )
522 42 : return NULL;
523 :
524 : /* Program headers */
525 51 : if( FD_UNLIKELY( (err=fd_sbpf_load_phdrs( info, elf, elf_sz ))!=0 ) )
526 0 : return NULL;
527 :
528 : /* Section headers */
529 51 : if( FD_UNLIKELY( (err=fd_sbpf_load_shdrs( info, elf, elf_sz, elf_deploy_checks ))!=0 ) )
530 3 : return NULL;
531 :
532 : /* Set SBPF version from ELF e_flags */
533 48 : info->sbpf_version = sbpf_max_version ? elf->ehdr.e_flags : 0UL;
534 :
535 48 : return info;
536 51 : }
537 :
538 : /* ELF loader, part 2 **************************************************
539 :
540 : Prepare a copy of a subrange of the ELF content: The rodata segment.
541 : Mangle the copy by applying dynamic relocations. Then, zero out
542 : parts of the segment that are not interesting to the loader.
543 :
544 : ### Terminology
545 :
546 : Shorthands for relocation handling:
547 :
548 : S: Symbol value (typically an ELF physical address)
549 : A: Implicit addend, i.e. the original value of the field that the
550 : relocation handler is about to write to
551 : V: Virtual address, i.e. the target value that the relocation
552 : handler is about to write into where the implicit addend was
553 : previously stored */
554 :
555 : ulong
556 33 : fd_sbpf_program_align( void ) {
557 33 : return alignof( fd_sbpf_program_t );
558 33 : }
559 :
560 : ulong
561 33 : fd_sbpf_program_footprint( fd_sbpf_elf_info_t const * info ) {
562 33 : FD_COMPILER_UNPREDICTABLE( info ); /* Make this appear as FD_FN_PURE (e.g. footprint might depened on info contents in future) */
563 33 : return FD_LAYOUT_FINI( FD_LAYOUT_APPEND( FD_LAYOUT_APPEND( FD_LAYOUT_INIT,
564 33 : alignof(fd_sbpf_program_t), sizeof(fd_sbpf_program_t) ),
565 33 : fd_sbpf_calldests_align(), fd_sbpf_calldests_footprint( info->rodata_sz / 8UL ) ), /* calldests bitmap */
566 33 : alignof(fd_sbpf_program_t) );
567 33 : }
568 :
569 : fd_sbpf_program_t *
570 : fd_sbpf_program_new( void * prog_mem,
571 : fd_sbpf_elf_info_t const * elf_info,
572 33 : void * rodata ) {
573 :
574 33 : if( FD_UNLIKELY( !prog_mem ) ) {
575 0 : FD_LOG_WARNING(( "NULL prog_mem" ));
576 0 : return NULL;
577 0 : }
578 :
579 33 : if( FD_UNLIKELY( !elf_info ) ) {
580 0 : FD_LOG_WARNING(( "NULL elf_info" ));
581 0 : return NULL;
582 0 : }
583 :
584 33 : if( FD_UNLIKELY( ((elf_info->rodata_footprint)>0U) & (!rodata)) ) {
585 0 : FD_LOG_WARNING(( "NULL rodata" ));
586 0 : return NULL;
587 0 : }
588 :
589 : /* https://github.com/solana-labs/rbpf/blob/v0.8.0/src/elf_parser/mod.rs#L99 */
590 33 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong) rodata, FD_SBPF_PROG_RODATA_ALIGN ) ) ){
591 0 : FD_LOG_WARNING(( "rodata is not 8-byte aligned" ));
592 0 : return NULL;
593 0 : }
594 :
595 : /* Initialize program struct */
596 :
597 33 : FD_SCRATCH_ALLOC_INIT( laddr, prog_mem );
598 33 : fd_sbpf_program_t * prog = FD_SCRATCH_ALLOC_APPEND( laddr, alignof(fd_sbpf_program_t), sizeof(fd_sbpf_program_t) );
599 :
600 0 : *prog = (fd_sbpf_program_t) {
601 33 : .info = *elf_info,
602 33 : .rodata = rodata,
603 33 : .rodata_sz = elf_info->rodata_sz,
604 33 : .text = (ulong *)((ulong)rodata + elf_info->text_off), /* FIXME: WHAT IF MISALIGNED */
605 33 : .text_off = elf_info->text_off,
606 33 : .text_cnt = elf_info->text_cnt,
607 33 : .text_sz = elf_info->text_sz,
608 33 : .entry_pc = elf_info->entry_pc
609 33 : };
610 :
611 : /* Initialize calldests map */
612 :
613 33 : ulong pc_max = elf_info->rodata_sz / 8UL;
614 33 : prog->calldests_shmem = fd_sbpf_calldests_new(
615 33 : FD_SCRATCH_ALLOC_APPEND( laddr, fd_sbpf_calldests_align(),
616 33 : fd_sbpf_calldests_footprint( pc_max ) ),
617 0 : pc_max );
618 0 : prog->calldests = fd_sbpf_calldests_join( prog->calldests_shmem );
619 :
620 33 : return prog;
621 33 : }
622 :
623 : void *
624 30 : fd_sbpf_program_delete( fd_sbpf_program_t * mem ) {
625 :
626 30 : fd_sbpf_calldests_delete( fd_sbpf_calldests_leave( mem->calldests ) );
627 30 : fd_memset( mem, 0, sizeof(fd_sbpf_program_t) );
628 :
629 30 : return (void *)mem;
630 30 : }
631 :
632 : /* fd_sbpf_loader_t contains various temporary state during loading */
633 :
634 : struct fd_sbpf_loader {
635 : /* External objects */
636 : ulong * calldests; /* owned by program */
637 : fd_sbpf_syscalls_t * syscalls; /* owned by caller */
638 :
639 : /* Dynamic table */
640 : uint dyn_off; /* File offset of dynamic table (UINT_MAX=missing) */
641 : uint dyn_cnt; /* Number of dynamic table entries */
642 :
643 : /* Dynamic table entries */
644 : ulong dt_rel;
645 : ulong dt_relent;
646 : ulong dt_relsz;
647 : ulong dt_symtab;
648 :
649 : /* Dynamic symbols */
650 : uint dynsym_off; /* File offset of .dynsym section (0=missing) */
651 : uint dynsym_cnt; /* Symbol count */
652 :
653 : int elf_deploy_checks;
654 : };
655 : typedef struct fd_sbpf_loader fd_sbpf_loader_t;
656 :
657 : /* FD_SBPF_SYM_NAME_SZ_MAX is the maximum length of a symbol name cstr
658 : including zero terminator.
659 : https://github.com/solana-labs/rbpf/blob/c168a8715da668a71584ea46696d85f25c8918f6/src/elf_parser/mod.rs#L13 */
660 174 : #define FD_SBPF_SYM_NAME_SZ_MAX (64UL)
661 :
662 :
663 : static int
664 : fd_sbpf_find_dynamic( fd_sbpf_loader_t * loader,
665 : fd_sbpf_elf_t const * elf,
666 : ulong elf_sz,
667 33 : fd_sbpf_elf_info_t const * info ) {
668 :
669 33 : fd_elf64_shdr const * shdrs = (fd_elf64_shdr const *)( elf->bin + elf->ehdr.e_shoff );
670 33 : fd_elf64_phdr const * phdrs = (fd_elf64_phdr const *)( elf->bin + elf->ehdr.e_phoff );
671 :
672 : /* Try first PT_DYNAMIC in program header table */
673 :
674 33 : if( (info->phndx_dyn)>=0 ) {
675 33 : ulong dyn_off = phdrs[ info->phndx_dyn ].p_offset;
676 33 : ulong dyn_sz = phdrs[ info->phndx_dyn ].p_filesz;
677 33 : ulong dyn_end = dyn_off+dyn_sz;
678 :
679 : /* Fall through to SHT_DYNAMIC if invalid */
680 :
681 33 : if( FD_LIKELY( ( dyn_end>=dyn_off ) /* overflow */
682 33 : & ( dyn_end<=elf_sz ) /* out of bounds */
683 33 : & fd_ulong_is_aligned( dyn_off, 8UL ) /* misaligned */
684 33 : & fd_ulong_is_aligned( dyn_sz, sizeof(fd_elf64_dyn) ) /* misaligned sz */ ) ) {
685 33 : loader->dyn_off = (uint)dyn_off;
686 33 : loader->dyn_cnt = (uint)(dyn_sz / sizeof(fd_elf64_dyn));
687 33 : return 0;
688 33 : }
689 33 : }
690 :
691 : /* Try first SHT_DYNAMIC in section header table */
692 :
693 0 : if( (info->shndx_dyn)>0 ) {
694 0 : ulong dyn_off = shdrs[ info->shndx_dyn ].sh_offset;
695 0 : ulong dyn_sz = shdrs[ info->shndx_dyn ].sh_size;
696 0 : ulong dyn_end = dyn_off+dyn_sz;
697 :
698 : /* This time, don't tolerate errors */
699 :
700 0 : REQUIRE( ( dyn_end>=dyn_off ) /* overflow */
701 0 : & ( dyn_end<=elf_sz ) /* out of bounds */
702 0 : & fd_ulong_is_aligned( dyn_off, 8UL ) /* misaligned */
703 0 : & fd_ulong_is_aligned( dyn_sz, sizeof(fd_elf64_dyn) ) /* misaligned sz */ );
704 :
705 0 : loader->dyn_off = (uint)dyn_off;
706 0 : loader->dyn_cnt = (uint)(dyn_sz / sizeof(fd_elf64_dyn));
707 0 : return 0;
708 0 : }
709 :
710 : /* Missing or invalid PT_DYNAMIC and missing SHT_DYNAMIC, skip. */
711 0 : return 0;
712 0 : }
713 :
714 : static int
715 : fd_sbpf_load_dynamic( fd_sbpf_loader_t * loader,
716 : fd_sbpf_elf_t const * elf,
717 33 : ulong elf_sz ) {
718 :
719 33 : fd_elf64_shdr const * shdrs = (fd_elf64_shdr const *)( elf->bin + elf->ehdr.e_shoff );
720 :
721 : /* Skip if no dynamic table was found */
722 :
723 33 : if( !loader->dyn_cnt ) return 0;
724 :
725 : /* Walk dynamic table */
726 :
727 33 : fd_elf64_dyn const * dyn = (fd_elf64_dyn const *)( elf->bin + loader->dyn_off );
728 33 : ulong const dyn_cnt = loader->dyn_cnt;
729 :
730 318 : for( ulong i=0; i<dyn_cnt; i++ ) {
731 318 : if( FD_UNLIKELY( dyn[i].d_tag==FD_ELF_DT_NULL ) ) break;
732 :
733 285 : ulong d_val = dyn[i].d_un.d_val;
734 285 : switch( dyn[i].d_tag ) {
735 18 : case FD_ELF_DT_REL: loader->dt_rel =d_val; break;
736 18 : case FD_ELF_DT_RELENT: loader->dt_relent=d_val; break;
737 18 : case FD_ELF_DT_RELSZ: loader->dt_relsz =d_val; break;
738 33 : case FD_ELF_DT_SYMTAB: loader->dt_symtab=d_val; break;
739 285 : }
740 285 : }
741 :
742 : /* Load dynamic symbol table */
743 :
744 33 : if( loader->dt_symtab ) {
745 : /* Search for dynamic symbol table
746 : FIXME unfortunate bounded O(n^2) -- could convert to binary search */
747 :
748 : /* FIXME this could be clobbered by relocations, causing strict
749 : aliasing violations */
750 :
751 33 : fd_elf64_shdr const * shdr_dynsym = NULL;
752 :
753 138 : for( ulong i=0; i<elf->ehdr.e_shnum; i++ ) {
754 138 : if( shdrs[ i ].sh_addr == loader->dt_symtab ) {
755 : /* TODO: verify this ... */
756 : /* Check section type */
757 33 : uint sh_type = shdrs[ i ].sh_type;
758 : // https://github.com/solana-labs/rbpf/blob/v0.8.5/src/elf_parser/mod.rs#L500
759 33 : REQUIRE( (sh_type==FD_ELF_SHT_SYMTAB) | (sh_type==FD_ELF_SHT_DYNSYM) );
760 :
761 33 : shdr_dynsym = &shdrs[ i ];
762 33 : break;
763 33 : }
764 138 : }
765 33 : REQUIRE( shdr_dynsym );
766 :
767 : /* Check if out of bounds or misaligned */
768 :
769 33 : ulong sh_offset = shdr_dynsym->sh_offset;
770 33 : ulong sh_size = shdr_dynsym->sh_size;
771 :
772 33 : REQUIRE( ( sh_offset+sh_size>=sh_offset )
773 33 : & ( sh_offset+sh_size<=elf_sz )
774 33 : & ( fd_ulong_is_aligned( sh_offset, 8UL ) )
775 33 : & ( sh_size % sizeof (fd_elf64_sym) == 0UL ) );
776 :
777 33 : loader->dynsym_off = (uint)sh_offset;
778 33 : loader->dynsym_cnt = (uint)(sh_size/sizeof(fd_elf64_sym));
779 33 : }
780 :
781 33 : return 0;
782 33 : }
783 :
784 : /* ELF Dynamic Relocations *********************************************
785 :
786 : ### Summary
787 :
788 : The sBPF ELF loader provides a limited dynamic relocation mechanism
789 : to fix up Clang-generated shared objects for execution in an sBPF VM.
790 :
791 : The relocation types themselves violate the eBPF and ELF specs in
792 : various ways. In short, the relocation table (via DT_REL) is used to
793 : shift program code from zero-based addressing to the MM_PROGRAM
794 : segment in the VM memory map (at 0x1_0000_0000).
795 :
796 : As part of the Solana VM protocol it abides by strict determinism
797 : requirements. This sadly means that we will have to replicate all
798 : edge cases and bugs in the Solana Labs ELF loader.
799 :
800 : Three relocation types are currently supported:
801 : - R_BPF_64_64: Sets an absolute address of a symbol as the
802 : 64-bit immediate field of an lddw instruction
803 : - R_BPF_64_RELATIVE: Adds MM_PROGRAM_START (0x1_0000_0000) to ...
804 : a) ... the 64-bit imm field of an lddw instruction (if in text)
805 : b) ... a 64-bit integer (if not in text section)
806 : - R_BPF_64_32: Sets the 32-bit immediate field of a call
807 : instruction to ...
808 : a) the ID of a local function (Murmur3 hash of function PC address)
809 : b) the ID of a syscall
810 :
811 : Obviously invalid relocations (e.g. out-of-bounds of ELF file or
812 : unsupported reloc type) raise an error.
813 : Relocations that would corrupt ELF data structures are silently
814 : ignored (using the fd_sbpf_reloc_mask mechanism).
815 :
816 : ### History
817 :
818 : The use of relocations is technically redundant, as the Solana VM
819 : memory map has been hardcoded in program runtime v1 (so far the only
820 : runtime). However, virtually all deployed programs as of April 2023
821 : are position-independent shared objects and make heavy use of such
822 : relocations.
823 :
824 : Relocations in the Solana VM have a complicated history. Over the
825 : course of years, multiple protocol bugs have been added and fixed.
826 : The ELF loader needs to handle all these edge cases to avoid breaking
827 : "userspace". I.e. any deployed programs which might be immutable
828 : must continue to function.
829 :
830 : While this complex logic will probably stick around for the next few
831 : years, the Solana protocol is getting increasingly restrictive for
832 : newly deployed ELFs. Another proposed change is upgrading to
833 : position-dependent binaries without any dynamic relocations. */
834 :
835 : /* R_BPF_64_64 relocates an absolute address into the extended imm field
836 : of an lddw-form instruction. (Two instruction slots, low 32 bits in
837 : first immediate field, high 32 bits in second immediate field)
838 :
839 : Bits 0..32 32..64 64..96 96..128
840 : [ ... ] [ IMM_LO ] [ ... ] [ IMM_HI ] */
841 :
842 : static int
843 : fd_sbpf_r_bpf_64_64( fd_sbpf_loader_t const * loader,
844 : fd_sbpf_elf_t const * elf,
845 : ulong elf_sz,
846 : uchar * rodata,
847 : fd_sbpf_elf_info_t const * info,
848 6 : fd_elf64_rel const * rel ) {
849 :
850 6 : (void)info;
851 :
852 6 : uint r_sym = FD_ELF64_R_SYM( rel->r_info );
853 6 : ulong r_offset = rel->r_offset;
854 :
855 : /* Bounds check */
856 6 : REQUIRE( ( r_offset+16UL> r_offset )
857 6 : & ( r_offset+16UL<=elf_sz ) );
858 :
859 : /* Offsets of implicit addend (immediate fields) */
860 6 : ulong A_off_lo = r_offset+ 4UL;
861 6 : ulong A_off_hi = r_offset+12UL;
862 :
863 : /* Read implicit addend (imm field of first insn slot) */
864 : // SBF_V2: ulong A_off = is_text ? r_offset+4UL : r_offset;
865 6 : REQUIRE( A_off_lo+4UL<elf_sz );
866 :
867 : /* Lookup symbol */
868 6 : REQUIRE( r_sym < loader->dynsym_cnt );
869 6 : fd_elf64_sym const * dynsyms = (fd_elf64_sym const *)( elf->bin + loader->dynsym_off );
870 6 : fd_elf64_sym const * sym = &dynsyms[ r_sym ];
871 6 : ulong S = sym->st_value;
872 :
873 : /* Relocate */
874 6 : ulong A = FD_LOAD( uint, &rodata[ A_off_lo ] );
875 6 : ulong V = fd_ulong_sat_add( S, A );
876 6 : if( V<FD_SBPF_MM_PROGRAM_ADDR ) V+=FD_SBPF_MM_PROGRAM_ADDR;
877 :
878 : /* Write back */
879 6 : FD_STORE( uint, &rodata[ A_off_lo ], (uint)(V ) );
880 6 : FD_STORE( uint, &rodata[ A_off_hi ], (uint)(V>>32UL) );
881 :
882 6 : return 0;
883 6 : }
884 :
885 : /* R_BPF_64_RELATIVE is almost entirely Solana specific. */
886 :
887 : static int
888 : fd_sbpf_r_bpf_64_relative( fd_sbpf_elf_t const * elf,
889 : ulong elf_sz,
890 : uchar * rodata,
891 : fd_sbpf_elf_info_t const * info,
892 162 : fd_elf64_rel const * rel ) {
893 :
894 162 : ulong r_offset = rel->r_offset;
895 :
896 : /* Is reloc target in .text section? */
897 162 : fd_elf64_shdr const * shdrs = (fd_elf64_shdr const *)( elf->bin + elf->ehdr.e_shoff );
898 162 : fd_elf64_shdr const * shdr_text = &shdrs[ info->shndx_text ];
899 162 : int is_text = ( ( r_offset >= shdr_text->sh_offset ) &
900 162 : ( r_offset < shdr_text->sh_offset +
901 162 : shdr_get_loaded_size( shdr_text ) ) );
902 :
903 162 : if( is_text ) {
904 : /* If reloc target is in .text, behave like R_BPF_64_64, except:
905 : - R_SYM(r_info) is ignored
906 : - If implicit addend looks like a physical address, make it
907 : a virtual address (by adding a constant offset)
908 :
909 : This relocation type seems to make little sense but is required
910 : for most programs. */
911 :
912 156 : REQUIRE( (r_offset+16UL>r_offset) & (r_offset+16UL<=elf_sz) );
913 156 : ulong imm_lo_off = r_offset+ 4UL;
914 156 : ulong imm_hi_off = r_offset+12UL;
915 :
916 : /* Read implicit addend */
917 156 : uint va_lo = FD_LOAD( uint, rodata+imm_lo_off );
918 156 : uint va_hi = FD_LOAD( uint, rodata+imm_hi_off );
919 156 : ulong va = ( (ulong)va_hi<<32UL ) | va_lo;
920 :
921 156 : REQUIRE( va!=0UL );
922 156 : va = va<FD_SBPF_MM_PROGRAM_ADDR ? va+FD_SBPF_MM_PROGRAM_ADDR : va;
923 :
924 : /* Write back
925 : Skip bounds check as .text is guaranteed to be writable */
926 156 : FD_STORE( uint, rodata+imm_lo_off, (uint)( va ) );
927 156 : FD_STORE( uint, rodata+imm_hi_off, (uint)( va>>32UL ) );
928 156 : } else {
929 : /* Outside .text do a 64-bit write */
930 :
931 : /* Bounds checks */
932 6 : REQUIRE( (r_offset+8UL>r_offset) & (r_offset+8UL<=elf_sz) );
933 :
934 : /* Read implicit addend */
935 6 : ulong va = FD_LOAD( uint, rodata+r_offset+4UL );
936 :
937 : /* Relocate */
938 6 : va = fd_ulong_sat_add( va, FD_SBPF_MM_PROGRAM_ADDR );
939 :
940 : /* Write back */
941 6 : FD_STORE( ulong, rodata+r_offset, va );
942 6 : }
943 :
944 162 : return 0;
945 162 : }
946 :
947 : static int
948 : fd_sbpf_r_bpf_64_32( fd_sbpf_loader_t const * loader,
949 : fd_sbpf_elf_t const * elf,
950 : ulong elf_sz,
951 : uchar * rodata,
952 : fd_sbpf_elf_info_t const * info,
953 174 : fd_elf64_rel const * rel ) {
954 :
955 174 : uint r_sym = FD_ELF64_R_SYM( rel->r_info );
956 174 : ulong r_offset = rel->r_offset;
957 :
958 : /* Lookup symbol */
959 174 : REQUIRE( r_sym < loader->dynsym_cnt );
960 174 : fd_elf64_shdr const * shdrs = (fd_elf64_shdr const *)( elf->bin + elf->ehdr.e_shoff );
961 174 : fd_elf64_sym const * dynsyms = (fd_elf64_sym const *)( elf->bin + loader->dynsym_off );
962 174 : fd_elf64_sym const * sym = &dynsyms[ r_sym ];
963 174 : ulong S = sym->st_value;
964 :
965 : /* Verify .dynstr (TODO can we lift this out of the reloc handler?) */
966 174 : REQUIRE( info->shndx_dynstr > 0 );
967 174 : REQUIRE( shdrs[ info->shndx_dynstr ].sh_type == FD_ELF_SHT_STRTAB );
968 :
969 : /* Verify symbol name */
970 174 : ulong name_len;
971 174 : char const * name = check_cstr( elf->bin + info->dynstr_off, info->dynstr_sz, sym->st_name, FD_SBPF_SYM_NAME_SZ_MAX-1UL, &name_len );
972 174 : REQUIRE( name );
973 :
974 : /* Value to write into relocated field */
975 174 : uint V;
976 :
977 174 : int is_func_call = ( FD_ELF64_ST_TYPE( sym->st_info ) == FD_ELF_STT_FUNC )
978 174 : & ( S!=0UL );
979 174 : if( is_func_call ) {
980 : /* Check whether function call is in virtual memory range of text section. */
981 105 : fd_elf64_shdr const * shdr_text = &shdrs[ info->shndx_text ];
982 105 : ulong sh_addr = shdr_text->sh_addr;
983 105 : ulong sh_size = shdr_text->sh_size;
984 105 : REQUIRE( (S>=sh_addr) & (S<sh_addr+sh_size) );
985 :
986 : /* Note: The above check is broken, as sh_size is interpreted as 0
987 : for SHT_NOBITS section. Yet, this is the "correct" loading
988 : behavior according to protocol rules. */
989 :
990 : /* Register function call */
991 105 : ulong target_pc = (S-sh_addr) / 8UL;
992 :
993 : /* TODO bounds check the target? */
994 :
995 : /* Register new entry */
996 105 : uint hash;
997 105 : if( name_len >= 10UL && 0==strncmp( name, "entrypoint", name_len ) ) {
998 : /* Skip insertion of "entrypoint" relocation entries to calldests. This
999 : emulates Solana/Agave's behavior of unregistering these entries before
1000 : registering the entrypoint manually.
1001 : Entrypoint is registered in fd_sbpf_program_load.
1002 : Hash is still applied. */
1003 9 : hash = 0x71e3cf81;
1004 96 : } else {
1005 96 : hash = fd_pchash( (uint)target_pc );
1006 96 : if( FD_LIKELY( target_pc < (info->rodata_sz / 8UL ) ) )
1007 96 : fd_sbpf_calldests_insert( loader->calldests, target_pc );
1008 96 : }
1009 :
1010 : /* Check for collision with syscall ID
1011 : https://github.com/solana-labs/rbpf/blob/57139e9e1fca4f01155f7d99bc55cdcc25b0bc04/src/program.rs#L142-L146 */
1012 105 : REQUIRE( !fd_sbpf_syscalls_query( loader->syscalls, hash, NULL ) );
1013 105 : V = (uint)hash;
1014 105 : } else {
1015 : /* FIXME Should cache Murmur hashes.
1016 : If max ELF size is 10MB, can fit about 640k relocs.
1017 : Each reloc could point to a symbol with the same st_name,
1018 : which results in 640MB hash input data without caching. */
1019 69 : uint hash = fd_murmur3_32( name, name_len, 0UL );
1020 : /* Ensure that requested syscall ID exists only when deploying
1021 : https://github.com/solana-labs/rbpf/blob/v0.8.0/src/elf.rs#L1097 */
1022 69 : if ( FD_UNLIKELY( loader->elf_deploy_checks ) ) {
1023 66 : REQUIRE( fd_sbpf_syscalls_query( loader->syscalls, hash, NULL ) );
1024 66 : }
1025 :
1026 69 : V = hash;
1027 69 : }
1028 :
1029 : /* Bounds checks */
1030 174 : REQUIRE( (r_offset+8UL>r_offset) & (r_offset+8UL<=elf_sz) );
1031 174 : ulong A_off = r_offset+4UL;
1032 :
1033 : /* Apply relocation */
1034 174 : FD_STORE( uint, rodata+A_off, V );
1035 :
1036 174 : return 0;
1037 174 : }
1038 :
1039 : static int
1040 : fd_sbpf_apply_reloc( fd_sbpf_loader_t const * loader,
1041 : fd_sbpf_elf_t const * elf,
1042 : ulong elf_sz,
1043 : uchar * rodata,
1044 : fd_sbpf_elf_info_t const * info,
1045 342 : fd_elf64_rel const * rel ) {
1046 342 : switch( FD_ELF64_R_TYPE( rel->r_info ) ) {
1047 6 : case FD_ELF_R_BPF_64_64:
1048 6 : return fd_sbpf_r_bpf_64_64 ( loader, elf, elf_sz, rodata, info, rel );
1049 162 : case FD_ELF_R_BPF_64_RELATIVE:
1050 162 : return fd_sbpf_r_bpf_64_relative( elf, elf_sz, rodata, info, rel );
1051 174 : case FD_ELF_R_BPF_64_32:
1052 174 : return fd_sbpf_r_bpf_64_32 ( loader, elf, elf_sz, rodata, info, rel );
1053 0 : default:
1054 0 : ERR( FD_SBPF_ERR_INVALID_ELF );
1055 342 : }
1056 342 : }
1057 :
1058 : /* fd_sbpf_hash_calls converts local call instructions in the "LLVM
1059 : form" (immediate is a program counter offset) to eBPF form (immediate
1060 : is a hash of the target program counter). Corresponds to
1061 : fixup_relative calls in solana-labs/rbpf.
1062 :
1063 : Assumes that the text section range exists, is within bounds, and
1064 : does not overlap with the ELF file header, program header table, or
1065 : section header table. */
1066 :
1067 : static int
1068 : fd_sbpf_hash_calls( fd_sbpf_loader_t * loader,
1069 : fd_sbpf_program_t * prog,
1070 33 : fd_sbpf_elf_t const * elf ) {
1071 :
1072 33 : fd_elf64_shdr const * shdrs = (fd_elf64_shdr const *)( elf->bin + elf->ehdr.e_shoff );
1073 33 : fd_sbpf_elf_info_t * info = &prog->info;
1074 33 : uchar * rodata = prog->rodata;
1075 :
1076 33 : fd_elf64_shdr const * shtext = &shdrs[ info->shndx_text ];
1077 33 : fd_sbpf_calldests_t * calldests = loader->calldests;
1078 :
1079 33 : uchar * ptr = rodata + shtext->sh_offset;
1080 33 : ulong insn_cnt = shdr_get_loaded_size( shtext ) / 8UL;
1081 :
1082 18762 : for( ulong i=0; i<insn_cnt; i++, ptr+=8UL ) {
1083 18729 : ulong insn = *((ulong *) ptr);
1084 :
1085 : /* Check for call instruction. If immediate is UINT_MAX, assume
1086 : that compiler generated a relocation instead. */
1087 18729 : ulong opc = insn & 0xFF;
1088 18729 : int imm = (int)(insn >> 32UL);
1089 18729 : if( (opc!=0x85) | (imm==-1) )
1090 18384 : continue;
1091 :
1092 : /* Mark function call destination */
1093 345 : long target_pc_s;
1094 345 : REQUIRE( 0==__builtin_saddl_overflow( (long)i+1L, imm, &target_pc_s ) );
1095 345 : ulong target_pc = (ulong)target_pc_s;
1096 345 : REQUIRE( target_pc<insn_cnt ); /* bounds check target */
1097 :
1098 345 : fd_sbpf_calldests_insert( calldests, target_pc );
1099 :
1100 : /* Replace immediate with hash */
1101 345 : uint pc_hash = fd_pchash( (uint)target_pc );
1102 : /* Check for collision with syscall ID
1103 : https://github.com/solana-labs/rbpf/blob/57139e9e1fca4f01155f7d99bc55cdcc25b0bc04/src/program.rs#L142-L146 */
1104 345 : REQUIRE( !fd_sbpf_syscalls_query( loader->syscalls, pc_hash, NULL ) );
1105 :
1106 345 : FD_STORE( uint, ptr+4UL, pc_hash );
1107 345 : }
1108 :
1109 33 : return 0;
1110 33 : }
1111 :
1112 : static int
1113 : fd_sbpf_relocate( fd_sbpf_loader_t const * loader,
1114 : fd_sbpf_elf_t const * elf,
1115 : ulong elf_sz,
1116 : uchar * rodata,
1117 33 : fd_sbpf_elf_info_t const * info ) {
1118 :
1119 33 : ulong const dt_rel = loader->dt_rel;
1120 33 : ulong const dt_relent = loader->dt_relent;
1121 33 : ulong const dt_relsz = loader->dt_relsz;
1122 :
1123 : /* Skip relocation if DT_REL is missing */
1124 :
1125 33 : if( dt_rel == 0UL ) return 0;
1126 :
1127 : /* Validate reloc table params */
1128 :
1129 18 : REQUIRE( dt_relent==sizeof(fd_elf64_rel) );
1130 18 : REQUIRE( dt_relsz !=0UL );
1131 18 : REQUIRE( (dt_relsz % sizeof(fd_elf64_rel))==0UL );
1132 :
1133 : /* Resolve DT_REL virtual address to file offset
1134 : First, attempt to find segment containing DT_REL */
1135 :
1136 18 : ulong rel_off = ULONG_MAX;
1137 :
1138 18 : fd_elf64_phdr const * phdrs = (fd_elf64_phdr const *)( elf->bin + elf->ehdr.e_phoff );
1139 18 : ulong rel_phnum;
1140 45 : for( rel_phnum=0; rel_phnum < elf->ehdr.e_phnum; rel_phnum++ ) {
1141 45 : ulong va_lo = phdrs[ rel_phnum ].p_vaddr;
1142 45 : ulong va_hi = phdrs[ rel_phnum ].p_memsz + va_lo;
1143 45 : REQUIRE( va_hi>=va_lo );
1144 45 : if( (dt_rel>=va_lo) & (dt_rel<va_hi) ) {
1145 : /* Found */
1146 18 : ulong va_off = dt_rel - va_lo;
1147 18 : ulong pa_lo = phdrs[ rel_phnum ].p_offset + va_off;
1148 : /* Overflow checks */
1149 18 : REQUIRE( (va_off<=dt_rel)
1150 18 : & (pa_lo >=va_off)
1151 18 : & (pa_lo < elf_sz) );
1152 18 : rel_off = pa_lo;
1153 18 : break;
1154 18 : }
1155 45 : }
1156 :
1157 : /* DT_REL not contained in any segment. Fallback to section header
1158 : table for finding first dynamic reloc section. */
1159 :
1160 18 : if( rel_phnum == elf->ehdr.e_phnum ) {
1161 0 : fd_elf64_shdr const * shdrs = (fd_elf64_shdr const *)( elf->bin + elf->ehdr.e_shoff );
1162 0 : ulong rel_shnum;
1163 0 : for( rel_shnum=0; rel_shnum < elf->ehdr.e_shnum; rel_shnum++ )
1164 0 : if( shdrs[ rel_shnum ].sh_addr==dt_rel )
1165 0 : break;
1166 0 : REQUIRE( rel_shnum < elf->ehdr.e_shnum );
1167 0 : rel_off = shdrs[ rel_shnum ].sh_offset;
1168 0 : }
1169 :
1170 18 : REQUIRE( fd_ulong_is_aligned( rel_off, 8UL ) );
1171 18 : REQUIRE( (rel_off < elf_sz)
1172 18 : & (dt_relsz <= elf_sz)
1173 18 : & ((rel_off+dt_relsz) <= elf_sz) );
1174 :
1175 : /* Load section and reloc tables
1176 : Assume section header already validated at this point */
1177 :
1178 18 : fd_elf64_rel const * rel = (fd_elf64_rel const *)( elf->bin + rel_off );
1179 18 : ulong rel_cnt = dt_relsz/sizeof(fd_elf64_rel);
1180 :
1181 : /* Apply each reloc */
1182 :
1183 360 : for( ulong i=0; i<rel_cnt; i++ ) {
1184 342 : int res = fd_sbpf_apply_reloc( loader, elf, elf_sz, rodata, info, &rel[ i ] );
1185 342 : if( res!=0 ) return res;
1186 342 : }
1187 :
1188 18 : return 0;
1189 18 : }
1190 :
1191 : static int
1192 : fd_sbpf_zero_rodata( fd_sbpf_elf_t * elf,
1193 : uchar * rodata,
1194 33 : fd_sbpf_elf_info_t const * info ) {
1195 :
1196 33 : fd_elf64_shdr const * shdrs = (fd_elf64_shdr const *)( elf->bin + elf->ehdr.e_shoff );
1197 :
1198 : /* memset gaps between sections to zero.
1199 : Assume section sh_addrs are monotonically increasing.
1200 : Assume section virtual address ranges equal physical address ranges.
1201 : Assume ranges are not overflowing. */
1202 : /* FIXME match Solana more closely here */
1203 :
1204 33 : ulong cursor = 0UL;
1205 387 : for( ulong i=0; i<elf->ehdr.e_shnum; i++ ) {
1206 354 : if( !( info->loaded_sections[ i>>6UL ] & (1UL<<(i&63UL)) ) ) continue;
1207 :
1208 54 : fd_elf64_shdr const * shdr = &shdrs[ i ];
1209 :
1210 : /* NOBITS sections are included in rodata, but may have invalid
1211 : offsets, thus we can't trust the shdr->sh_offset field. */
1212 54 : if( FD_UNLIKELY( shdr->sh_type==FD_ELF_SHT_NOBITS ) ) continue;
1213 :
1214 54 : ulong off = shdr->sh_offset;
1215 54 : ulong sz = shdr->sh_size;
1216 54 : assert( cursor<=off ); /* Invariant: Monotonically increasing offsets */
1217 0 : assert( off+sz>=off ); /* Invariant: No integer overflow */
1218 0 : assert( off+sz<=info->rodata_sz ); /* Invariant: No buffer overflow */
1219 :
1220 : /* Fill gap with zeros */
1221 0 : ulong gap = off - cursor;
1222 54 : fd_memset( rodata+cursor, 0, gap );
1223 :
1224 54 : cursor = off+sz;
1225 54 : }
1226 :
1227 33 : fd_memset( rodata+cursor, 0, info->rodata_sz - cursor );
1228 :
1229 33 : return 0;
1230 33 : }
1231 :
1232 : int
1233 : fd_sbpf_program_load( fd_sbpf_program_t * prog,
1234 : void const * _bin,
1235 : ulong elf_sz,
1236 : fd_sbpf_syscalls_t * syscalls,
1237 33 : int elf_deploy_checks ) {
1238 33 : fd_sbpf_loader_seterr( 0, 0 );
1239 :
1240 33 : int err;
1241 33 : fd_sbpf_elf_t * elf = (fd_sbpf_elf_t *)_bin;
1242 :
1243 33 : fd_sbpf_loader_t loader = {
1244 33 : .calldests = prog->calldests,
1245 33 : .syscalls = syscalls,
1246 :
1247 33 : .dyn_off = 0U,
1248 33 : .dyn_cnt = 0U,
1249 :
1250 33 : .dt_rel = 0UL,
1251 33 : .dt_relent = 0UL,
1252 33 : .dt_relsz = 0UL,
1253 33 : .dt_symtab = 0UL,
1254 :
1255 33 : .dynsym_off = 0U,
1256 33 : .dynsym_cnt = 0U,
1257 33 : .elf_deploy_checks = elf_deploy_checks
1258 33 : };
1259 :
1260 : /* Find dynamic section */
1261 33 : if( FD_UNLIKELY( (err=fd_sbpf_find_dynamic( &loader, elf, elf_sz, &prog->info ))!=0 ) )
1262 0 : return err;
1263 :
1264 : /* Load dynamic section */
1265 33 : if( FD_UNLIKELY( (err=fd_sbpf_load_dynamic( &loader, elf, elf_sz ))!=0 ) )
1266 0 : return err;
1267 :
1268 : /* Register entrypoint to calldests. */
1269 33 : fd_sbpf_calldests_insert( prog->calldests, prog->entry_pc );
1270 :
1271 : /* Copy rodata segment */
1272 33 : fd_memcpy( prog->rodata, elf->bin, prog->info.rodata_footprint );
1273 :
1274 : /* Convert calls with PC relative immediate to hashes */
1275 33 : if( FD_UNLIKELY( (err=fd_sbpf_hash_calls ( &loader, prog, elf ))!=0 ) )
1276 0 : return err;
1277 :
1278 : /* Apply relocations */
1279 33 : if( FD_UNLIKELY( (err=fd_sbpf_relocate ( &loader, elf, elf_sz, prog->rodata, &prog->info ))!=0 ) )
1280 0 : return err;
1281 :
1282 : /* Create read-only segment */
1283 33 : if( FD_UNLIKELY( (err=fd_sbpf_zero_rodata( elf, prog->rodata, &prog->info ))!=0 ) )
1284 0 : return err;
1285 :
1286 33 : return 0;
1287 33 : }
1288 :
1289 : #undef ERR
1290 : #undef FAIL
1291 : #undef REQUIRE
|
