Line data Source code
1 : #include "fd_sbpf_loader.h"
2 : #include "fd_sbpf_opcodes.h"
3 : #include "../../util/fd_util.h"
4 : #include "../../util/bits/fd_sat.h"
5 : #include "../murmur3/fd_murmur3.h"
6 :
7 : #include <assert.h>
8 : #include <stdio.h>
9 :
10 : /* Error handling *****************************************************/
11 :
12 : /* Thread local storage last error value */
13 :
14 : static FD_TL int ldr_errno = 0;
15 : static FD_TL int ldr_err_srcln = -1;
16 : #define FD_SBPF_ERRBUF_SZ (128UL)
17 : static FD_TL char fd_sbpf_errbuf[ FD_SBPF_ERRBUF_SZ ] = {0};
18 :
19 : /* fd_sbpf_loader_seterr remembers the error ID and line number of the
20 : current file at which the last error occurred. */
21 :
22 : __attribute__((cold,noinline)) static int
23 : fd_sbpf_loader_seterr( int err,
24 17016 : int srcln ) {
25 17016 : ldr_errno = err;
26 17016 : ldr_err_srcln = srcln;
27 17016 : return err;
28 17016 : }
29 :
30 : /* Macros for returning an error from the current function while also
31 : remembering the error code. */
32 :
33 13338 : #define ERR( err ) return fd_sbpf_loader_seterr( (err), __LINE__ )
34 13281 : #define FAIL() ERR( FD_SBPF_ERR_INVALID_ELF )
35 16353375 : #define REQUIRE(x) do { if ( FD_UNLIKELY( !(x) ) ) FAIL(); } while (0)
36 :
37 : char const *
38 64668 : fd_sbpf_strerror( void ) {
39 64668 : if( FD_UNLIKELY( ldr_errno==0 ) )
40 2634 : strcpy( fd_sbpf_errbuf, "ok" );
41 62034 : else
42 62034 : snprintf( fd_sbpf_errbuf, FD_SBPF_ERRBUF_SZ,
43 64668 : "code %d at %s(%d)", ldr_errno, __FILE__, ldr_err_srcln );
44 64668 : return fd_sbpf_errbuf;
45 64668 : }
46 :
47 : /* ELF loader, part 1 **************************************************
48 :
49 : Start with a static piece of scratch memory and do basic validation
50 : of the file content. Walk the section table once and remember
51 : sections of interest.
52 :
53 : ### Terminology
54 :
55 : This source follows common ELF naming practices.
56 :
57 : section: a named data region present in the ELF file
58 : segment: a contiguous memory region containing sections
59 : (not necessarily contiguous in the ELF file)
60 :
61 : physical address (paddr): Byte offset into ELF file (uchar * bin)
62 : virtual address (vaddr): VM memory address */
63 :
64 : /* Provide convenient access to file header and ELF content */
65 :
66 : __extension__ union fd_sbpf_elf {
67 : fd_elf64_ehdr ehdr;
68 : uchar bin[0];
69 : };
70 : typedef union fd_sbpf_elf fd_sbpf_elf_t;
71 :
72 : /* FD_SBPF_MM_{...}_ADDR are hardcoded virtual addresses of segments
73 : in the sBPF virtual machine.
74 :
75 : FIXME: These should be defined elsewhere */
76 :
77 2592510 : #define FD_SBPF_MM_PROGRAM_ADDR (0x100000000UL) /* readonly program data */
78 : #define FD_SBPF_MM_STACK_ADDR (0x200000000UL) /* stack (with gaps) */
79 :
80 : /* _fd_int_store_if_negative stores x to *p if *p is negative (branchless) */
81 :
82 : static inline int
83 : _fd_int_store_if_negative( int * p,
84 8139 : int x ) {
85 8139 : return (*p = fd_int_if( (*p)<0, x, *p ));
86 8139 : }
87 :
88 : /* fd_sbpf_check_ehdr verifies the ELF file header. */
89 :
90 : static int
91 : fd_sbpf_check_ehdr( fd_elf64_ehdr const * ehdr,
92 : ulong elf_sz,
93 : uint min_version,
94 16071 : uint max_version ) {
95 :
96 : /* Validate ELF magic */
97 16071 : REQUIRE( ( fd_uint_load_4( ehdr->e_ident )==0x464c457fU )
98 : /* Validate file type/target identification
99 : Solana/Agave performs header checks across two places:
100 : - Elf64::parse https://github.com/solana-labs/rbpf/blob/v0.8.0/src/elf_parser/mod.rs#L108
101 : - Executable::validate https://github.com/solana-labs/rbpf/blob/v0.8.0/src/elf.rs#L518
102 : These two sections are executed in close proximity, with no modifications to the header in between.
103 : We can therefore consolidate the checks in one place.
104 : */
105 16071 : & ( ehdr->e_ident[ FD_ELF_EI_CLASS ]==FD_ELF_CLASS_64 )
106 16071 : & ( ehdr->e_ident[ FD_ELF_EI_DATA ]==FD_ELF_DATA_LE )
107 16071 : & ( ehdr->e_ident[ FD_ELF_EI_VERSION ]==1 )
108 16071 : & ( ehdr->e_ident[ FD_ELF_EI_OSABI ]==FD_ELF_OSABI_NONE )
109 16071 : & ( ehdr->e_type ==FD_ELF_ET_DYN )
110 16071 : & ( ( ehdr->e_machine ==FD_ELF_EM_BPF )
111 16071 : | ( ehdr->e_machine ==FD_ELF_EM_SBPF ) )
112 16071 : & ( ehdr->e_version ==1 )
113 : /* Coherence checks */
114 16071 : & ( ehdr->e_ehsize ==sizeof(fd_elf64_ehdr) )
115 16071 : & ( ehdr->e_phentsize==sizeof(fd_elf64_phdr) )
116 16071 : & ( ehdr->e_shentsize==sizeof(fd_elf64_shdr) )
117 16071 : & ( ehdr->e_shstrndx < ehdr->e_shnum )
118 16071 : & ( ehdr->e_flags >= min_version )
119 16071 : & ( max_version
120 16071 : ? ( ehdr->e_flags <= max_version )
121 16071 : : ( ehdr->e_flags != FD_ELF_EF_SBPF_V2 )
122 16071 : )
123 16071 : );
124 :
125 : /* Bounds check program header table */
126 :
127 4365 : ulong const phoff = ehdr->e_phoff;
128 4365 : ulong const phnum = ehdr->e_phnum;
129 4365 : REQUIRE( ( fd_ulong_is_aligned( phoff, 8UL ) )
130 4365 : & ( phoff<=elf_sz ) ); /* out of bounds */
131 :
132 4326 : REQUIRE( phnum<=(ULONG_MAX/sizeof(fd_elf64_phdr)) ); /* overflow */
133 4326 : ulong const phsz = phnum*sizeof(fd_elf64_phdr);
134 :
135 4326 : ulong const phoff_end = phoff+phsz;
136 4326 : REQUIRE( ( phoff_end>=phoff ) /* overflow */
137 4326 : & ( phoff_end<=elf_sz ) /* out of bounds */
138 4326 : & ( (phoff_end==0UL) /* overlaps file header */
139 4326 : | (phoff>=sizeof(fd_elf64_ehdr)) ) );
140 :
141 : /* Bounds check section header table */
142 :
143 4326 : ulong const shoff = ehdr->e_shoff;
144 4326 : ulong const shnum = ehdr->e_shnum;
145 4326 : REQUIRE( ( fd_ulong_is_aligned( shoff, 8UL ) )
146 4326 : & ( shoff>=sizeof(fd_elf64_ehdr) ) /* overlaps file header */
147 4326 : & ( shoff< elf_sz ) /* out of bounds */
148 4326 : & ( shnum> 0UL ) ); /* not enough sections */
149 :
150 4323 : REQUIRE( shoff<=(ULONG_MAX/sizeof(fd_elf64_shdr)) ); /* overflow */
151 4323 : ulong const shsz = shnum*sizeof(fd_elf64_shdr);
152 :
153 4323 : ulong const shoff_end = shoff+shsz;
154 4323 : REQUIRE( ( shoff_end>=shoff ) /* overflow */
155 4323 : & ( shoff_end<=elf_sz ) ); /* out of bounds */
156 :
157 : /* Overlap checks */
158 :
159 4320 : REQUIRE( (phoff>=shoff_end) | (shoff>=phoff_end) ); /* overlap shdrs<>phdrs */
160 :
161 4287 : return 0;
162 4320 : }
163 :
164 : /* shdr_get_loaded_size returns the loaded size of a section, i.e. the
165 : number of bytes loaded into the rodata segment. sBPF ELFs grossly
166 : misuse the sh_size parameter. When SHT_NOBITS is set, the actual
167 : section size is zero, and the section size is ignored. */
168 :
169 : static ulong
170 1554696 : shdr_get_loaded_size( fd_elf64_shdr const * shdr ) {
171 1554696 : return fd_ulong_if( shdr->sh_type==FD_ELF_SHT_NOBITS, 0UL, shdr->sh_size );
172 1554696 : }
173 :
174 : /* check_cstr verifies a string in a string table. Returns non-NULL if
175 : the string is null terminated and contains at most max non-NULL
176 : characters. Returns NULL if the off is out of bounds, or if the max
177 : or EOF are reached before the null terminator. */
178 :
179 : static char const *
180 : check_cstr( uchar const * bin,
181 : ulong bin_sz,
182 : ulong off,
183 : ulong max,
184 490236 : ulong * opt_sz ) {
185 490236 : if( FD_UNLIKELY( off>=bin_sz ) ) return NULL;
186 490236 : max += 1UL; /* include NULL terminator */
187 490236 : max = fd_ulong_min( max, bin_sz-off ); /* truncate to available size */
188 490236 : char const * cstr = (char const *)( bin+off );
189 490236 : ulong len = strnlen( cstr, max );
190 490236 : if( opt_sz ) *opt_sz = len;
191 490236 : return len<max ? cstr : NULL;
192 490236 : }
193 :
194 : /* fd_sbpf_load_phdrs walks the program header table. Remembers info
195 : along the way, and performs various validations.
196 :
197 : Assumes that ...
198 : - table does not overlap with file header or section header table and
199 : is within bounds
200 : - offset of program header table is 8 byte aligned */
201 :
202 : static int
203 : fd_sbpf_load_phdrs( fd_sbpf_elf_info_t * info,
204 : fd_sbpf_elf_t const * elf,
205 4287 : ulong elf_sz ) {
206 :
207 4287 : ulong const pht_offset = elf->ehdr.e_phoff;
208 4287 : ulong const pht_cnt = elf->ehdr.e_phnum;
209 :
210 : /* Virtual address of last seen program header */
211 4287 : ulong p_load_vaddr = 0UL;
212 :
213 : /* Read program header table */
214 4287 : fd_elf64_phdr const * phdr = (fd_elf64_phdr const *)( elf->bin + pht_offset );
215 20778 : for( ulong i=0; i<pht_cnt; i++ ) {
216 16515 : switch( phdr[i].p_type ) {
217 4254 : case FD_ELF_PT_DYNAMIC:
218 : /* Remember first PT_DYNAMIC segment */
219 4254 : _fd_int_store_if_negative( &info->phndx_dyn, (int)i );
220 4254 : break;
221 9201 : case FD_ELF_PT_LOAD:
222 : /* LOAD segments must be ordered */
223 9201 : REQUIRE( phdr[ i ].p_vaddr >= p_load_vaddr );
224 9201 : p_load_vaddr = phdr[ i ].p_vaddr;
225 : /* Segment must be within bounds */
226 9201 : REQUIRE( ( phdr[ i ].p_offset + phdr[ i ].p_filesz >= phdr[ i ].p_offset )
227 9201 : & ( phdr[ i ].p_offset + phdr[ i ].p_filesz <= elf_sz ) );
228 : /* No overlap checks */
229 9177 : break;
230 9177 : default:
231 : /* Ignore other segment types */
232 3060 : break;
233 16515 : }
234 16515 : }
235 :
236 4263 : return 0;
237 4287 : }
238 :
239 : /* FD_SBPF_SECTION_NAME_SZ_MAX is the maximum length of a symbol name cstr
240 : including zero terminator.
241 : https://github.com/solana-labs/rbpf/blob/c168a8715da668a71584ea46696d85f25c8918f6/src/elf_parser/mod.rs#L12 */
242 96753 : #define FD_SBPF_SECTION_NAME_SZ_MAX (16UL)
243 :
244 : /* fd_sbpf_load_shdrs walks the section header table. Remembers info
245 : along the way, and performs various validations.
246 :
247 : Assumes that ...
248 : - table does not overlap with file header or program header table and
249 : is within bounds
250 : - offset of section header table is 8 byte aligned
251 : - section header table has at least one entry */
252 :
253 : static int
254 : fd_sbpf_load_shdrs( fd_sbpf_elf_info_t * info,
255 : fd_sbpf_elf_t const * elf,
256 : ulong elf_sz,
257 4263 : int elf_deploy_checks ) {
258 :
259 : /* File Header */
260 4263 : ulong const eh_offset = 0UL;
261 4263 : ulong const eh_offend = sizeof(fd_elf64_ehdr);
262 :
263 : /* Section Header Table */
264 4263 : ulong const sht_offset = elf->ehdr.e_shoff;
265 4263 : ulong const sht_cnt = elf->ehdr.e_shnum;
266 4263 : ulong const sht_sz = sht_cnt*sizeof(fd_elf64_shdr);
267 4263 : ulong const sht_offend = sht_offset + sht_sz;
268 :
269 4263 : fd_elf64_shdr const * shdr = (fd_elf64_shdr const *)( elf->bin + sht_offset );
270 :
271 : /* Program Header Table */
272 4263 : ulong const pht_offset = elf->ehdr.e_phoff;
273 4263 : ulong const pht_cnt = elf->ehdr.e_phnum;
274 4263 : ulong const pht_offend = pht_offset + (pht_cnt*sizeof(fd_elf64_phdr));
275 :
276 : /* Overlap checks */
277 4263 : REQUIRE( (sht_offset>=eh_offend ) | (sht_offend<=eh_offset ) ); /* overlaps ELF file header */
278 4263 : REQUIRE( (sht_offset>=pht_offend) | (sht_offend<=pht_offset) ); /* overlaps program header table */
279 :
280 : /* Require SHT_STRTAB for section name table */
281 :
282 4263 : REQUIRE( elf->ehdr.e_shstrndx < sht_cnt ); /* out of bounds */
283 4263 : REQUIRE( shdr[ elf->ehdr.e_shstrndx ].sh_type==FD_ELF_SHT_STRTAB );
284 :
285 4212 : ulong shstr_off = shdr[ elf->ehdr.e_shstrndx ].sh_offset;
286 4212 : ulong shstr_sz = shdr[ elf->ehdr.e_shstrndx ].sh_size;
287 4212 : REQUIRE( shstr_off<elf_sz );
288 4212 : shstr_sz = fd_ulong_min( shstr_sz, elf_sz-shstr_off );
289 :
290 : /* Clear the "loaded sections" bitmap */
291 :
292 4212 : fd_memset( info->loaded_sections, 0, sizeof(info->loaded_sections) );
293 :
294 : /* Validate section header table.
295 : Check that all sections are in bounds, ordered, and don't overlap. */
296 :
297 4212 : ulong min_sh_offset = 0UL; /* lowest permitted section offset */
298 :
299 : /* Keep track of the physical (file address) end of all relevant
300 : sections to determine rodata_sz */
301 4212 : ulong psegment_end = 0UL; /* Upper bound of physical (file) addressing */
302 :
303 : /* While validating section header table, also figure out size
304 : of the rodata segment. This is the minimal virtual address range
305 : that spans all sections. */
306 4212 : ulong vsegment_start = FD_SBPF_MM_PROGRAM_ADDR; /* Lower bound of segment virtual address */
307 4212 : ulong vsegment_end = 0UL; /* Upper bound of segment virtual address */
308 :
309 4212 : ulong tot_section_sz = 0UL; /* Size of all sections */
310 :
311 52431 : for( ulong i=0UL; i<sht_cnt; i++ ) {
312 48564 : uint sh_type = shdr[ i ].sh_type;
313 48564 : uint sh_name = shdr[ i ].sh_name;
314 48564 : ulong sh_addr = shdr[ i ].sh_addr;
315 48564 : ulong sh_offset = shdr[ i ].sh_offset;
316 48564 : ulong sh_size = shdr[ i ].sh_size;
317 48564 : ulong sh_offend = sh_offset + sh_size;
318 :
319 : /* First section must be SHT_NULL */
320 48564 : REQUIRE( i>0UL || sh_type==FD_ELF_SHT_NULL );
321 :
322 : /* check that physical range has no overflow and is within bounds */
323 48561 : REQUIRE( sh_offend >= sh_offset );
324 48519 : REQUIRE( sh_offend <= elf_sz ); // https://github.com/solana-labs/rbpf/blob/v0.8.0/src/elf_parser/mod.rs#L180
325 :
326 48513 : if( sh_type!=FD_ELF_SHT_NOBITS ) {
327 : /* Overlap checks */
328 47853 : REQUIRE( (sh_offset>=eh_offend ) | (sh_offend<=eh_offset ) ); /* overlaps ELF file header */
329 47853 : REQUIRE( (sh_offset>=pht_offend) | (sh_offend<=pht_offset) ); /* overlaps program header table */
330 47853 : REQUIRE( (sh_offset>=sht_offend) | (sh_offend<=sht_offset) ); /* overlaps section header table */
331 :
332 : /* Ordering and overlap check
333 : https://github.com/solana-labs/rbpf/blob/v0.8.0/src/elf_parser/mod.rs#L177
334 : */
335 47853 : REQUIRE( sh_offset >= min_sh_offset );
336 47853 : min_sh_offset = sh_offend;
337 47853 : }
338 :
339 48513 : if( sh_type==FD_ELF_SHT_DYNAMIC ) {
340 : /* Remember first SHT_DYNAMIC segment */
341 3885 : _fd_int_store_if_negative( &info->shndx_dyn, (int)i );
342 3885 : }
343 :
344 48513 : ulong name_off = shstr_off + (ulong)sh_name;
345 48513 : REQUIRE( ( name_off<elf_sz ) /* out of bounds */
346 48513 : & ( sh_name <shstr_sz ) );
347 :
348 : /* Create name cstr */
349 :
350 48513 : char const * name_ptr = check_cstr( elf->bin + shstr_off, shstr_sz, sh_name, FD_SBPF_SECTION_NAME_SZ_MAX-1UL, NULL );
351 48513 : REQUIRE( name_ptr );
352 48240 : char __attribute__((aligned(16UL))) name[ FD_SBPF_SECTION_NAME_SZ_MAX ] = {0};
353 48240 : strncpy( name, name_ptr, FD_SBPF_SECTION_NAME_SZ_MAX-1UL );
354 :
355 : /* Check name */
356 : /* TODO switch table for this? */
357 : /* TODO reject duplicate sections */
358 :
359 48240 : int load = 0; /* should section be loaded? */
360 :
361 48240 : /**/ if( 0==memcmp( name, ".text", 6UL /* equals */ ) ) {
362 3951 : REQUIRE( (info->shndx_text)<0 ); /* check for duplicate */
363 3951 : info->shndx_text = (int)i;
364 3951 : load = 1;
365 3951 : }
366 44289 : else if( (0==memcmp( name, ".rodata", 8UL /* equals */ ) )
367 44289 : | (0==memcmp( name, ".data.rel.ro", 13UL /* equals */ ) )
368 44289 : | (0==memcmp( name, ".eh_frame", 10UL /* equals */ ) ) ) {
369 9051 : load = 1;
370 9051 : }
371 35238 : else if( 0==memcmp( name, ".symtab", 8UL /* equals */ ) ) {
372 1170 : REQUIRE( (info->shndx_symtab)<0 );
373 1170 : info->shndx_symtab = (int)i;
374 1170 : }
375 34068 : else if( 0==memcmp( name, ".strtab", 8UL /* equals */ ) ) {
376 1197 : REQUIRE( (info->shndx_strtab)<0 );
377 1176 : info->shndx_strtab = (int)i;
378 1176 : }
379 32871 : else if( 0==memcmp( name, ".dynstr", 8UL /* equals */ ) ) {
380 3837 : REQUIRE( (info->shndx_dynstr)<0 );
381 3837 : info->shndx_dynstr = (int)i;
382 3837 : }
383 29034 : else if( 0==memcmp( name, ".bss", 4UL /* has prefix */ ) ) {
384 0 : FAIL();
385 0 : }
386 29034 : else if( 0==memcmp( name, ".data.rel", 9UL /* has prefix */ ) ) {} /* ignore */
387 28725 : else if( (0==memcmp( name, ".data", 5UL /* has prefix */ ) )
388 28725 : & ( ( shdr[ i ].sh_flags & (FD_ELF_SHF_ALLOC|FD_ELF_SHF_WRITE) )
389 28725 : ==(FD_ELF_SHF_ALLOC|FD_ELF_SHF_WRITE) ) ) {
390 0 : FAIL();
391 0 : }
392 28725 : else {} /* ignore */
393 : /* else ignore */
394 :
395 48219 : if( load ) {
396 : /* Remember that section should be loaded */
397 :
398 13002 : info->loaded_sections[ i>>6UL ] |= (1UL)<<(i&63UL);
399 :
400 : /* Check that virtual address range is in MM_PROGRAM bounds */
401 :
402 13002 : ulong sh_actual_size = shdr_get_loaded_size( &shdr[ i ] );
403 13002 : ulong sh_virtual_end = sh_addr + sh_actual_size;
404 :
405 : /* https://github.com/solana-labs/rbpf/blob/v0.8.0/src/elf.rs#L426 */
406 13002 : if ( FD_UNLIKELY( elf_deploy_checks ) ){
407 2493 : REQUIRE( sh_addr == sh_offset );
408 2493 : }
409 13002 : REQUIRE( sh_addr < FD_SBPF_MM_PROGRAM_ADDR ); /* overflow check */
410 13002 : REQUIRE( sh_actual_size < FD_SBPF_MM_PROGRAM_ADDR ); /* overflow check */
411 13002 : REQUIRE( sh_virtual_end <= FD_SBPF_MM_STACK_ADDR-FD_SBPF_MM_PROGRAM_ADDR ); /* check overlap with stack */
412 :
413 : /* Check that physical address range is in bounds
414 : (Seems redundant?) */
415 13002 : ulong paddr_end = sh_offset + sh_actual_size;
416 13002 : REQUIRE( paddr_end >= sh_offset );
417 13002 : REQUIRE( paddr_end <= elf_sz );
418 :
419 13002 : vsegment_start = fd_ulong_min( vsegment_start, sh_addr );
420 : /* Expand range to fit section */
421 13002 : psegment_end = fd_ulong_max( psegment_end, paddr_end );
422 13002 : vsegment_end = fd_ulong_max( vsegment_end, sh_virtual_end );
423 :
424 : /* Coherence check sum of section sizes */
425 13002 : REQUIRE( tot_section_sz + sh_actual_size >= tot_section_sz ); /* overflow check */
426 13002 : tot_section_sz += sh_actual_size;
427 13002 : }
428 48219 : }
429 :
430 : /* More coherence checks */
431 3867 : REQUIRE( psegment_end <= elf_sz ); // https://github.com/solana-labs/rbpf/blob/v0.8.0/src/elf.rs#L782
432 :
433 :
434 : /* Check that the rodata segment is within bounds
435 : https://github.com/solana-labs/rbpf/blob/v0.8.0/src/elf.rs#L725 */
436 3867 : if ( FD_UNLIKELY( elf_deploy_checks ) ){
437 708 : REQUIRE( fd_ulong_sat_add( vsegment_start, tot_section_sz) <= vsegment_end );
438 708 : }
439 :
440 : /* Require .text section */
441 :
442 3858 : REQUIRE( (info->shndx_text)>=0 );
443 3693 : fd_elf64_shdr const * shdr_text = &shdr[ info->shndx_text ];
444 3693 : REQUIRE( (shdr_text->sh_addr <= elf->ehdr.e_entry)
445 : /* check that entrypoint is in text VM range */
446 3693 : & (elf->ehdr.e_entry < fd_ulong_sat_add( shdr_text->sh_addr, shdr_text->sh_size ) ) );
447 : /* NOTE: Does NOT check that the entrypoint is in text section file
448 : range (which may be 0 sz if SHT_NOBITS). This check is
449 : separately done in the sBPF verifier. */
450 :
451 3693 : info->text_off = (uint)shdr_text->sh_offset;
452 3693 : ulong text_size = shdr_get_loaded_size( shdr_text );
453 3693 : info->text_sz = text_size;
454 3693 : info->text_cnt = (uint) text_size / 8U;
455 :
456 :
457 : /* Convert entrypoint offset to program counter */
458 :
459 3693 : info->rodata_sz = (uint)psegment_end;
460 3693 : info->rodata_footprint = (uint)elf_sz;
461 :
462 3693 : ulong entry_off = fd_ulong_sat_sub( elf->ehdr.e_entry, shdr_text->sh_addr );
463 3693 : ulong entry_pc = entry_off / 8UL;
464 :
465 : /* Follows https://github.com/solana-labs/rbpf/blob/v0.8.0/src/elf.rs#L443 */
466 3693 : REQUIRE( fd_ulong_is_aligned( entry_off, 8UL ) );
467 3693 : REQUIRE( entry_pc < ( info->rodata_sz / 8UL ) );
468 3693 : info->entry_pc = (uint)entry_pc;
469 :
470 3693 : if( (info->shndx_dynstr)>=0 ) {
471 3621 : fd_elf64_shdr const * shdr_dynstr = &shdr[ info->shndx_dynstr ];
472 3621 : ulong sh_offset = shdr_dynstr->sh_offset;
473 3621 : ulong sh_size = shdr_dynstr->sh_size;
474 3621 : REQUIRE( (sh_offset+sh_size>=sh_offset) & (sh_offset+sh_size<=info->rodata_footprint) );
475 3621 : info->dynstr_off = (uint)sh_offset;
476 3621 : info->dynstr_sz = (uint)sh_size;
477 3621 : }
478 :
479 3693 : return 0;
480 3693 : }
481 :
482 : fd_sbpf_elf_info_t *
483 : fd_sbpf_elf_peek( fd_sbpf_elf_info_t * info,
484 : void const * bin,
485 : ulong elf_sz,
486 : int elf_deploy_checks,
487 : uint sbpf_min_version,
488 68310 : uint sbpf_max_version ) {
489 :
490 : /* ELFs must have a file header */
491 68310 : if( FD_UNLIKELY( elf_sz<=sizeof(fd_elf64_ehdr) ) )
492 51090 : return NULL;
493 :
494 : /* Reject overlong ELFs (using uint addressing internally).
495 : This is well beyond Solana's max account size of 10 MB. */
496 17220 : if( FD_UNLIKELY( elf_sz>UINT_MAX ) )
497 1149 : return NULL;
498 :
499 : /* Initialize info struct */
500 16071 : *info = (fd_sbpf_elf_info_t) {
501 16071 : .text_off = 0U,
502 16071 : .text_cnt = 0U,
503 16071 : .dynstr_off = 0U,
504 16071 : .dynstr_sz = 0U,
505 16071 : .rodata_footprint = 0U,
506 16071 : .rodata_sz = 0U,
507 16071 : .shndx_text = -1,
508 16071 : .shndx_symtab = -1,
509 16071 : .shndx_strtab = -1,
510 16071 : .shndx_dyn = -1,
511 16071 : .shndx_dynstr = -1,
512 16071 : .phndx_dyn = -1,
513 16071 : .sbpf_version = 0U,
514 : /* !!! Keep this in sync with -Werror=missing-field-initializers */
515 16071 : };
516 :
517 16071 : fd_sbpf_elf_t const * elf = (fd_sbpf_elf_t const *)bin;
518 16071 : int err;
519 :
520 : /* Validate file header */
521 16071 : if( FD_UNLIKELY( (err=fd_sbpf_check_ehdr( &elf->ehdr, elf_sz, sbpf_min_version, sbpf_max_version ))!=0 ) )
522 11784 : return NULL;
523 :
524 : /* Program headers */
525 4287 : if( FD_UNLIKELY( (err=fd_sbpf_load_phdrs( info, elf, elf_sz ))!=0 ) )
526 24 : return NULL;
527 :
528 : /* Section headers */
529 4263 : if( FD_UNLIKELY( (err=fd_sbpf_load_shdrs( info, elf, elf_sz, elf_deploy_checks ))!=0 ) )
530 570 : return NULL;
531 :
532 : /* Set SBPF version from ELF e_flags */
533 3693 : info->sbpf_version = sbpf_max_version ? elf->ehdr.e_flags : 0UL;
534 :
535 3693 : return info;
536 4263 : }
537 :
538 : /* ELF loader, part 2 **************************************************
539 :
540 : Prepare a copy of a subrange of the ELF content: The rodata segment.
541 : Mangle the copy by applying dynamic relocations. Then, zero out
542 : parts of the segment that are not interesting to the loader.
543 :
544 : ### Terminology
545 :
546 : Shorthands for relocation handling:
547 :
548 : S: Symbol value (typically an ELF physical address)
549 : A: Implicit addend, i.e. the original value of the field that the
550 : relocation handler is about to write to
551 : V: Virtual address, i.e. the target value that the relocation
552 : handler is about to write into where the implicit addend was
553 : previously stored */
554 :
555 : ulong
556 3678 : fd_sbpf_program_align( void ) {
557 3678 : return alignof( fd_sbpf_program_t );
558 3678 : }
559 :
560 : ulong
561 3678 : fd_sbpf_program_footprint( fd_sbpf_elf_info_t const * info ) {
562 3678 : FD_COMPILER_UNPREDICTABLE( info ); /* Make this appear as FD_FN_PURE (e.g. footprint might depened on info contents in future) */
563 3678 : return FD_LAYOUT_FINI( FD_LAYOUT_APPEND( FD_LAYOUT_APPEND( FD_LAYOUT_INIT,
564 3678 : alignof(fd_sbpf_program_t), sizeof(fd_sbpf_program_t) ),
565 3678 : fd_sbpf_calldests_align(), fd_sbpf_calldests_footprint( info->rodata_sz / 8UL ) ), /* calldests bitmap */
566 3678 : alignof(fd_sbpf_program_t) );
567 3678 : }
568 :
569 : fd_sbpf_program_t *
570 : fd_sbpf_program_new( void * prog_mem,
571 : fd_sbpf_elf_info_t const * elf_info,
572 3678 : void * rodata ) {
573 :
574 3678 : if( FD_UNLIKELY( !prog_mem ) ) {
575 0 : FD_LOG_WARNING(( "NULL prog_mem" ));
576 0 : return NULL;
577 0 : }
578 :
579 3678 : if( FD_UNLIKELY( !elf_info ) ) {
580 0 : FD_LOG_WARNING(( "NULL elf_info" ));
581 0 : return NULL;
582 0 : }
583 :
584 3678 : if( FD_UNLIKELY( ((elf_info->rodata_footprint)>0U) & (!rodata)) ) {
585 0 : FD_LOG_WARNING(( "NULL rodata" ));
586 0 : return NULL;
587 0 : }
588 :
589 : /* https://github.com/solana-labs/rbpf/blob/v0.8.0/src/elf_parser/mod.rs#L99 */
590 3678 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong) rodata, FD_SBPF_PROG_RODATA_ALIGN ) ) ){
591 0 : FD_LOG_WARNING(( "rodata is not 8-byte aligned" ));
592 0 : return NULL;
593 0 : }
594 :
595 : /* Initialize program struct */
596 :
597 3678 : FD_SCRATCH_ALLOC_INIT( laddr, prog_mem );
598 3678 : fd_sbpf_program_t * prog = FD_SCRATCH_ALLOC_APPEND( laddr, alignof(fd_sbpf_program_t), sizeof(fd_sbpf_program_t) );
599 :
600 0 : *prog = (fd_sbpf_program_t) {
601 3678 : .info = *elf_info,
602 3678 : .rodata = rodata,
603 3678 : .rodata_sz = elf_info->rodata_sz,
604 3678 : .text = (ulong *)((ulong)rodata + elf_info->text_off), /* FIXME: WHAT IF MISALIGNED */
605 3678 : .text_off = elf_info->text_off,
606 3678 : .text_cnt = elf_info->text_cnt,
607 3678 : .text_sz = elf_info->text_sz,
608 3678 : .entry_pc = elf_info->entry_pc
609 3678 : };
610 :
611 : /* Initialize calldests map */
612 :
613 3678 : ulong pc_max = elf_info->rodata_sz / 8UL;
614 3678 : prog->calldests_shmem = fd_sbpf_calldests_new(
615 3678 : FD_SCRATCH_ALLOC_APPEND( laddr, fd_sbpf_calldests_align(),
616 3678 : fd_sbpf_calldests_footprint( pc_max ) ),
617 0 : pc_max );
618 0 : prog->calldests = fd_sbpf_calldests_join( prog->calldests_shmem );
619 :
620 3678 : return prog;
621 3678 : }
622 :
623 : void *
624 30 : fd_sbpf_program_delete( fd_sbpf_program_t * mem ) {
625 :
626 30 : fd_sbpf_calldests_delete( fd_sbpf_calldests_leave( mem->calldests ) );
627 30 : fd_memset( mem, 0, sizeof(fd_sbpf_program_t) );
628 :
629 30 : return (void *)mem;
630 30 : }
631 :
632 : /* fd_sbpf_loader_t contains various temporary state during loading */
633 :
634 : struct fd_sbpf_loader {
635 : /* External objects */
636 : ulong * calldests; /* owned by program */
637 : fd_sbpf_syscalls_t * syscalls; /* owned by caller */
638 :
639 : /* Dynamic table */
640 : uint dyn_off; /* File offset of dynamic table (UINT_MAX=missing) */
641 : uint dyn_cnt; /* Number of dynamic table entries */
642 :
643 : /* Dynamic table entries */
644 : ulong dt_rel;
645 : ulong dt_relent;
646 : ulong dt_relsz;
647 : ulong dt_symtab;
648 :
649 : /* Dynamic symbols */
650 : uint dynsym_off; /* File offset of .dynsym section (0=missing) */
651 : uint dynsym_cnt; /* Symbol count */
652 :
653 : int elf_deploy_checks;
654 : };
655 : typedef struct fd_sbpf_loader fd_sbpf_loader_t;
656 :
657 : /* FD_SBPF_SYM_NAME_SZ_MAX is the maximum length of a symbol name cstr
658 : including zero terminator.
659 : https://github.com/solana-labs/rbpf/blob/c168a8715da668a71584ea46696d85f25c8918f6/src/elf_parser/mod.rs#L13 */
660 441723 : #define FD_SBPF_SYM_NAME_SZ_MAX (64UL)
661 :
662 :
663 : static int
664 : fd_sbpf_find_dynamic( fd_sbpf_loader_t * loader,
665 : fd_sbpf_elf_t const * elf,
666 : ulong elf_sz,
667 3678 : fd_sbpf_elf_info_t const * info ) {
668 :
669 3678 : fd_elf64_shdr const * shdrs = (fd_elf64_shdr const *)( elf->bin + elf->ehdr.e_shoff );
670 3678 : fd_elf64_phdr const * phdrs = (fd_elf64_phdr const *)( elf->bin + elf->ehdr.e_phoff );
671 :
672 : /* Try first PT_DYNAMIC in program header table */
673 :
674 3678 : if( (info->phndx_dyn)>=0 ) {
675 3588 : ulong dyn_off = phdrs[ info->phndx_dyn ].p_offset;
676 3588 : ulong dyn_sz = phdrs[ info->phndx_dyn ].p_filesz;
677 3588 : ulong dyn_end = dyn_off+dyn_sz;
678 :
679 : /* Fall through to SHT_DYNAMIC if invalid */
680 :
681 3588 : if( FD_LIKELY( ( dyn_end>=dyn_off ) /* overflow */
682 3588 : & ( dyn_end<=elf_sz ) /* out of bounds */
683 3588 : & fd_ulong_is_aligned( dyn_off, 8UL ) /* misaligned */
684 3588 : & fd_ulong_is_aligned( dyn_sz, sizeof(fd_elf64_dyn) ) /* misaligned sz */ ) ) {
685 3339 : loader->dyn_off = (uint)dyn_off;
686 3339 : loader->dyn_cnt = (uint)(dyn_sz / sizeof(fd_elf64_dyn));
687 3339 : return 0;
688 3339 : }
689 3588 : }
690 :
691 : /* Try first SHT_DYNAMIC in section header table */
692 :
693 339 : if( (info->shndx_dyn)>0 ) {
694 258 : ulong dyn_off = shdrs[ info->shndx_dyn ].sh_offset;
695 258 : ulong dyn_sz = shdrs[ info->shndx_dyn ].sh_size;
696 258 : ulong dyn_end = dyn_off+dyn_sz;
697 :
698 : /* This time, don't tolerate errors */
699 :
700 258 : REQUIRE( ( dyn_end>=dyn_off ) /* overflow */
701 258 : & ( dyn_end<=elf_sz ) /* out of bounds */
702 258 : & fd_ulong_is_aligned( dyn_off, 8UL ) /* misaligned */
703 258 : & fd_ulong_is_aligned( dyn_sz, sizeof(fd_elf64_dyn) ) /* misaligned sz */ );
704 :
705 258 : loader->dyn_off = (uint)dyn_off;
706 258 : loader->dyn_cnt = (uint)(dyn_sz / sizeof(fd_elf64_dyn));
707 258 : return 0;
708 258 : }
709 :
710 : /* Missing or invalid PT_DYNAMIC and missing SHT_DYNAMIC, skip. */
711 81 : return 0;
712 339 : }
713 :
714 : static int
715 : fd_sbpf_load_dynamic( fd_sbpf_loader_t * loader,
716 : fd_sbpf_elf_t const * elf,
717 3678 : ulong elf_sz ) {
718 :
719 3678 : fd_elf64_shdr const * shdrs = (fd_elf64_shdr const *)( elf->bin + elf->ehdr.e_shoff );
720 :
721 : /* Skip if no dynamic table was found */
722 :
723 3678 : if( !loader->dyn_cnt ) return 0;
724 :
725 : /* Walk dynamic table */
726 :
727 3588 : fd_elf64_dyn const * dyn = (fd_elf64_dyn const *)( elf->bin + loader->dyn_off );
728 3588 : ulong const dyn_cnt = loader->dyn_cnt;
729 :
730 109248 : for( ulong i=0; i<dyn_cnt; i++ ) {
731 109161 : if( FD_UNLIKELY( dyn[i].d_tag==FD_ELF_DT_NULL ) ) break;
732 :
733 105660 : ulong d_val = dyn[i].d_un.d_val;
734 105660 : switch( dyn[i].d_tag ) {
735 3240 : case FD_ELF_DT_REL: loader->dt_rel =d_val; break;
736 3294 : case FD_ELF_DT_RELENT: loader->dt_relent=d_val; break;
737 3294 : case FD_ELF_DT_RELSZ: loader->dt_relsz =d_val; break;
738 3411 : case FD_ELF_DT_SYMTAB: loader->dt_symtab=d_val; break;
739 105660 : }
740 105660 : }
741 :
742 : /* Load dynamic symbol table */
743 :
744 3588 : if( loader->dt_symtab ) {
745 : /* Search for dynamic symbol table
746 : FIXME unfortunate bounded O(n^2) -- could convert to binary search */
747 :
748 : /* FIXME this could be clobbered by relocations, causing strict
749 : aliasing violations */
750 :
751 3411 : fd_elf64_shdr const * shdr_dynsym = NULL;
752 :
753 14796 : for( ulong i=0; i<elf->ehdr.e_shnum; i++ ) {
754 14796 : if( shdrs[ i ].sh_addr == loader->dt_symtab ) {
755 : /* TODO: verify this ... */
756 : /* Check section type */
757 3411 : uint sh_type = shdrs[ i ].sh_type;
758 : // https://github.com/solana-labs/rbpf/blob/v0.8.5/src/elf_parser/mod.rs#L500
759 3411 : REQUIRE( (sh_type==FD_ELF_SHT_SYMTAB) | (sh_type==FD_ELF_SHT_DYNSYM) );
760 :
761 2835 : shdr_dynsym = &shdrs[ i ];
762 2835 : break;
763 3411 : }
764 14796 : }
765 2835 : REQUIRE( shdr_dynsym );
766 :
767 : /* Check if out of bounds or misaligned */
768 :
769 2835 : ulong sh_offset = shdr_dynsym->sh_offset;
770 2835 : ulong sh_size = shdr_dynsym->sh_size;
771 :
772 2835 : REQUIRE( ( sh_offset+sh_size>=sh_offset )
773 2835 : & ( sh_offset+sh_size<=elf_sz )
774 2835 : & ( fd_ulong_is_aligned( sh_offset, 8UL ) )
775 2835 : & ( sh_size % sizeof (fd_elf64_sym) == 0UL ) );
776 :
777 2835 : loader->dynsym_off = (uint)sh_offset;
778 2835 : loader->dynsym_cnt = (uint)(sh_size/sizeof(fd_elf64_sym));
779 2835 : }
780 :
781 3012 : return 0;
782 3588 : }
783 :
784 : /* ELF Dynamic Relocations *********************************************
785 :
786 : ### Summary
787 :
788 : The sBPF ELF loader provides a limited dynamic relocation mechanism
789 : to fix up Clang-generated shared objects for execution in an sBPF VM.
790 :
791 : The relocation types themselves violate the eBPF and ELF specs in
792 : various ways. In short, the relocation table (via DT_REL) is used to
793 : shift program code from zero-based addressing to the MM_PROGRAM
794 : segment in the VM memory map (at 0x1_0000_0000).
795 :
796 : As part of the Solana VM protocol it abides by strict determinism
797 : requirements. This sadly means that we will have to replicate all
798 : edge cases and bugs in the Solana Labs ELF loader.
799 :
800 : Three relocation types are currently supported:
801 : - R_BPF_64_64: Sets an absolute address of a symbol as the
802 : 64-bit immediate field of an lddw instruction
803 : - R_BPF_64_RELATIVE: Adds MM_PROGRAM_START (0x1_0000_0000) to ...
804 : a) ... the 64-bit imm field of an lddw instruction (if in text)
805 : b) ... a 64-bit integer (if not in text section)
806 : - R_BPF_64_32: Sets the 32-bit immediate field of a call
807 : instruction to ...
808 : a) the ID of a local function (Murmur3 hash of function PC address)
809 : b) the ID of a syscall
810 :
811 : Obviously invalid relocations (e.g. out-of-bounds of ELF file or
812 : unsupported reloc type) raise an error.
813 : Relocations that would corrupt ELF data structures are silently
814 : ignored (using the fd_sbpf_reloc_mask mechanism).
815 :
816 : ### History
817 :
818 : The use of relocations is technically redundant, as the Solana VM
819 : memory map has been hardcoded in program runtime v1 (so far the only
820 : runtime). However, virtually all deployed programs as of April 2023
821 : are position-independent shared objects and make heavy use of such
822 : relocations.
823 :
824 : Relocations in the Solana VM have a complicated history. Over the
825 : course of years, multiple protocol bugs have been added and fixed.
826 : The ELF loader needs to handle all these edge cases to avoid breaking
827 : "userspace". I.e. any deployed programs which might be immutable
828 : must continue to function.
829 :
830 : While this complex logic will probably stick around for the next few
831 : years, the Solana protocol is getting increasingly restrictive for
832 : newly deployed ELFs. Another proposed change is upgrading to
833 : position-dependent binaries without any dynamic relocations. */
834 :
835 : /* R_BPF_64_64 relocates an absolute address into the extended imm field
836 : of an lddw-form instruction. (Two instruction slots, low 32 bits in
837 : first immediate field, high 32 bits in second immediate field)
838 :
839 : Bits 0..32 32..64 64..96 96..128
840 : [ ... ] [ IMM_LO ] [ ... ] [ IMM_HI ] */
841 :
842 : static int
843 : fd_sbpf_r_bpf_64_64( fd_sbpf_loader_t const * loader,
844 : fd_sbpf_elf_t const * elf,
845 : ulong elf_sz,
846 : uchar * rodata,
847 : fd_sbpf_elf_info_t const * info,
848 123 : fd_elf64_rel const * rel ) {
849 :
850 123 : (void)info;
851 :
852 123 : uint r_sym = FD_ELF64_R_SYM( rel->r_info );
853 123 : ulong r_offset = rel->r_offset;
854 :
855 : /* Bounds check */
856 123 : REQUIRE( ( r_offset+16UL> r_offset )
857 123 : & ( r_offset+16UL<=elf_sz ) );
858 :
859 : /* Offsets of implicit addend (immediate fields) */
860 123 : ulong A_off_lo = r_offset+ 4UL;
861 123 : ulong A_off_hi = r_offset+12UL;
862 :
863 : /* Read implicit addend (imm field of first insn slot) */
864 : // SBF_V2: ulong A_off = is_text ? r_offset+4UL : r_offset;
865 123 : REQUIRE( A_off_lo+4UL<elf_sz );
866 :
867 : /* Lookup symbol */
868 123 : REQUIRE( r_sym < loader->dynsym_cnt );
869 72 : fd_elf64_sym const * dynsyms = (fd_elf64_sym const *)( elf->bin + loader->dynsym_off );
870 72 : fd_elf64_sym const * sym = &dynsyms[ r_sym ];
871 72 : ulong S = sym->st_value;
872 :
873 : /* Relocate */
874 72 : ulong A = FD_LOAD( uint, &rodata[ A_off_lo ] );
875 72 : ulong V = fd_ulong_sat_add( S, A );
876 72 : if( V<FD_SBPF_MM_PROGRAM_ADDR ) V+=FD_SBPF_MM_PROGRAM_ADDR;
877 :
878 : /* Write back */
879 72 : FD_STORE( uint, &rodata[ A_off_lo ], (uint)(V ) );
880 72 : FD_STORE( uint, &rodata[ A_off_hi ], (uint)(V>>32UL) );
881 :
882 72 : return 0;
883 123 : }
884 :
885 : /* R_BPF_64_RELATIVE is almost entirely Solana specific. */
886 :
887 : static int
888 : fd_sbpf_r_bpf_64_relative( fd_sbpf_elf_t const * elf,
889 : ulong elf_sz,
890 : uchar * rodata,
891 : fd_sbpf_elf_info_t const * info,
892 1534899 : fd_elf64_rel const * rel ) {
893 :
894 1534899 : ulong r_offset = rel->r_offset;
895 :
896 : /* Is reloc target in .text section? */
897 1534899 : fd_elf64_shdr const * shdrs = (fd_elf64_shdr const *)( elf->bin + elf->ehdr.e_shoff );
898 1534899 : fd_elf64_shdr const * shdr_text = &shdrs[ info->shndx_text ];
899 1534899 : int is_text = ( ( r_offset >= shdr_text->sh_offset ) &
900 1534899 : ( r_offset < shdr_text->sh_offset +
901 1534899 : shdr_get_loaded_size( shdr_text ) ) );
902 :
903 1534899 : if( is_text ) {
904 : /* If reloc target is in .text, behave like R_BPF_64_64, except:
905 : - R_SYM(r_info) is ignored
906 : - If implicit addend looks like a physical address, make it
907 : a virtual address (by adding a constant offset)
908 :
909 : This relocation type seems to make little sense but is required
910 : for most programs. */
911 :
912 1089933 : REQUIRE( (r_offset+16UL>r_offset) & (r_offset+16UL<=elf_sz) );
913 1089930 : ulong imm_lo_off = r_offset+ 4UL;
914 1089930 : ulong imm_hi_off = r_offset+12UL;
915 :
916 : /* Read implicit addend */
917 1089930 : uint va_lo = FD_LOAD( uint, rodata+imm_lo_off );
918 1089930 : uint va_hi = FD_LOAD( uint, rodata+imm_hi_off );
919 1089930 : ulong va = ( (ulong)va_hi<<32UL ) | va_lo;
920 :
921 1089930 : REQUIRE( va!=0UL );
922 1089930 : va = va<FD_SBPF_MM_PROGRAM_ADDR ? va+FD_SBPF_MM_PROGRAM_ADDR : va;
923 :
924 : /* Write back
925 : Skip bounds check as .text is guaranteed to be writable */
926 1089930 : FD_STORE( uint, rodata+imm_lo_off, (uint)( va ) );
927 1089930 : FD_STORE( uint, rodata+imm_hi_off, (uint)( va>>32UL ) );
928 1089930 : } else {
929 : /* Outside .text do a 64-bit write */
930 :
931 : /* Bounds checks */
932 444966 : REQUIRE( (r_offset+8UL>r_offset) & (r_offset+8UL<=elf_sz) );
933 :
934 : /* Read implicit addend */
935 444966 : ulong va = FD_LOAD( uint, rodata+r_offset+4UL );
936 :
937 : /* Relocate */
938 444966 : va = fd_ulong_sat_add( va, FD_SBPF_MM_PROGRAM_ADDR );
939 :
940 : /* Write back */
941 444966 : FD_STORE( ulong, rodata+r_offset, va );
942 444966 : }
943 :
944 1534896 : return 0;
945 1534899 : }
946 :
947 : static int
948 : fd_sbpf_r_bpf_64_32( fd_sbpf_loader_t const * loader,
949 : fd_sbpf_elf_t const * elf,
950 : ulong elf_sz,
951 : uchar * rodata,
952 : fd_sbpf_elf_info_t const * info,
953 441723 : fd_elf64_rel const * rel ) {
954 :
955 441723 : uint r_sym = FD_ELF64_R_SYM( rel->r_info );
956 441723 : ulong r_offset = rel->r_offset;
957 :
958 : /* Lookup symbol */
959 441723 : REQUIRE( r_sym < loader->dynsym_cnt );
960 441723 : fd_elf64_shdr const * shdrs = (fd_elf64_shdr const *)( elf->bin + elf->ehdr.e_shoff );
961 441723 : fd_elf64_sym const * dynsyms = (fd_elf64_sym const *)( elf->bin + loader->dynsym_off );
962 441723 : fd_elf64_sym const * sym = &dynsyms[ r_sym ];
963 441723 : ulong S = sym->st_value;
964 :
965 : /* Verify .dynstr (TODO can we lift this out of the reloc handler?) */
966 441723 : REQUIRE( info->shndx_dynstr > 0 );
967 441723 : REQUIRE( shdrs[ info->shndx_dynstr ].sh_type == FD_ELF_SHT_STRTAB );
968 :
969 : /* Verify symbol name */
970 441723 : ulong name_len;
971 441723 : char const * name = check_cstr( elf->bin + info->dynstr_off, info->dynstr_sz, sym->st_name, FD_SBPF_SYM_NAME_SZ_MAX-1UL, &name_len );
972 441723 : REQUIRE( name );
973 :
974 : /* Value to write into relocated field */
975 441723 : uint V;
976 :
977 441723 : int is_func_call = ( FD_ELF64_ST_TYPE( sym->st_info ) == FD_ELF_STT_FUNC )
978 441723 : & ( S!=0UL );
979 441723 : if( is_func_call ) {
980 : /* Check whether function call is in virtual memory range of text section. */
981 3837 : fd_elf64_shdr const * shdr_text = &shdrs[ info->shndx_text ];
982 3837 : ulong sh_addr = shdr_text->sh_addr;
983 3837 : ulong sh_size = shdr_text->sh_size;
984 3837 : REQUIRE( (S>=sh_addr) & (S<sh_addr+sh_size) );
985 :
986 : /* Note: The above check is broken, as sh_size is interpreted as 0
987 : for SHT_NOBITS section. Yet, this is the "correct" loading
988 : behavior according to protocol rules. */
989 :
990 : /* Register function call */
991 3837 : ulong target_pc = (S-sh_addr) / 8UL;
992 :
993 : /* TODO bounds check the target? */
994 :
995 : /* Register new entry */
996 3837 : uint hash;
997 3837 : if( name_len >= 10UL && 0==strncmp( name, "entrypoint", name_len ) ) {
998 : /* Skip insertion of "entrypoint" relocation entries to calldests. This
999 : emulates Solana/Agave's behavior of unregistering these entries before
1000 : registering the entrypoint manually.
1001 : Entrypoint is registered in fd_sbpf_program_load.
1002 : Hash is still applied. */
1003 111 : hash = 0x71e3cf81;
1004 3726 : } else {
1005 3726 : hash = fd_pchash( (uint)target_pc );
1006 3726 : if( FD_LIKELY( target_pc < (info->rodata_sz / 8UL ) ) )
1007 3726 : fd_sbpf_calldests_insert( loader->calldests, target_pc );
1008 3726 : }
1009 :
1010 : /* Check for collision with syscall ID
1011 : https://github.com/solana-labs/rbpf/blob/57139e9e1fca4f01155f7d99bc55cdcc25b0bc04/src/program.rs#L142-L146 */
1012 3837 : REQUIRE( !fd_sbpf_syscalls_query( loader->syscalls, hash, NULL ) );
1013 3837 : V = (uint)hash;
1014 437886 : } else {
1015 : /* FIXME Should cache Murmur hashes.
1016 : If max ELF size is 10MB, can fit about 640k relocs.
1017 : Each reloc could point to a symbol with the same st_name,
1018 : which results in 640MB hash input data without caching. */
1019 437886 : uint hash = fd_murmur3_32( name, name_len, 0UL );
1020 : /* Ensure that requested syscall ID exists only when deploying
1021 : https://github.com/solana-labs/rbpf/blob/v0.8.0/src/elf.rs#L1097 */
1022 437886 : if ( FD_UNLIKELY( loader->elf_deploy_checks ) ) {
1023 8295 : REQUIRE( fd_sbpf_syscalls_query( loader->syscalls, hash, NULL ) );
1024 8295 : }
1025 :
1026 437790 : V = hash;
1027 437790 : }
1028 :
1029 : /* Bounds checks */
1030 441627 : REQUIRE( (r_offset+8UL>r_offset) & (r_offset+8UL<=elf_sz) );
1031 441567 : ulong A_off = r_offset+4UL;
1032 :
1033 : /* Apply relocation */
1034 441567 : FD_STORE( uint, rodata+A_off, V );
1035 :
1036 441567 : return 0;
1037 441627 : }
1038 :
1039 : static int
1040 : fd_sbpf_apply_reloc( fd_sbpf_loader_t const * loader,
1041 : fd_sbpf_elf_t const * elf,
1042 : ulong elf_sz,
1043 : uchar * rodata,
1044 : fd_sbpf_elf_info_t const * info,
1045 1976802 : fd_elf64_rel const * rel ) {
1046 1976802 : switch( FD_ELF64_R_TYPE( rel->r_info ) ) {
1047 123 : case FD_ELF_R_BPF_64_64:
1048 123 : return fd_sbpf_r_bpf_64_64 ( loader, elf, elf_sz, rodata, info, rel );
1049 1534899 : case FD_ELF_R_BPF_64_RELATIVE:
1050 1534899 : return fd_sbpf_r_bpf_64_relative( elf, elf_sz, rodata, info, rel );
1051 441723 : case FD_ELF_R_BPF_64_32:
1052 441723 : return fd_sbpf_r_bpf_64_32 ( loader, elf, elf_sz, rodata, info, rel );
1053 57 : default:
1054 57 : ERR( FD_SBPF_ERR_INVALID_ELF );
1055 1976802 : }
1056 1976802 : }
1057 :
1058 : /* fd_sbpf_hash_calls converts local call instructions in the "LLVM
1059 : form" (immediate is a program counter offset) to eBPF form (immediate
1060 : is a hash of the target program counter). Corresponds to
1061 : fixup_relative calls in solana-labs/rbpf.
1062 :
1063 : Assumes that the text section range exists, is within bounds, and
1064 : does not overlap with the ELF file header, program header table, or
1065 : section header table. */
1066 :
1067 : static int
1068 : fd_sbpf_hash_calls( fd_sbpf_loader_t * loader,
1069 : fd_sbpf_program_t * prog,
1070 3102 : fd_sbpf_elf_t const * elf ) {
1071 :
1072 3102 : fd_elf64_shdr const * shdrs = (fd_elf64_shdr const *)( elf->bin + elf->ehdr.e_shoff );
1073 3102 : fd_sbpf_elf_info_t * info = &prog->info;
1074 3102 : uchar * rodata = prog->rodata;
1075 :
1076 3102 : fd_elf64_shdr const * shtext = &shdrs[ info->shndx_text ];
1077 3102 : fd_sbpf_calldests_t * calldests = loader->calldests;
1078 :
1079 3102 : uchar * ptr = rodata + shtext->sh_offset;
1080 3102 : ulong insn_cnt = shdr_get_loaded_size( shtext ) / 8UL;
1081 :
1082 55483380 : for( ulong i=0; i<insn_cnt; i++, ptr+=8UL ) {
1083 55480389 : ulong insn = *((ulong *) ptr);
1084 :
1085 : /* Check for call instruction. If immediate is UINT_MAX, assume
1086 : that compiler generated a relocation instead. */
1087 55480389 : ulong opc = insn & 0xFF;
1088 55480389 : int imm = (int)(insn >> 32UL);
1089 55480389 : if( (opc!=0x85) | (imm==-1) )
1090 51866913 : continue;
1091 :
1092 : /* Mark function call destination */
1093 3613476 : long target_pc_s;
1094 3613476 : REQUIRE( 0==__builtin_saddl_overflow( (long)i+1L, imm, &target_pc_s ) );
1095 3613476 : ulong target_pc = (ulong)target_pc_s;
1096 3613476 : REQUIRE( target_pc<insn_cnt ); /* bounds check target */
1097 :
1098 3613365 : fd_sbpf_calldests_insert( calldests, target_pc );
1099 :
1100 : /* Replace immediate with hash */
1101 3613365 : uint pc_hash = fd_pchash( (uint)target_pc );
1102 : /* Check for collision with syscall ID
1103 : https://github.com/solana-labs/rbpf/blob/57139e9e1fca4f01155f7d99bc55cdcc25b0bc04/src/program.rs#L142-L146 */
1104 3613365 : REQUIRE( !fd_sbpf_syscalls_query( loader->syscalls, pc_hash, NULL ) );
1105 :
1106 3613365 : FD_STORE( uint, ptr+4UL, pc_hash );
1107 3613365 : }
1108 :
1109 2991 : return 0;
1110 3102 : }
1111 :
1112 : static int
1113 : fd_sbpf_relocate( fd_sbpf_loader_t const * loader,
1114 : fd_sbpf_elf_t const * elf,
1115 : ulong elf_sz,
1116 : uchar * rodata,
1117 2991 : fd_sbpf_elf_info_t const * info ) {
1118 :
1119 2991 : ulong const dt_rel = loader->dt_rel;
1120 2991 : ulong const dt_relent = loader->dt_relent;
1121 2991 : ulong const dt_relsz = loader->dt_relsz;
1122 :
1123 : /* Skip relocation if DT_REL is missing */
1124 :
1125 2991 : if( dt_rel == 0UL ) return 0;
1126 :
1127 : /* Validate reloc table params */
1128 :
1129 2616 : REQUIRE( dt_relent==sizeof(fd_elf64_rel) );
1130 2610 : REQUIRE( dt_relsz !=0UL );
1131 2610 : REQUIRE( (dt_relsz % sizeof(fd_elf64_rel))==0UL );
1132 :
1133 : /* Resolve DT_REL virtual address to file offset
1134 : First, attempt to find segment containing DT_REL */
1135 :
1136 2610 : ulong rel_off = ULONG_MAX;
1137 :
1138 2610 : fd_elf64_phdr const * phdrs = (fd_elf64_phdr const *)( elf->bin + elf->ehdr.e_phoff );
1139 2610 : ulong rel_phnum;
1140 4506 : for( rel_phnum=0; rel_phnum < elf->ehdr.e_phnum; rel_phnum++ ) {
1141 4446 : ulong va_lo = phdrs[ rel_phnum ].p_vaddr;
1142 4446 : ulong va_hi = phdrs[ rel_phnum ].p_memsz + va_lo;
1143 4446 : REQUIRE( va_hi>=va_lo );
1144 4446 : if( (dt_rel>=va_lo) & (dt_rel<va_hi) ) {
1145 : /* Found */
1146 2550 : ulong va_off = dt_rel - va_lo;
1147 2550 : ulong pa_lo = phdrs[ rel_phnum ].p_offset + va_off;
1148 : /* Overflow checks */
1149 2550 : REQUIRE( (va_off<=dt_rel)
1150 2550 : & (pa_lo >=va_off)
1151 2550 : & (pa_lo < elf_sz) );
1152 2550 : rel_off = pa_lo;
1153 2550 : break;
1154 2550 : }
1155 4446 : }
1156 :
1157 : /* DT_REL not contained in any segment. Fallback to section header
1158 : table for finding first dynamic reloc section. */
1159 :
1160 2610 : if( rel_phnum == elf->ehdr.e_phnum ) {
1161 60 : fd_elf64_shdr const * shdrs = (fd_elf64_shdr const *)( elf->bin + elf->ehdr.e_shoff );
1162 60 : ulong rel_shnum;
1163 462 : for( rel_shnum=0; rel_shnum < elf->ehdr.e_shnum; rel_shnum++ )
1164 462 : if( shdrs[ rel_shnum ].sh_addr==dt_rel )
1165 60 : break;
1166 60 : REQUIRE( rel_shnum < elf->ehdr.e_shnum );
1167 60 : rel_off = shdrs[ rel_shnum ].sh_offset;
1168 60 : }
1169 :
1170 2610 : REQUIRE( fd_ulong_is_aligned( rel_off, 8UL ) );
1171 2610 : REQUIRE( (rel_off < elf_sz)
1172 2610 : & (dt_relsz <= elf_sz)
1173 2610 : & ((rel_off+dt_relsz) <= elf_sz) );
1174 :
1175 : /* Load section and reloc tables
1176 : Assume section header already validated at this point */
1177 :
1178 2610 : fd_elf64_rel const * rel = (fd_elf64_rel const *)( elf->bin + rel_off );
1179 2610 : ulong rel_cnt = dt_relsz/sizeof(fd_elf64_rel);
1180 :
1181 : /* Apply each reloc */
1182 :
1183 1979145 : for( ulong i=0; i<rel_cnt; i++ ) {
1184 1976802 : int res = fd_sbpf_apply_reloc( loader, elf, elf_sz, rodata, info, &rel[ i ] );
1185 1976802 : if( res!=0 ) return res;
1186 1976802 : }
1187 :
1188 2343 : return 0;
1189 2610 : }
1190 :
1191 : static int
1192 : fd_sbpf_zero_rodata( fd_sbpf_elf_t * elf,
1193 : uchar * rodata,
1194 2718 : fd_sbpf_elf_info_t const * info ) {
1195 :
1196 2718 : fd_elf64_shdr const * shdrs = (fd_elf64_shdr const *)( elf->bin + elf->ehdr.e_shoff );
1197 :
1198 : /* memset gaps between sections to zero.
1199 : Assume section sh_addrs are monotonically increasing.
1200 : Assume section virtual address ranges equal physical address ranges.
1201 : Assume ranges are not overflowing. */
1202 : /* FIXME match Solana more closely here */
1203 :
1204 2718 : ulong cursor = 0UL;
1205 33453 : for( ulong i=0; i<elf->ehdr.e_shnum; i++ ) {
1206 30735 : if( !( info->loaded_sections[ i>>6UL ] & (1UL<<(i&63UL)) ) ) continue;
1207 :
1208 9642 : fd_elf64_shdr const * shdr = &shdrs[ i ];
1209 :
1210 : /* NOBITS sections are included in rodata, but may have invalid
1211 : offsets, thus we can't trust the shdr->sh_offset field. */
1212 9642 : if( FD_UNLIKELY( shdr->sh_type==FD_ELF_SHT_NOBITS ) ) continue;
1213 :
1214 9603 : ulong off = shdr->sh_offset;
1215 9603 : ulong sz = shdr->sh_size;
1216 9603 : assert( cursor<=off ); /* Invariant: Monotonically increasing offsets */
1217 0 : assert( off+sz>=off ); /* Invariant: No integer overflow */
1218 0 : assert( off+sz<=info->rodata_sz ); /* Invariant: No buffer overflow */
1219 :
1220 : /* Fill gap with zeros */
1221 0 : ulong gap = off - cursor;
1222 9603 : fd_memset( rodata+cursor, 0, gap );
1223 :
1224 9603 : cursor = off+sz;
1225 9603 : }
1226 :
1227 2718 : fd_memset( rodata+cursor, 0, info->rodata_sz - cursor );
1228 :
1229 2718 : return 0;
1230 2718 : }
1231 :
1232 : int
1233 : fd_sbpf_program_load( fd_sbpf_program_t * prog,
1234 : void const * _bin,
1235 : ulong elf_sz,
1236 : fd_sbpf_syscalls_t * syscalls,
1237 3678 : int elf_deploy_checks ) {
1238 3678 : fd_sbpf_loader_seterr( 0, 0 );
1239 :
1240 3678 : int err;
1241 3678 : fd_sbpf_elf_t * elf = (fd_sbpf_elf_t *)_bin;
1242 :
1243 3678 : fd_sbpf_loader_t loader = {
1244 3678 : .calldests = prog->calldests,
1245 3678 : .syscalls = syscalls,
1246 :
1247 3678 : .dyn_off = 0U,
1248 3678 : .dyn_cnt = 0U,
1249 :
1250 3678 : .dt_rel = 0UL,
1251 3678 : .dt_relent = 0UL,
1252 3678 : .dt_relsz = 0UL,
1253 3678 : .dt_symtab = 0UL,
1254 :
1255 3678 : .dynsym_off = 0U,
1256 3678 : .dynsym_cnt = 0U,
1257 3678 : .elf_deploy_checks = elf_deploy_checks
1258 3678 : };
1259 :
1260 : /* Find dynamic section */
1261 3678 : if( FD_UNLIKELY( (err=fd_sbpf_find_dynamic( &loader, elf, elf_sz, &prog->info ))!=0 ) )
1262 0 : return err;
1263 :
1264 : /* Load dynamic section */
1265 3678 : if( FD_UNLIKELY( (err=fd_sbpf_load_dynamic( &loader, elf, elf_sz ))!=0 ) )
1266 576 : return err;
1267 :
1268 : /* Register entrypoint to calldests. */
1269 3102 : fd_sbpf_calldests_insert( prog->calldests, prog->entry_pc );
1270 :
1271 : /* Copy rodata segment */
1272 3102 : fd_memcpy( prog->rodata, elf->bin, prog->info.rodata_footprint );
1273 :
1274 : /* Convert calls with PC relative immediate to hashes */
1275 3102 : if( FD_UNLIKELY( (err=fd_sbpf_hash_calls ( &loader, prog, elf ))!=0 ) )
1276 111 : return err;
1277 :
1278 : /* Apply relocations */
1279 2991 : if( FD_UNLIKELY( (err=fd_sbpf_relocate ( &loader, elf, elf_sz, prog->rodata, &prog->info ))!=0 ) )
1280 273 : return err;
1281 :
1282 : /* Create read-only segment */
1283 2718 : if( FD_UNLIKELY( (err=fd_sbpf_zero_rodata( elf, prog->rodata, &prog->info ))!=0 ) )
1284 0 : return err;
1285 :
1286 2718 : return 0;
1287 2718 : }
1288 :
1289 : #undef ERR
1290 : #undef FAIL
1291 : #undef REQUIRE
|
