Line data Source code
1 : #include "fd_bpf_loader_serialization.h"
2 : #include "../fd_borrowed_account.h"
3 : #include "../fd_runtime.h"
4 :
5 : /* This file is responsible for serializing and deserializing
6 : the input region of the BPF virtual machine. The input region contains
7 : instruction information, account metadata, and account data. The high level
8 : format is as follows:
9 :
10 : [ account 1 metadata, account 1 data, account 2 metadata, account 2 data, ...,
11 : account N metadata, account N data, instruction info. ]
12 :
13 : This format by no means comprehensive, but it should give an idea of how
14 : the input region is laid out. When direct mapping is not enabled, the input
15 : region is stored as a single contiguous buffer. This buffer in the host
16 : address space is then mapped to the VM virtual address space (the range
17 : starting with 0x400...). This means to serialize into the input region, we
18 : need to copy in the account metadata and account data into the buffer for
19 : each account. Everything must get copied out after execution is complete.
20 : A consequence of this is that a memcpy for the account data is required
21 : for each serialize and deserialize operation: this can potentially become
22 : expensive if there are many accounts and many nested CPI calls. Also, the
23 : entire memory region is treated as writable even though many accounts are
24 : read-only. This means that for all read-only accounts, a memcmp must be done
25 : while deserializing to make sure that the account (meta)data has not changed.
26 :
27 : Direct mapping offers a solution to this by introducing a more sophisticated
28 : memory translation protocol. Now the account data is not copied into a single
29 : contiguous buffer, but instead a borrowed account's data is directly mapped
30 : into the VM's virtual address space. The host memory for the input region is
31 : now represented by a list of fragmented memory regions. These sub regions
32 : also have different write permissions. This should solve the problem of
33 : having to memcpy/memcmp account data regions (which can be up to 10MiB each).
34 : There is some nuance to this, as the account data can be resized. This means
35 : that memcpys for account data regions can't totally be avoided.
36 :
37 : SERIALIZATION BEHAVIOR
38 : ==========================================
39 :
40 : This implementation supports three distinct serialization modes based on two
41 : feature flags: stricter_abi_and_runtime_constraints and
42 : account_data_direct_mapping.
43 :
44 : MODE 1
45 : --------------------------------------
46 : stricter_abi_and_runtime_constraints = false
47 : account_data_direct_mapping = false
48 :
49 : Memory Layout:
50 : - Single contiguous buffer in host memory
51 : - Buffer contains: [metadata1, data1, realloc_buffer1, metadata2, data2,
52 : realloc_buffer2, ..., metadataN, dataN, realloc_bufferN, instruction_info]
53 : - Each account gets: original data + MAX_PERMITTED_DATA_INCREASE (10KiB)
54 : - Padding added to maintain 16-byte alignment between accounts
55 : - Entire buffer is writable
56 :
57 : Memory Regions:
58 : - The entire input region buffer is mapped as one contiguous VM address
59 : space region
60 :
61 : Serialization Process:
62 : - Account data is memcpy'd into the buffer
63 : - 10KiB realloc buffer is zeroed out and appended after each account's data
64 : - Alignment padding is zeroed and added after realloc buffer
65 :
66 : Deserialization Process:
67 : - Account data must be memcpy'd back from buffer to borrowed account
68 : - For writable accounts: always copy data back
69 : - For read-only accounts: memcmp to verify data unchanged, error if modified
70 : - Account resizing allowed if account permissions permit it
71 :
72 : MODE 2
73 : -------------------------------------------
74 : stricter_abi_and_runtime_constraints = true
75 : account_data_direct_mapping = false
76 :
77 : Memory Layout:
78 : - Still uses a single contiguous buffer, but organized into fragmented
79 : regions.
80 : - Each account now has separate regions for metadata and data+realloc.
81 : - Buffer contains: [metadata1, data1+realloc1, metadata2, data2+realloc2, ...,
82 : metadataN, dataN+reallocN, instruction_info].
83 : - Each metadata region and data region tracked separately in
84 : input_mem_regions.
85 :
86 : Memory Regions:
87 : - For each account:
88 : * Region 0: Account metadata (writable)
89 : * Region 1: Account data + realloc space (writable if account is writable)
90 : - If the account is owned by the deprecated loader, no realloc region is
91 : created as the deprecated loader does not support resizing accounts.
92 :
93 : Serialization:
94 : - Account metadata serialized first, added as a memory region.
95 : - Account data memcpy'd into buffer - not directly mapped.
96 : - 10KiB realloc buffer zeroed and appended (not direct mapped).
97 : - Data region created pointing to copied data in buffer.
98 :
99 : MODE 3: Direct Mapping (requires stricter_abi_and_runtime_constraints)
100 : -----------------------------------------------
101 : stricter_abi_and_runtime_constraints = true
102 : account_data_direct_mapping = true
103 :
104 : This is very similar to stricter_abi_and_runtime_constraints, but account
105 : data is NOT copied into the input region buffer.
106 :
107 : Instead, the data region points directly to the staging area for the
108 : account in the transaction account's data. This staging area has enough
109 : space to hold the account data and the realloc buffer. Changes to this
110 : staging area will be written back to the account database in transaction
111 : finalization.
112 : */
113 :
114 : /* Add a new memory region to represent the input region. All of the memory
115 : regions here have sorted virtual addresses. These regions may or may not
116 : correspond to an account's data region. If it corresponds to metadata,
117 : the pubkey for the region will be NULL. */
118 : static void
119 : new_input_mem_region( fd_vm_input_region_t * input_mem_regions,
120 : uint * input_mem_regions_cnt,
121 : const uchar * buffer,
122 : ulong region_sz,
123 : ulong address_space_reserved,
124 : ulong padding,
125 : uchar is_writable,
126 0 : ulong acc_region_meta_idx ) {
127 :
128 : /* The start vaddr of the new region should be equal to start of the previous
129 : region added to the address space reserved for the region. */
130 0 : ulong vaddr_offset = *input_mem_regions_cnt==0UL ? 0UL : input_mem_regions[ *input_mem_regions_cnt-1U ].vaddr_offset +
131 0 : input_mem_regions[ *input_mem_regions_cnt-1U ].address_space_reserved;
132 0 : input_mem_regions[ *input_mem_regions_cnt ].is_writable = is_writable;
133 0 : input_mem_regions[ *input_mem_regions_cnt ].haddr = (ulong)buffer;
134 0 : input_mem_regions[ *input_mem_regions_cnt ].region_sz = (uint)region_sz;
135 0 : input_mem_regions[ *input_mem_regions_cnt ].address_space_reserved = address_space_reserved;
136 0 : input_mem_regions[ *input_mem_regions_cnt ].vaddr_offset = vaddr_offset;
137 0 : input_mem_regions[ *input_mem_regions_cnt ].padding = padding;
138 0 : input_mem_regions[ *input_mem_regions_cnt ].acc_region_meta_idx = acc_region_meta_idx;
139 0 : (*input_mem_regions_cnt)++;
140 0 : }
141 :
142 : /* https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L127-L189 */
143 : /* This function handles casing for direct mapping being enabled as well as if
144 : the alignment is being stored. In the case where direct mapping is not
145 : enabled, we copy in the account data and a 10KiB buffer into the input region.
146 : These both go into the same memory buffer. However, when direct mapping is
147 : enabled, the account data and resizing buffers are represented by two
148 : different memory regions. In both cases, padding is used to maintain 8 byte
149 : alignment. If alignment is not required, then a resizing buffer is not used
150 : as the deprecated loader doesn't allow for resizing accounts. */
151 : static ulong
152 : write_account( fd_borrowed_account_t * account,
153 : uchar instr_acc_idx,
154 : uchar * * serialized_params,
155 : uchar * * serialized_params_start,
156 : fd_vm_input_region_t * input_mem_regions,
157 : uint * input_mem_regions_cnt,
158 : ulong padding,
159 : fd_vm_acc_region_meta_t * acc_region_metas,
160 : int is_loader_v1,
161 : int stricter_abi_and_runtime_constraints,
162 0 : int direct_mapping ) {
163 :
164 0 : uchar const * data = account ? fd_borrowed_account_get_data( account ) : NULL;
165 0 : ulong dlen = account ? fd_borrowed_account_get_data_len( account ) : 0UL;
166 :
167 0 : acc_region_metas[instr_acc_idx].original_data_len = dlen;
168 0 : acc_region_metas[instr_acc_idx].acct = account->acct;
169 :
170 : /* Legacy behavior: no stricter_abi_and_runtime_constraints (also implies no direct mapping)
171 : https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L131-L140 */
172 0 : if( !stricter_abi_and_runtime_constraints ) {
173 : /* Copy the account data into input region buffer */
174 0 : fd_memcpy( *serialized_params, data, dlen );
175 0 : *serialized_params += dlen;
176 :
177 0 : if( FD_LIKELY( !is_loader_v1 ) ) {
178 : /* Zero out padding bytes and max permitted data increase */
179 0 : ulong align_offset = fd_ulong_align_up( dlen, FD_BPF_ALIGN_OF_U128 ) - dlen;
180 0 : fd_memset( *serialized_params, 0, MAX_PERMITTED_DATA_INCREASE + align_offset );
181 0 : *serialized_params += MAX_PERMITTED_DATA_INCREASE + align_offset;
182 0 : }
183 0 : acc_region_metas[instr_acc_idx].region_idx = UINT_MAX;
184 0 : } else { /* stricter_abi_and_runtime_constraints == true */
185 :
186 : /* Set up account region metadata */
187 0 : acc_region_metas[instr_acc_idx].region_idx = *input_mem_regions_cnt;
188 :
189 : /* First, push on the region for the metadata that has just been serialized.
190 : This function will push the metadata in the serialized_params from
191 : serialized_params_start to serialized_params as a region to the input
192 : memory regions array.
193 :
194 : https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L142 */
195 0 : ulong region_sz = (ulong)(*serialized_params) - (ulong)(*serialized_params_start);
196 0 : new_input_mem_region( input_mem_regions, input_mem_regions_cnt, *serialized_params_start, region_sz, region_sz, padding, 1U, ULONG_MAX );
197 :
198 : /* If direct mapping isn't enabled, then copy the account data in directly
199 : https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L144-L150 */
200 0 : if( !direct_mapping ) {
201 0 : fd_memcpy( *serialized_params, data, dlen );
202 0 : *serialized_params += dlen;
203 0 : if( FD_LIKELY( !is_loader_v1 ) ) {
204 0 : fd_memset( *serialized_params, 0, MAX_PERMITTED_DATA_INCREASE );
205 0 : *serialized_params += MAX_PERMITTED_DATA_INCREASE;
206 0 : }
207 0 : }
208 :
209 : /* Calculate address space reserved for account (data + realloc space)
210 : https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L151-L158 */
211 0 : ulong address_space_reserved = !is_loader_v1 ?
212 0 : fd_ulong_sat_add( dlen, MAX_PERMITTED_DATA_INCREASE ) : dlen;
213 :
214 : /* https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L159-L169 */
215 0 : if( address_space_reserved > 0 ) {
216 0 : int err = 0;
217 0 : uchar is_writable = !!(fd_borrowed_account_can_data_be_changed( account, &err ) && !err);
218 :
219 0 : if( !direct_mapping ) {
220 : /* Create region pointing to the copied data in buffer
221 : https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L160-L164 */
222 0 : uchar * data_start = *serialized_params - address_space_reserved;
223 0 : new_input_mem_region( input_mem_regions, input_mem_regions_cnt, data_start, dlen, address_space_reserved, 0UL, is_writable, instr_acc_idx );
224 0 : } else {
225 : /* Direct mapping: create region pointing directly to account data */
226 0 : new_input_mem_region( input_mem_regions, input_mem_regions_cnt, data, dlen, address_space_reserved, 0UL, is_writable, instr_acc_idx );
227 0 : }
228 0 : }
229 :
230 0 : *serialized_params_start = *serialized_params;
231 :
232 : /* https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L170-L186 */
233 0 : if( FD_LIKELY( !is_loader_v1 ) ) {
234 0 : ulong align_offset = fd_ulong_align_up( dlen, FD_BPF_ALIGN_OF_U128 ) - dlen;
235 0 : if( !direct_mapping ) {
236 : /* If direct mapping is not enabled, we do not align the start of each
237 : region metadata to FD_BPF_ALIGN_OF_U128, but we do align the start
238 : of the actual contents of the metadata region.
239 :
240 : This follows Agave's logic
241 : https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L173-L176 */
242 0 : fd_memset( *serialized_params, 0, align_offset );
243 0 : *serialized_params += align_offset;
244 0 : } else {
245 : /* If direct mapping is enabled, we align the start of each region
246 : metadata to FD_BPF_ALIGN_OF_U128. */
247 0 : fd_memset( *serialized_params, 0, FD_BPF_ALIGN_OF_U128 );
248 0 : *serialized_params += FD_BPF_ALIGN_OF_U128;
249 0 : *serialized_params_start += fd_ulong_sat_sub( FD_BPF_ALIGN_OF_U128, align_offset );
250 0 : }
251 0 : }
252 :
253 0 : return (ulong)(*serialized_params - *serialized_params_start);
254 0 : }
255 :
256 0 : return 0UL;
257 0 : }
258 :
259 : /* https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L466 */
260 : static uchar *
261 : fd_bpf_loader_input_serialize_aligned( fd_exec_instr_ctx_t * ctx,
262 : ulong * sz,
263 : ulong * pre_lens,
264 : fd_vm_input_region_t * input_mem_regions,
265 : uint * input_mem_regions_cnt,
266 : fd_vm_acc_region_meta_t * acc_region_metas,
267 : int stricter_abi_and_runtime_constraints,
268 0 : int direct_mapping ) {
269 0 : fd_pubkey_t * txn_accs = ctx->txn_ctx->account_keys;
270 :
271 0 : uchar acc_idx_seen[256] = {0};
272 0 : ushort dup_acc_idx[256] = {0};
273 :
274 : /* https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L482-L511 */
275 0 : ulong serialized_size = 0UL;
276 0 : serialized_size += sizeof(ulong); // acct_cnt
277 : /* First pass is to calculate size of buffer to allocate */
278 0 : for( ushort i=0; i<ctx->instr->acct_cnt; i++ ) {
279 0 : uchar acc_idx = (uchar)ctx->instr->accounts[i].index_in_transaction;
280 :
281 0 : serialized_size++; // dup byte
282 0 : if( FD_UNLIKELY( acc_idx_seen[acc_idx] ) ) {
283 0 : serialized_size += 7UL; // pad to 64-bit alignment
284 0 : } else {
285 0 : acc_idx_seen[acc_idx] = 1;
286 0 : dup_acc_idx[acc_idx] = i;
287 :
288 : /* Borrow the account without checking the error, as it is guaranteed to exist
289 : https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L244-L257 */
290 0 : fd_guarded_borrowed_account_t view_acc = {0};
291 0 : fd_exec_instr_ctx_try_borrow_instr_account( ctx, i, &view_acc );
292 :
293 0 : ulong acc_data_len = fd_borrowed_account_get_data_len( &view_acc );
294 :
295 0 : serialized_size += sizeof(uchar) // is_signer
296 0 : + sizeof(uchar) // is_writable
297 0 : + sizeof(uchar) // executable
298 0 : + sizeof(uint) // original_data_len
299 0 : + sizeof(fd_pubkey_t) // key
300 0 : + sizeof(fd_pubkey_t) // owner
301 0 : + sizeof(ulong) // lamports
302 0 : + sizeof(ulong) // data len
303 0 : + sizeof(ulong); // rent_epoch
304 : /* https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L499-L505 */
305 0 : if( !(stricter_abi_and_runtime_constraints && direct_mapping) ) {
306 0 : serialized_size += MAX_PERMITTED_DATA_INCREASE + fd_ulong_align_up( acc_data_len, FD_BPF_ALIGN_OF_U128 );
307 0 : } else {
308 0 : serialized_size += FD_BPF_ALIGN_OF_U128;
309 0 : }
310 0 : }
311 0 : }
312 :
313 0 : serialized_size += sizeof(ulong) // data len
314 0 : + ctx->instr->data_sz
315 0 : + sizeof(fd_pubkey_t); // program id
316 :
317 : /* 16-byte aligned buffer:
318 : https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L60
319 : */
320 0 : uchar * serialized_params = fd_spad_alloc(
321 0 : ctx->txn_ctx->spad,
322 0 : FD_RUNTIME_EBPF_HOST_ALIGN,
323 0 : fd_ulong_align_up( serialized_size, FD_RUNTIME_EBPF_HOST_ALIGN ) );
324 0 : uchar * serialized_params_start = serialized_params;
325 0 : uchar * curr_serialized_params_start = serialized_params;
326 :
327 : /* https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L522 */
328 0 : FD_STORE( ulong, serialized_params, ctx->instr->acct_cnt );
329 0 : serialized_params += sizeof(ulong);
330 0 : ulong padding = sizeof(ulong);
331 :
332 : /* Second pass over the account is to serialize into the buffer.
333 : https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L523-L557 */
334 0 : for( ushort i=0; i<ctx->instr->acct_cnt; i++ ) {
335 0 : uchar acc_idx = (uchar)ctx->instr->accounts[i].index_in_transaction;
336 0 : fd_pubkey_t * acc = &txn_accs[acc_idx];
337 :
338 0 : if( FD_UNLIKELY( acc_idx_seen[acc_idx] && dup_acc_idx[acc_idx] != i ) ) {
339 : /* Duplicate. Store 8 byte buffer to maintain alignment but store the
340 : account index in the first byte.
341 :
342 : https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L551-L555 */
343 0 : FD_STORE( ulong, serialized_params, 0UL );
344 0 : FD_STORE( uchar, serialized_params, (uchar)dup_acc_idx[acc_idx] );
345 0 : serialized_params += sizeof(ulong);
346 0 : padding += sizeof(ulong);
347 0 : } else {
348 : /* https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L526 */
349 0 : FD_STORE( uchar, serialized_params, FD_NON_DUP_MARKER );
350 0 : serialized_params += sizeof(uchar);
351 :
352 : /* Borrow the account without checking the error, as it is guaranteed to exist
353 : https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L244-L257 */
354 0 : fd_guarded_borrowed_account_t view_acc = {0};
355 0 : fd_exec_instr_ctx_try_borrow_instr_account( ctx, i, &view_acc );
356 :
357 : /* https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L525 */
358 0 : fd_account_meta_t const * metadata = fd_borrowed_account_get_acc_meta( &view_acc );
359 :
360 : /* https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L527 */
361 0 : uchar is_signer = (uchar)fd_instr_acc_is_signer_idx( ctx->instr, (uchar)i, NULL );
362 0 : FD_STORE( uchar, serialized_params, is_signer );
363 0 : serialized_params += sizeof(uchar);
364 :
365 : /* https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L528 */
366 0 : uchar is_writable = (uchar)fd_instr_acc_is_writable_idx( ctx->instr, (uchar)i );
367 0 : FD_STORE( uchar, serialized_params, is_writable );
368 0 : serialized_params += sizeof(uchar);
369 :
370 : /* https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L529-L530 */
371 0 : uchar is_executable = (uchar)metadata->executable;
372 0 : FD_STORE( uchar, serialized_params, is_executable );
373 0 : serialized_params += sizeof(uchar);
374 :
375 : /* The original data len field is intentionally NOT populated. */
376 : /* https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L531 */
377 0 : uint padding_0 = 0U;
378 0 : FD_STORE( uint, serialized_params, padding_0 );
379 0 : serialized_params += sizeof(uint);
380 :
381 : /* https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L532 */
382 0 : fd_pubkey_t key = *acc;
383 0 : FD_STORE( fd_pubkey_t, serialized_params, key );
384 0 : serialized_params += sizeof(fd_pubkey_t);
385 :
386 : /* https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L533 */
387 0 : fd_pubkey_t owner = *(fd_pubkey_t *)&metadata->owner;
388 0 : FD_STORE( fd_pubkey_t, serialized_params, owner );
389 0 : serialized_params += sizeof(fd_pubkey_t);
390 :
391 : /* https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L534 */
392 0 : ulong lamports = metadata->lamports;
393 0 : FD_STORE( ulong, serialized_params, lamports );
394 0 : serialized_params += sizeof(ulong);
395 :
396 0 : ulong acc_data_len = metadata->dlen;
397 0 : pre_lens[i] = acc_data_len;
398 :
399 : /* https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L535 */
400 0 : ulong data_len = acc_data_len;
401 0 : FD_STORE( ulong, serialized_params, data_len );
402 0 : serialized_params += sizeof(ulong);
403 :
404 : /* https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L536 */
405 0 : padding = write_account(
406 0 : &view_acc,
407 0 : (uchar)i,
408 0 : &serialized_params,
409 0 : &curr_serialized_params_start,
410 0 : input_mem_regions,
411 0 : input_mem_regions_cnt,
412 0 : padding,
413 0 : acc_region_metas,
414 0 : 0,
415 0 : stricter_abi_and_runtime_constraints,
416 0 : direct_mapping );
417 :
418 : /* https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L537-L541 */
419 0 : FD_STORE( ulong, serialized_params, ULONG_MAX );
420 0 : serialized_params += sizeof(ulong);
421 0 : padding += sizeof(ulong);
422 0 : }
423 :
424 0 : }
425 :
426 : /* https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L558 */
427 0 : ulong instr_data_len = ctx->instr->data_sz;
428 0 : FD_STORE( ulong, serialized_params, instr_data_len );
429 0 : serialized_params += sizeof(ulong);
430 :
431 : /* https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L559 */
432 0 : uchar * instr_data = ctx->instr->data;
433 0 : fd_memcpy( serialized_params, instr_data, instr_data_len );
434 0 : serialized_params += instr_data_len;
435 :
436 : /* https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L560 */
437 0 : FD_STORE( fd_pubkey_t, serialized_params, txn_accs[ctx->instr->program_id] );
438 0 : serialized_params += sizeof(fd_pubkey_t);
439 :
440 : /* Write out the final region. */
441 0 : ulong region_sz = (ulong)(serialized_params - curr_serialized_params_start);
442 0 : new_input_mem_region( input_mem_regions, input_mem_regions_cnt, curr_serialized_params_start,
443 0 : region_sz, region_sz, padding, 1U, ULONG_MAX );
444 :
445 0 : *sz = serialized_size;
446 :
447 0 : return serialized_params_start;
448 0 : }
449 :
450 : /* https://github.com/anza-xyz/agave/blob/v3.0.4/program-runtime/src/serialization.rs#L566-L653 */
451 : static int
452 : fd_bpf_loader_input_deserialize_aligned( fd_exec_instr_ctx_t * ctx,
453 : ulong const * pre_lens,
454 : uchar * buffer,
455 : ulong FD_FN_UNUSED buffer_sz,
456 : int stricter_abi_and_runtime_constraints,
457 0 : int direct_mapping ) {
458 : /* TODO: An optimization would be to skip ahead through non-writable accounts */
459 : /* https://github.com/anza-xyz/agave/blob/v3.0.4/program-runtime/src/serialization.rs#L573 */
460 0 : ulong start = 0UL;
461 :
462 0 : uchar acc_idx_seen[256] = {0};
463 :
464 0 : start += sizeof(ulong); // number of accounts
465 : /* https://github.com/anza-xyz/agave/blob/v3.0.4/program-runtime/src/serialization.rs#L574-L650 */
466 0 : for( ushort i=0; i<ctx->instr->acct_cnt; i++ ) {
467 0 : uchar acc_idx = (uchar)ctx->instr->accounts[i].index_in_transaction;
468 :
469 0 : start++; // position
470 :
471 : /* get the borrowed account
472 : https://github.com/anza-xyz/agave/blob/v3.0.4/program-runtime/src/serialization.rs#L584-L585 */
473 0 : fd_guarded_borrowed_account_t view_acc = {0};
474 0 : FD_TRY_BORROW_INSTR_ACCOUNT_DEFAULT_ERR_CHECK( ctx, i, &view_acc );
475 :
476 0 : if( FD_UNLIKELY( acc_idx_seen[acc_idx] ) ) {
477 : /* https://github.com/anza-xyz/agave/blob/v3.0.4/program-runtime/src/serialization.rs#L582 */
478 0 : start += 7UL;
479 0 : } else {
480 : /* https://github.com/anza-xyz/agave/blob/v3.0.4/program-runtime/src/serialization.rs#L586-L590 */
481 0 : acc_idx_seen[acc_idx] = 1;
482 0 : start += sizeof(uchar) // is_signer
483 0 : + sizeof(uchar) // is_writable
484 0 : + sizeof(uchar) // executable
485 0 : + sizeof(uint) // original_data_len
486 0 : + sizeof(fd_pubkey_t); // key
487 :
488 : /* https://github.com/anza-xyz/agave/blob/v3.0.4/program-runtime/src/serialization.rs#L591-L593 */
489 :
490 0 : fd_pubkey_t * owner = (fd_pubkey_t *)(buffer+start);
491 0 : start += sizeof(fd_pubkey_t); // owner
492 :
493 0 : ulong lamports = FD_LOAD( ulong, buffer+start );
494 0 : if( lamports!=fd_borrowed_account_get_lamports( &view_acc ) ) {
495 0 : int err = fd_borrowed_account_set_lamports( &view_acc, lamports );
496 0 : if( FD_UNLIKELY( err ) ) {
497 0 : return err;
498 0 : }
499 0 : }
500 0 : start += sizeof(ulong); // lamports
501 :
502 0 : ulong post_len = FD_LOAD( ulong, buffer+start );
503 0 : start += sizeof(ulong); // data length
504 :
505 0 : ulong pre_len = pre_lens[i];
506 0 : ulong alignment_offset = fd_ulong_align_up( pre_len, FD_BPF_ALIGN_OF_U128 ) - pre_len;
507 :
508 0 : uchar * post_data = buffer+start;
509 :
510 0 : fd_account_meta_t const * metadata_check = fd_borrowed_account_get_acc_meta( &view_acc );
511 0 : if( FD_UNLIKELY( fd_ulong_sat_sub( post_len, metadata_check->dlen )>MAX_PERMITTED_DATA_INCREASE ||
512 0 : post_len>MAX_PERMITTED_DATA_LENGTH ) ) {
513 0 : return FD_EXECUTOR_INSTR_ERR_INVALID_REALLOC;
514 0 : }
515 :
516 0 : int can_data_be_changed_err = 0;
517 0 : if( !stricter_abi_and_runtime_constraints ) {
518 : /* https://github.com/anza-xyz/agave/blob/v3.0.4/program-runtime/src/serialization.rs#L617-L627 */
519 :
520 : /* https://github.com/anza-xyz/agave/blob/v3.0.4/program-runtime/src/serialization.rs#L618-L620 */
521 0 : if( FD_UNLIKELY( start + post_len > buffer_sz ) ) {
522 0 : return FD_EXECUTOR_INSTR_ERR_INVALID_ARG;
523 0 : }
524 :
525 : /* https://github.com/anza-xyz/agave/blob/v3.0.4/program-runtime/src/serialization.rs#L621-L626 */
526 0 : int can_data_be_resized_err = 0;
527 0 : if( fd_borrowed_account_can_data_be_resized( &view_acc, post_len, &can_data_be_resized_err ) &&
528 0 : fd_borrowed_account_can_data_be_changed( &view_acc, &can_data_be_changed_err ) ) {
529 0 : int set_data_err = fd_borrowed_account_set_data_from_slice( &view_acc, post_data, post_len );
530 0 : if( FD_UNLIKELY( set_data_err ) ) {
531 0 : return set_data_err;
532 0 : }
533 0 : } else {
534 0 : if( FD_UNLIKELY( fd_borrowed_account_get_data_len( &view_acc )!=post_len ||
535 0 : memcmp( fd_borrowed_account_get_data( &view_acc ), post_data, post_len ) ) ) {
536 0 : return can_data_be_resized_err ? can_data_be_resized_err : can_data_be_changed_err;
537 0 : }
538 0 : }
539 :
540 0 : } else if( !direct_mapping && fd_borrowed_account_can_data_be_changed( &view_acc, &can_data_be_changed_err ) ) {
541 : /* https://github.com/anza-xyz/agave/blob/v3.0.4/program-runtime/src/serialization.rs#L629-L631 */
542 0 : if( FD_UNLIKELY( start + post_len > buffer_sz ) ) {
543 0 : return FD_EXECUTOR_INSTR_ERR_INVALID_ARG;
544 0 : }
545 :
546 : /* https://github.com/anza-xyz/agave/blob/v3.0.4/program-runtime/src/serialization.rs#L627-L633 */
547 0 : int set_data_err = fd_borrowed_account_set_data_from_slice( &view_acc, post_data, post_len );
548 0 : if( FD_UNLIKELY( set_data_err ) ) {
549 0 : return set_data_err;
550 0 : }
551 0 : } else if( fd_borrowed_account_get_data_len( &view_acc ) != post_len ) {
552 : /* https://github.com/anza-xyz/agave/blob/v3.0.4/program-runtime/src/serialization.rs#L633-L635 */
553 0 : int set_data_length_err = fd_borrowed_account_set_data_length( &view_acc, post_len );
554 0 : if( FD_UNLIKELY( set_data_length_err ) ) {
555 0 : return set_data_length_err;
556 0 : }
557 0 : }
558 :
559 : /* https://github.com/anza-xyz/agave/blob/v3.0.4/program-runtime/src/serialization.rs#L636-L644 */
560 0 : if( !( stricter_abi_and_runtime_constraints && direct_mapping ) ) {
561 0 : start += fd_ulong_sat_add( MAX_PERMITTED_DATA_INCREASE, fd_ulong_sat_add( pre_len, alignment_offset ) );
562 0 : } else {
563 0 : start += FD_BPF_ALIGN_OF_U128;
564 0 : }
565 :
566 : /* https://github.com/anza-xyz/agave/blob/v3.0.4/program-runtime/src/serialization.rs#L645 */
567 0 : start += sizeof(ulong); // rent epoch
568 0 : if( memcmp( fd_borrowed_account_get_owner( &view_acc ), owner, sizeof(fd_pubkey_t) ) ) {
569 0 : int err = fd_borrowed_account_set_owner( &view_acc, owner );
570 0 : if( FD_UNLIKELY( err ) ) {
571 0 : return err;
572 0 : }
573 0 : }
574 0 : }
575 0 : }
576 :
577 0 : return FD_EXECUTOR_INSTR_SUCCESS;
578 0 : }
579 :
580 : static uchar *
581 : fd_bpf_loader_input_serialize_unaligned( fd_exec_instr_ctx_t * ctx,
582 : ulong * sz,
583 : ulong * pre_lens,
584 : fd_vm_input_region_t * input_mem_regions,
585 : uint * input_mem_regions_cnt,
586 : fd_vm_acc_region_meta_t * acc_region_metas,
587 : int stricter_abi_and_runtime_constraints,
588 0 : int direct_mapping ) {
589 0 : ulong serialized_size = 0UL;
590 0 : fd_pubkey_t const * txn_accs = ctx->txn_ctx->account_keys;
591 :
592 0 : uchar acc_idx_seen[256] = {0};
593 0 : ushort dup_acc_idx[256] = {0};
594 :
595 0 : serialized_size += sizeof(ulong);
596 0 : for( ushort i=0; i<ctx->instr->acct_cnt; i++ ) {
597 0 : uchar acc_idx = (uchar)ctx->instr->accounts[i].index_in_transaction;
598 :
599 0 : serialized_size++; // dup
600 0 : if( FD_UNLIKELY( acc_idx_seen[acc_idx] ) ) {
601 0 : continue;
602 0 : }
603 :
604 0 : acc_idx_seen[acc_idx] = 1;
605 0 : dup_acc_idx[acc_idx] = i;
606 :
607 : /* Borrow the account without checking the error, as it is guaranteed to exist
608 : https://github.com/anza-xyz/agave/blob/v2.1.4/programs/bpf_loader/src/serialization.rs#L225 */
609 0 : fd_guarded_borrowed_account_t view_acc = {0};
610 0 : fd_exec_instr_ctx_try_borrow_instr_account( ctx, i, &view_acc );
611 :
612 0 : ulong acc_data_len = fd_borrowed_account_get_data_len( &view_acc );
613 :
614 0 : pre_lens[i] = acc_data_len;
615 :
616 0 : serialized_size += sizeof(uchar) // is_signer
617 0 : + sizeof(uchar) // is_writable
618 0 : + sizeof(fd_pubkey_t) // key
619 0 : + sizeof(ulong) // lamports
620 0 : + sizeof(ulong) // data_len
621 0 : + sizeof(fd_pubkey_t) // owner
622 0 : + sizeof(uchar) // executable
623 0 : + sizeof(ulong); // rent_epoch
624 0 : if( !(stricter_abi_and_runtime_constraints && direct_mapping) ) {
625 0 : serialized_size += acc_data_len;
626 0 : }
627 0 : }
628 :
629 0 : serialized_size += sizeof(ulong) // instruction data len
630 0 : + ctx->instr->data_sz // instruction data
631 0 : + sizeof(fd_pubkey_t); // program id
632 :
633 : /* 16-byte aligned buffer:
634 : https://github.com/anza-xyz/agave/blob/v2.2.13/programs/bpf_loader/src/serialization.rs#L32
635 : */
636 0 : uchar * serialized_params = fd_spad_alloc( ctx->txn_ctx->spad, FD_RUNTIME_EBPF_HOST_ALIGN, serialized_size );
637 0 : uchar * serialized_params_start = serialized_params;
638 0 : uchar * curr_serialized_params_start = serialized_params;
639 :
640 0 : FD_STORE( ulong, serialized_params, ctx->instr->acct_cnt );
641 0 : serialized_params += sizeof(ulong);
642 0 : ulong padding = sizeof(ulong);
643 :
644 0 : for( ushort i=0; i<ctx->instr->acct_cnt; i++ ) {
645 0 : uchar acc_idx = (uchar)ctx->instr->accounts[i].index_in_transaction;
646 0 : fd_pubkey_t const * acc = &txn_accs[acc_idx];
647 :
648 0 : if( FD_UNLIKELY( acc_idx_seen[acc_idx] && dup_acc_idx[acc_idx] != i ) ) {
649 : // Duplicate
650 0 : FD_STORE( uchar, serialized_params, (uchar)dup_acc_idx[acc_idx] );
651 0 : serialized_params += sizeof(uchar);
652 0 : padding += sizeof(uchar);
653 0 : } else {
654 0 : FD_STORE( uchar, serialized_params, FD_NON_DUP_MARKER );
655 0 : serialized_params += sizeof(uchar);
656 :
657 : /* Borrow the account without checking the error, as it is guaranteed to exist
658 : https://github.com/anza-xyz/agave/blob/v2.1.4/programs/bpf_loader/src/serialization.rs#L225 */
659 0 : fd_guarded_borrowed_account_t view_acc = {0};
660 0 : fd_exec_instr_ctx_try_borrow_instr_account( ctx, i, &view_acc );
661 :
662 0 : fd_account_meta_t const * metadata = fd_borrowed_account_get_acc_meta( &view_acc );
663 :
664 0 : uchar is_signer = (uchar)fd_instr_acc_is_signer_idx( ctx->instr, (uchar)i, NULL );
665 0 : FD_STORE( uchar, serialized_params, is_signer );
666 0 : serialized_params += sizeof(uchar);
667 :
668 0 : uchar is_writable = (uchar)fd_instr_acc_is_writable_idx( ctx->instr, (uchar)i );
669 0 : FD_STORE( uchar, serialized_params, is_writable );
670 0 : serialized_params += sizeof(uchar);
671 :
672 0 : fd_pubkey_t key = *acc;
673 0 : FD_STORE( fd_pubkey_t, serialized_params, key );
674 0 : serialized_params += sizeof(fd_pubkey_t);
675 :
676 0 : ulong lamports = metadata->lamports;
677 0 : FD_STORE( ulong, serialized_params, lamports );
678 0 : serialized_params += sizeof(ulong);
679 :
680 0 : ulong acc_data_len = metadata->dlen;
681 0 : FD_STORE( ulong, serialized_params, acc_data_len );
682 0 : serialized_params += sizeof(ulong);
683 :
684 0 : padding = write_account( &view_acc, (uchar)i, &serialized_params, &curr_serialized_params_start,
685 0 : input_mem_regions, input_mem_regions_cnt, padding, acc_region_metas, 1,
686 0 : stricter_abi_and_runtime_constraints, direct_mapping );
687 :
688 0 : fd_pubkey_t owner = *(fd_pubkey_t *)&metadata->owner;
689 0 : FD_STORE( fd_pubkey_t, serialized_params, owner );
690 0 : serialized_params += sizeof(fd_pubkey_t);
691 :
692 0 : uchar is_executable = (uchar)metadata->executable;
693 0 : FD_STORE( uchar, serialized_params, is_executable );
694 0 : serialized_params += sizeof(uchar);
695 :
696 0 : FD_STORE( ulong, serialized_params, ULONG_MAX );
697 0 : serialized_params += sizeof(ulong);
698 0 : }
699 0 : }
700 :
701 0 : ulong instr_data_len = ctx->instr->data_sz;
702 0 : FD_STORE( ulong, serialized_params, instr_data_len );
703 0 : serialized_params += sizeof(ulong);
704 :
705 0 : uchar * instr_data = (uchar *)ctx->instr->data;
706 0 : fd_memcpy( serialized_params, instr_data, instr_data_len );
707 0 : serialized_params += instr_data_len;
708 :
709 0 : FD_STORE( fd_pubkey_t, serialized_params, txn_accs[ctx->instr->program_id] );
710 0 : serialized_params += sizeof(fd_pubkey_t);
711 :
712 0 : FD_TEST( serialized_params == serialized_params_start + serialized_size );
713 0 : *sz = serialized_size;
714 :
715 0 : ulong region_sz = (ulong)(serialized_params - curr_serialized_params_start);
716 0 : new_input_mem_region( input_mem_regions, input_mem_regions_cnt, curr_serialized_params_start,
717 0 : region_sz, region_sz, padding, 1U, ULONG_MAX );
718 :
719 0 : return serialized_params_start;
720 0 : }
721 :
722 : /* https://github.com/anza-xyz/agave/blob/v3.0.4/program-runtime/src/serialization.rs#L404 */
723 : static int
724 : fd_bpf_loader_input_deserialize_unaligned( fd_exec_instr_ctx_t * ctx,
725 : ulong const * pre_lens,
726 : uchar * input,
727 : ulong input_sz,
728 : int stricter_abi_and_runtime_constraints,
729 0 : int direct_mapping ) {
730 0 : uchar * input_cursor = input;
731 0 : uchar acc_idx_seen[256] = {0};
732 :
733 0 : input_cursor += sizeof(ulong);
734 :
735 0 : for( ushort i=0; i<ctx->instr->acct_cnt; i++ ) {
736 0 : uchar acc_idx = (uchar)ctx->instr->accounts[i].index_in_transaction;
737 :
738 0 : input_cursor++; /* is_dup */
739 0 : if( FD_UNLIKELY( acc_idx_seen[acc_idx] ) ) {
740 : /* no-op */
741 0 : } else {
742 0 : acc_idx_seen[acc_idx] = 1;
743 0 : input_cursor += sizeof(uchar) + /* is_signer */
744 0 : sizeof(uchar) + /* is_writable */
745 0 : sizeof(fd_pubkey_t); /* key */
746 :
747 : /* https://github.com/anza-xyz/agave/blob/v2.1.4/programs/bpf_loader/src/serialization.rs#L378 */
748 0 : fd_guarded_borrowed_account_t view_acc = {0};
749 0 : FD_TRY_BORROW_INSTR_ACCOUNT_DEFAULT_ERR_CHECK( ctx, i, &view_acc );
750 :
751 0 : ulong lamports = FD_LOAD( ulong, input_cursor );
752 0 : if( fd_borrowed_account_get_acc_meta( &view_acc ) && fd_borrowed_account_get_lamports( &view_acc )!=lamports ) {
753 0 : int err = fd_borrowed_account_set_lamports( &view_acc, lamports );
754 0 : if( FD_UNLIKELY( err ) ) {
755 0 : return err;
756 0 : }
757 0 : }
758 :
759 0 : input_cursor += sizeof(ulong); /* lamports */
760 :
761 0 : ulong post_len = FD_LOAD( ulong, input_cursor );
762 0 : input_cursor += sizeof(ulong); /* data length */
763 :
764 0 : ulong pre_len = pre_lens[i];
765 0 : uchar * post_data = input_cursor;
766 :
767 : /* https://github.com/anza-xyz/agave/blob/v3.0.4/program-runtime/src/serialization.rs#L436-L446 */
768 0 : int can_data_be_changed_err = 0;
769 0 : if( !stricter_abi_and_runtime_constraints ) {
770 0 : int can_data_be_resized_err = 0;
771 0 : if( fd_borrowed_account_can_data_be_resized( &view_acc, pre_len, &can_data_be_resized_err ) &&
772 0 : fd_borrowed_account_can_data_be_changed( &view_acc, &can_data_be_changed_err ) ) {
773 0 : int set_data_err = fd_borrowed_account_set_data_from_slice( &view_acc, post_data, pre_len );
774 0 : if( FD_UNLIKELY( set_data_err ) ) {
775 0 : return set_data_err;
776 0 : }
777 0 : } else if( fd_borrowed_account_get_data_len( &view_acc ) != pre_len ||
778 0 : memcmp( post_data, fd_borrowed_account_get_data( &view_acc ), pre_len ) ) {
779 0 : return can_data_be_resized_err ? can_data_be_resized_err : can_data_be_changed_err;
780 0 : }
781 0 : } else if( !direct_mapping && fd_borrowed_account_can_data_be_changed( &view_acc, &can_data_be_changed_err ) ) {
782 : /* https://github.com/anza-xyz/agave/blob/v3.0.4/program-runtime/src/serialization.rs#L446-L452 */
783 0 : int set_data_err = fd_borrowed_account_set_data_from_slice( &view_acc, post_data, post_len );
784 0 : if( FD_UNLIKELY( set_data_err ) ) {
785 0 : return set_data_err;
786 0 : }
787 0 : } else if( fd_borrowed_account_get_data_len( &view_acc ) != pre_len ) {
788 : /* https://github.com/anza-xyz/agave/blob/v3.0.4/program-runtime/src/serialization.rs#L452-L454 */
789 0 : int set_data_length_err = fd_borrowed_account_set_data_length( &view_acc, pre_len );
790 0 : if( FD_UNLIKELY( set_data_length_err ) ) {
791 0 : return set_data_length_err;
792 0 : }
793 0 : }
794 :
795 : /* https://github.com/anza-xyz/agave/blob/v3.0.4/program-runtime/src/serialization.rs#L455-L457 */
796 0 : if( !( stricter_abi_and_runtime_constraints && direct_mapping ) ) {
797 0 : input_cursor += pre_len;
798 0 : }
799 0 : input_cursor += sizeof(fd_pubkey_t) + /* owner */
800 0 : sizeof(uchar) + /* executable */
801 0 : sizeof(ulong); /* rent_epoch*/
802 0 : }
803 0 : }
804 :
805 0 : if( FD_UNLIKELY( input_cursor>input+input_sz ) ) {
806 0 : return FD_EXECUTOR_INSTR_ERR_INVALID_ARG;
807 0 : }
808 :
809 0 : return 0;
810 0 : }
811 :
812 : /* https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L221 */
813 : int
814 : fd_bpf_loader_input_serialize_parameters( fd_exec_instr_ctx_t * instr_ctx,
815 : ulong * sz,
816 : ulong * pre_lens,
817 : fd_vm_input_region_t * input_mem_regions,
818 : uint * input_mem_regions_cnt,
819 : fd_vm_acc_region_meta_t * acc_region_metas,
820 : int stricter_abi_and_runtime_constraints,
821 : int direct_mapping,
822 : uchar is_deprecated,
823 0 : uchar ** out /* output */ ) {
824 :
825 : /* https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L234-L237 */
826 0 : ulong num_ix_accounts = instr_ctx->instr->acct_cnt;
827 0 : if( FD_UNLIKELY( num_ix_accounts>=FD_INSTR_ACCT_MAX ) ) {
828 0 : return FD_EXECUTOR_INSTR_ERR_MAX_ACCS_EXCEEDED;
829 0 : }
830 :
831 : /* TODO: Like Agave's serialization functions, ours should probably return error codes
832 :
833 : https://github.com/anza-xyz/agave/blob/v2.1.11/programs/bpf_loader/src/serialization.rs#L237-L251 */
834 0 : if( FD_UNLIKELY( is_deprecated ) ) {
835 0 : *out = fd_bpf_loader_input_serialize_unaligned( instr_ctx, sz, pre_lens,
836 0 : input_mem_regions, input_mem_regions_cnt,
837 0 : acc_region_metas, stricter_abi_and_runtime_constraints,
838 0 : direct_mapping );
839 0 : } else {
840 0 : *out = fd_bpf_loader_input_serialize_aligned( instr_ctx, sz, pre_lens,
841 0 : input_mem_regions, input_mem_regions_cnt,
842 0 : acc_region_metas, stricter_abi_and_runtime_constraints,
843 0 : direct_mapping );
844 0 : }
845 :
846 0 : return FD_EXECUTOR_INSTR_SUCCESS;
847 0 : }
848 :
849 : /* https://github.com/anza-xyz/agave/blob/v3.0.4/program-runtime/src/serialization.rs#L284-L311 */
850 : int
851 : fd_bpf_loader_input_deserialize_parameters( fd_exec_instr_ctx_t * ctx,
852 : ulong const * pre_lens,
853 : uchar * input,
854 : ulong input_sz,
855 : int stricter_abi_and_runtime_constraints,
856 : int direct_mapping,
857 0 : uchar is_deprecated ) {
858 0 : if( FD_UNLIKELY( is_deprecated ) ) {
859 0 : return fd_bpf_loader_input_deserialize_unaligned(
860 0 : ctx, pre_lens, input, input_sz, stricter_abi_and_runtime_constraints, direct_mapping );
861 0 : } else {
862 0 : return fd_bpf_loader_input_deserialize_aligned(
863 0 : ctx, pre_lens, input, input_sz, stricter_abi_and_runtime_constraints, direct_mapping );
864 0 : }
865 0 : }
|
