Line data Source code
1 : #include "fd_ssparse.h"
2 :
3 : #include "../../../util/log/fd_log.h"
4 : #include "../../../util/archive/fd_tar.h"
5 : #include "../../../flamenco/runtime/fd_runtime_const.h"
6 : #include "../../../flamenco/runtime/fd_system_ids.h"
7 :
8 : #include <stdlib.h>
9 : #include <errno.h>
10 :
11 50562 : #define FD_SSPARSE_STATE_TAR_HEADER (0)
12 32112 : #define FD_SSPARSE_STATE_SCROLL_TAR_HEADER (1)
13 648 : #define FD_SSPARSE_STATE_VERSION (2)
14 531 : #define FD_SSPARSE_STATE_MANIFEST (3)
15 2634 : #define FD_SSPARSE_STATE_ACCOUNT_HEADER (4)
16 276 : #define FD_SSPARSE_STATE_ACCOUNT_DATA (5)
17 228 : #define FD_SSPARSE_STATE_ACCOUNT_PADDING (6)
18 423 : #define FD_SSPARSE_STATE_STATUS_CACHE (7)
19 618 : #define FD_SSPARSE_STATE_SCROLL_ACCOUNT_GARBAGE (8)
20 : #define FD_SSPARSE_STATE_ACCOUNT_BATCH (9)
21 :
22 : fd_ssparse_t *
23 1554 : fd_ssparse_init( fd_ssparse_t * ssparse ) {
24 1554 : memset( ssparse, 0, sizeof(fd_ssparse_t) );
25 1554 : ssparse->state = FD_SSPARSE_STATE_TAR_HEADER;
26 1554 : return ssparse;
27 1554 : }
28 :
29 : static int
30 : parse_tar_header_name( char const * name,
31 : ulong * id,
32 135 : ulong * slot ) {
33 135 : char name_buf[ FD_TAR_NAME_SZ ];
34 135 : fd_memcpy( name_buf, name, FD_TAR_NAME_SZ );
35 135 : name_buf[ FD_TAR_NAME_SZ-1 ] = '\0';
36 :
37 135 : char const * ptr = name_buf;
38 :
39 135 : if( FD_UNLIKELY( strncmp( ptr, "accounts/", 9UL ) ) ) {
40 0 : *id = ULONG_MAX;
41 0 : *slot = ULONG_MAX;
42 0 : return -1;
43 0 : }
44 :
45 135 : ptr += 9UL;
46 135 : char const * next = strchr( ptr, '.' );
47 135 : if( FD_UNLIKELY( !next ) ) {
48 0 : *id = ULONG_MAX;
49 0 : *slot = ULONG_MAX;
50 0 : return -1;
51 0 : }
52 135 : errno = 0;
53 135 : char * endptr;
54 135 : *slot = strtoul( ptr, &endptr, 10 );
55 135 : if( FD_UNLIKELY( errno==ERANGE || *endptr!='.' || endptr==ptr ) ) {
56 6 : *id = ULONG_MAX;
57 6 : *slot = ULONG_MAX;
58 6 : return -1;
59 6 : }
60 :
61 135 : errno = 0;
62 129 : ptr = next + 1;
63 129 : *id = strtoul( ptr, &endptr, 10 );
64 129 : if( FD_UNLIKELY( errno==ERANGE || *endptr!='\0' || endptr==ptr ) ) {
65 6 : *id = ULONG_MAX;
66 6 : *slot = ULONG_MAX;
67 6 : return -1;
68 6 : }
69 :
70 123 : return 0;
71 129 : }
72 :
73 : static int
74 : advance_tar( fd_ssparse_t * ssparse,
75 : uchar const * data,
76 : ulong data_sz,
77 48540 : fd_ssparse_advance_result_t * result ) {
78 48540 : ulong consume = fd_ulong_min( data_sz, 512UL - ssparse->tar.header_bytes_consumed );
79 48540 : if( FD_UNLIKELY( !consume ) ) {
80 0 : FD_LOG_WARNING(( "unexpected end of data in tar header, data_sz=%lu, header_bytes_consumed=%lu", data_sz, ssparse->tar.header_bytes_consumed ));
81 0 : return FD_SSPARSE_ADVANCE_ERROR;
82 0 : }
83 :
84 48540 : fd_memcpy( ssparse->tar.header+ssparse->tar.header_bytes_consumed, data, consume );
85 48540 : ssparse->bytes_consumed += consume;
86 48540 : result->bytes_consumed = consume;
87 48540 : ssparse->tar.header_bytes_consumed += consume;
88 :
89 48540 : if( FD_UNLIKELY( ssparse->tar.header_bytes_consumed<512UL ) ) return FD_SSPARSE_ADVANCE_AGAIN;
90 :
91 735 : fd_tar_meta_t const * hdr = (fd_tar_meta_t const *)ssparse->tar.header;
92 735 : ssparse->tar.header_bytes_consumed = 0UL;
93 :
94 : /* "ustar\x00" and "ustar \x00" (overlaps with version) are both
95 : valid values for magic. These are POSIX ustar and OLDGNU versions
96 : respectively. */
97 735 : if( FD_UNLIKELY( memcmp( hdr->magic, FD_TAR_MAGIC, FD_TAR_MAGIC_SZ ) ) ) {
98 192 : int not_zero = 0;
99 98496 : for( ulong i=0UL; i<512UL; i++ ) not_zero |= ssparse->tar.header[ i ];
100 192 : if( FD_UNLIKELY( not_zero ) ) {
101 6 : FD_LOG_WARNING(( "invalid tar header magic `%." FD_EXPAND_THEN_STRINGIFY(FD_TAR_MAGIC_SZ) "s`", hdr->magic ));
102 6 : return FD_SSPARSE_ADVANCE_ERROR;
103 6 : }
104 :
105 186 : if( FD_LIKELY( ssparse->flags.seen_zero_tar_frame ) ) {
106 90 : if( FD_UNLIKELY( !ssparse->flags.seen_version || !ssparse->flags.seen_manifest || !ssparse->flags.seen_status_cache ) ) {
107 9 : FD_LOG_WARNING(( "unexpected end of file before version or manifest or status cache" ));
108 9 : return FD_SSPARSE_ADVANCE_ERROR;
109 9 : }
110 :
111 81 : return FD_SSPARSE_ADVANCE_DONE;
112 90 : }
113 :
114 96 : ssparse->flags.seen_zero_tar_frame = 1;
115 96 : return FD_SSPARSE_ADVANCE_AGAIN;
116 186 : }
117 :
118 543 : if( FD_UNLIKELY( ssparse->flags.seen_zero_tar_frame ) ) {
119 6 : FD_LOG_WARNING(( "unexpected valid tar header after zero frame" ));
120 6 : return FD_SSPARSE_ADVANCE_ERROR;
121 6 : }
122 :
123 537 : ssparse->tar.file_bytes = fd_tar_meta_get_size( hdr );
124 537 : if( FD_UNLIKELY( ssparse->tar.file_bytes==ULONG_MAX ) ) {
125 0 : FD_LOG_WARNING(( "invalid tar header size %." FD_EXPAND_THEN_STRINGIFY(FD_TAR_SIZE_SZ) "s "
126 0 : "for tar header name %." FD_EXPAND_THEN_STRINGIFY(FD_TAR_NAME_SZ) "s", hdr->size, hdr->name ));
127 0 : return FD_SSPARSE_ADVANCE_ERROR;
128 0 : }
129 :
130 537 : if( FD_UNLIKELY( hdr->typeflag==FD_TAR_TYPE_DIR ) ) {
131 6 : if( FD_UNLIKELY( ssparse->tar.file_bytes ) ) {
132 3 : FD_LOG_WARNING(( "invalid tar directory entry with non-zero size %lu", ssparse->tar.file_bytes ));
133 3 : return FD_SSPARSE_ADVANCE_ERROR;
134 3 : }
135 3 : return FD_SSPARSE_ADVANCE_AGAIN;
136 6 : }
137 :
138 531 : if( FD_UNLIKELY( !fd_tar_meta_is_reg( hdr ) ) ) {
139 3 : FD_LOG_WARNING(( "invalid tar header type %d", hdr->typeflag ));
140 3 : return FD_SSPARSE_ADVANCE_ERROR;
141 3 : }
142 528 : if( FD_UNLIKELY( !ssparse->tar.file_bytes ) ) {
143 3 : FD_LOG_WARNING(( "invalid tar header size %lu", ssparse->tar.file_bytes ));
144 3 : return FD_SSPARSE_ADVANCE_ERROR;
145 3 : }
146 :
147 : /* TODO: Check every header field here for validity? */
148 :
149 525 : int desired_state;
150 525 : if( FD_LIKELY( !strncmp( hdr->name, "version", 7UL ) ) ) {
151 150 : desired_state = FD_SSPARSE_STATE_VERSION;
152 150 : if( FD_UNLIKELY( ssparse->tar.file_bytes!=5UL ) ) {
153 6 : FD_LOG_WARNING(( "invalid version file size %lu", ssparse->tar.file_bytes ));
154 6 : return FD_SSPARSE_ADVANCE_ERROR;
155 6 : }
156 375 : } else if( FD_LIKELY( !strncmp( hdr->name, "accounts/", 9UL ) ) ) {
157 135 : ssparse->account.header_bytes_consumed = 0UL;
158 135 : desired_state = FD_SSPARSE_STATE_ACCOUNT_HEADER;
159 135 : ulong id, slot;
160 135 : if( FD_UNLIKELY( -1==parse_tar_header_name( hdr->name, &id, &slot ) ) ) {
161 12 : FD_LOG_WARNING(( "invalid account append vec name %." FD_EXPAND_THEN_STRINGIFY(FD_TAR_NAME_SZ) "s", hdr->name ));
162 12 : return FD_SSPARSE_ADVANCE_ERROR;
163 12 : }
164 123 : ssparse->slot = slot;
165 123 : ssparse->acc_vec_bytes = ssparse->tar.file_bytes;
166 240 : } else if( FD_LIKELY( !strncmp( hdr->name, "snapshots/status_cache", 22UL ) ) ) desired_state = FD_SSPARSE_STATE_STATUS_CACHE;
167 135 : else if( FD_LIKELY( !strncmp( hdr->name, "snapshots/", 10UL ) ) ) {
168 132 : desired_state = FD_SSPARSE_STATE_MANIFEST;
169 132 : } else {
170 3 : FD_LOG_WARNING(( "unexpected tar header name `%." FD_EXPAND_THEN_STRINGIFY(FD_TAR_NAME_SZ) "s`", hdr->name ));
171 3 : return FD_SSPARSE_ADVANCE_ERROR;
172 3 : }
173 :
174 504 : ssparse->tar.file_bytes_consumed = 0UL;
175 :
176 504 : switch( desired_state ) {
177 144 : case FD_SSPARSE_STATE_VERSION:
178 144 : if( FD_UNLIKELY( ssparse->flags.seen_version ) ) {
179 3 : FD_LOG_WARNING(( "unexpected duplicate version file" ));
180 3 : return FD_SSPARSE_ADVANCE_ERROR;
181 3 : }
182 :
183 141 : ssparse->flags.seen_version = 1;
184 141 : ssparse->state = FD_SSPARSE_STATE_VERSION;
185 141 : break;
186 132 : case FD_SSPARSE_STATE_MANIFEST:
187 132 : if( FD_UNLIKELY( ssparse->flags.seen_manifest ) ) {
188 3 : FD_LOG_WARNING(( "unexpected duplicate manifest file" ));
189 3 : return FD_SSPARSE_ADVANCE_ERROR;
190 3 : }
191 :
192 129 : ssparse->flags.seen_manifest = 1;
193 129 : ssparse->state = FD_SSPARSE_STATE_MANIFEST;
194 129 : break;
195 123 : case FD_SSPARSE_STATE_ACCOUNT_HEADER:
196 123 : if( FD_UNLIKELY( !ssparse->flags.seen_manifest ) ) {
197 3 : FD_LOG_WARNING(( "unexpected account append vec file before manifest" ));
198 3 : return FD_SSPARSE_ADVANCE_ERROR;
199 3 : }
200 :
201 120 : ssparse->account.header_bytes_consumed = 0UL;
202 120 : ssparse->state = FD_SSPARSE_STATE_SCROLL_ACCOUNT_GARBAGE;
203 120 : result->appendvec.slot = ssparse->slot;
204 120 : result->appendvec.data_sz = ssparse->tar.file_bytes;
205 120 : return FD_SSPARSE_ADVANCE_APPENDVEC;
206 105 : case FD_SSPARSE_STATE_STATUS_CACHE:
207 105 : if( FD_UNLIKELY( ssparse->flags.seen_status_cache ) ) {
208 3 : FD_LOG_WARNING(( "unexpected status cache file" ));
209 3 : return FD_SSPARSE_ADVANCE_ERROR;
210 3 : }
211 :
212 102 : ssparse->flags.seen_status_cache = 1;
213 102 : ssparse->state = FD_SSPARSE_STATE_STATUS_CACHE;
214 102 : break;
215 0 : default:
216 0 : FD_LOG_ERR(( "unexpected tar header desired state %d", desired_state ));
217 0 : break;
218 504 : }
219 :
220 372 : return FD_SSPARSE_ADVANCE_AGAIN;
221 504 : }
222 :
223 : static int
224 : advance_version( fd_ssparse_t * ssparse,
225 : uchar const * data,
226 : ulong data_sz,
227 213 : fd_ssparse_advance_result_t * result ) {
228 213 : ulong consume = fd_ulong_min( data_sz, ssparse->tar.file_bytes-ssparse->tar.file_bytes_consumed );
229 213 : if( FD_UNLIKELY( !consume ) ) {
230 0 : FD_LOG_WARNING(( "unexpected end of data while parsing version file, data_sz=%lu, file_bytes_consumed=%lu, file_bytes=%lu", data_sz, ssparse->tar.file_bytes_consumed, ssparse->tar.file_bytes ));
231 0 : return FD_SSPARSE_ADVANCE_ERROR;
232 0 : }
233 :
234 213 : fd_memcpy( ssparse->version+ssparse->tar.file_bytes_consumed, data, consume );
235 :
236 213 : ssparse->tar.file_bytes_consumed += consume;
237 213 : ssparse->bytes_consumed += consume;
238 213 : result->bytes_consumed = consume;
239 :
240 213 : if( FD_LIKELY( ssparse->tar.file_bytes_consumed<ssparse->tar.file_bytes ) ) return FD_SSPARSE_ADVANCE_AGAIN;
241 :
242 138 : FD_TEST( ssparse->tar.file_bytes_consumed==ssparse->tar.file_bytes );
243 138 : FD_TEST( ssparse->tar.file_bytes_consumed==5UL );
244 :
245 138 : if( FD_UNLIKELY( memcmp( ssparse->version, "1.2.0", 5UL ) ) ) {
246 3 : FD_LOG_WARNING(( "invalid version file %.*s", 5, ssparse->version ));
247 3 : return FD_SSPARSE_ADVANCE_ERROR;
248 3 : }
249 :
250 135 : ssparse->state = FD_SSPARSE_STATE_SCROLL_TAR_HEADER;
251 135 : return FD_SSPARSE_ADVANCE_AGAIN;
252 138 : }
253 :
254 : static int
255 : advance_status_cache( fd_ssparse_t * ssparse,
256 : uchar const * data,
257 : ulong data_sz,
258 111 : fd_ssparse_advance_result_t * result ) {
259 111 : ulong consume = fd_ulong_min( data_sz, ssparse->tar.file_bytes-ssparse->tar.file_bytes_consumed );
260 111 : if( FD_UNLIKELY( !consume ) ) {
261 0 : FD_LOG_WARNING(( "unexpected end of data while parsing status cache, data_sz=%lu, file_bytes_consumed=%lu, file_bytes=%lu", data_sz, ssparse->tar.file_bytes_consumed, ssparse->tar.file_bytes ));
262 0 : return FD_SSPARSE_ADVANCE_ERROR;
263 0 : }
264 :
265 111 : ssparse->tar.file_bytes_consumed += consume;
266 111 : ssparse->bytes_consumed += consume;
267 :
268 111 : result->bytes_consumed = consume;
269 111 : result->status_cache.data = data;
270 111 : result->status_cache.data_sz = consume;
271 111 : result->status_cache.done = ssparse->tar.file_bytes_consumed==ssparse->tar.file_bytes;
272 :
273 111 : if( FD_LIKELY( ssparse->tar.file_bytes_consumed<ssparse->tar.file_bytes ) ) {
274 12 : return FD_SSPARSE_ADVANCE_STATUS_CACHE;
275 12 : }
276 99 : else { /* ssparse->tar.file_bytes_consumed==ssparse->tar.file_bytes */
277 : /* finished parsing status cache */
278 99 : ssparse->state = FD_SSPARSE_STATE_SCROLL_TAR_HEADER;
279 99 : return FD_SSPARSE_ADVANCE_STATUS_CACHE;
280 99 : }
281 111 : }
282 :
283 : static int
284 : advance_manifest( fd_ssparse_t * ssparse,
285 : uchar const * data,
286 : ulong data_sz,
287 138 : fd_ssparse_advance_result_t * result ) {
288 138 : ulong consume = fd_ulong_min( data_sz, ssparse->tar.file_bytes-ssparse->tar.file_bytes_consumed );
289 138 : if( FD_UNLIKELY( !consume ) ) {
290 0 : FD_LOG_WARNING(( "unexpected end of data while parsing manifest, data_sz=%lu, file_bytes_consumed=%lu, file_bytes=%lu", data_sz, ssparse->tar.file_bytes_consumed, ssparse->tar.file_bytes ));
291 0 : return FD_SSPARSE_ADVANCE_ERROR;
292 0 : }
293 :
294 138 : ssparse->tar.file_bytes_consumed += consume;
295 138 : ssparse->bytes_consumed += consume;
296 :
297 138 : result->bytes_consumed = consume;
298 138 : result->manifest.data = data;
299 138 : result->manifest.data_sz = consume;
300 :
301 138 : if( FD_LIKELY( ssparse->tar.file_bytes_consumed<ssparse->tar.file_bytes ) ) return FD_SSPARSE_ADVANCE_MANIFEST;
302 :
303 126 : ssparse->state = FD_SSPARSE_STATE_SCROLL_TAR_HEADER;
304 126 : return FD_SSPARSE_ADVANCE_MANIFEST_DONE;
305 138 : }
306 :
307 : static int
308 : advance_next_tar( fd_ssparse_t * ssparse,
309 : uchar const * data,
310 : ulong data_sz,
311 31644 : fd_ssparse_advance_result_t * result ) {
312 31644 : (void)data;
313 : /* skip padding */
314 31644 : ulong bytes_remaining = fd_ulong_align_up( ssparse->bytes_consumed, 512UL ) - ssparse->bytes_consumed;
315 31644 : ulong pad_sz = bytes_remaining;
316 31644 : pad_sz = fd_ulong_min( pad_sz, data_sz );
317 31644 : if( FD_UNLIKELY( !pad_sz && bytes_remaining ) ) {
318 0 : FD_LOG_WARNING(( "unexpected end of data while parsing tar header padding, data_sz=%lu, bytes_consumed=%lu, bytes_remaining=%lu", data_sz, ssparse->bytes_consumed, bytes_remaining ));
319 0 : return FD_SSPARSE_ADVANCE_ERROR;
320 0 : }
321 :
322 31644 : ssparse->bytes_consumed += pad_sz;
323 31644 : result->bytes_consumed = pad_sz;
324 31644 : bytes_remaining -= pad_sz;
325 :
326 31644 : if( FD_LIKELY( !bytes_remaining ) ) ssparse->state = FD_SSPARSE_STATE_TAR_HEADER;
327 31644 : return FD_SSPARSE_ADVANCE_AGAIN;
328 31644 : }
329 :
330 : static int
331 : advance_account_batch( fd_ssparse_t * ssparse,
332 : uchar const * data,
333 : ulong data_sz,
334 60 : fd_ssparse_advance_result_t * result ) {
335 : /* Cannot create a batch unless the parser is aligned to an account. */
336 60 : if( FD_UNLIKELY( ssparse->account.header_bytes_consumed ) ) return FD_SSPARSE_ADVANCE_AGAIN;
337 :
338 : /* Each account is at least 136 bytes large. Don't attempt to create
339 : a batch unless at least 4 accounts fit. */
340 60 : ulong avail = fd_ulong_min( data_sz, ssparse->acc_vec_bytes - ssparse->tar.file_bytes_consumed );
341 60 : if( FD_UNLIKELY( avail<(4*136UL) ) ) return FD_SSPARSE_ADVANCE_AGAIN;
342 :
343 : /* Prefetch data to reduce cache misses in hot path. */
344 321 : for( ulong i=0UL; i<fd_ulong_min( avail, 4096UL ); i+=64UL ) {
345 303 : __builtin_prefetch( (uchar *)data+i, 0, 0 );
346 303 : }
347 :
348 : /* Skip over accounts until we reached EOF or batch is full */
349 18 : result->account_batch.batch_cnt = 0;
350 18 : ulong off = 0UL;
351 135 : for( ulong idx=0UL; idx<FD_SSPARSE_ACC_BATCH_MAX && off+136UL<=avail; idx++ ) {
352 120 : uchar const * acc_hdr = (uchar *)data+off;
353 :
354 : /* We want ConfigProgram accounts to go through the slow path,
355 : since they are published from there to consumers for monitoring. */
356 120 : if( FD_UNLIKELY( !memcmp( acc_hdr+64UL, fd_solana_config_program_id.key, sizeof(fd_hash_t) ) ) ) {
357 3 : if( FD_UNLIKELY( idx==0UL ) ) return FD_SSPARSE_ADVANCE_AGAIN; /* At the front of the batch, abort */
358 0 : else break; /* otherwise, break early. */
359 3 : }
360 :
361 117 : ulong acc_data_sz = fd_ulong_load_8_fast( acc_hdr+8 );
362 117 : if( FD_UNLIKELY( acc_data_sz>FD_RUNTIME_ACC_SZ_MAX ) ) {
363 0 : FD_LOG_WARNING(( "invalid account data size %lu", acc_data_sz ));
364 0 : return FD_SSPARSE_ADVANCE_ERROR;
365 0 : }
366 : /* acc_hdr[96] is the executable flag (uchar), must be 0 or 1 */
367 117 : if( FD_UNLIKELY( acc_hdr[ 96UL ]>1 ) ) {
368 0 : FD_LOG_WARNING(( "invalid account header executable %u", acc_hdr[ 96UL ] ));
369 0 : return FD_SSPARSE_ADVANCE_ERROR;
370 0 : }
371 117 : ulong next_off = off+136UL+acc_data_sz;
372 117 : ulong pad_sz = fd_ulong_align_up( ssparse->tar.file_bytes_consumed+next_off, 8UL ) -
373 117 : ( ssparse->tar.file_bytes_consumed+next_off );
374 117 : next_off += pad_sz;
375 117 : if( FD_UNLIKELY( next_off>avail ) ) break; /* account is fragmented */
376 117 : result->account_batch.batch_cnt = idx+1UL;
377 117 : result->account_batch.batch[ idx ] = acc_hdr;
378 117 : ssparse->account.header_bytes_consumed = 136UL;
379 117 : ssparse->account.data_bytes_consumed = acc_data_sz;
380 117 : ssparse->account.data_len = acc_data_sz;
381 117 : off = next_off;
382 117 : }
383 :
384 : /* Skip if no batches to process. */
385 15 : if( FD_UNLIKELY( !result->account_batch.batch_cnt ) ) {
386 0 : return FD_SSPARSE_ADVANCE_AGAIN;
387 0 : }
388 :
389 15 : ssparse->tar.file_bytes_consumed += off;
390 15 : ssparse->bytes_consumed += off;
391 15 : result->bytes_consumed = off;
392 :
393 : /* reset state */
394 :
395 15 : ssparse->state = FD_SSPARSE_STATE_ACCOUNT_PADDING;
396 :
397 15 : result->account_batch.slot = ssparse->slot;
398 :
399 15 : return FD_SSPARSE_ADVANCE_ACCOUNT_BATCH;
400 15 : }
401 :
402 : static int
403 : advance_account_header( fd_ssparse_t * ssparse,
404 : uchar const * data,
405 : ulong data_sz,
406 2241 : fd_ssparse_advance_result_t * result ) {
407 2241 : ulong consume = fd_ulong_min( 136UL-ssparse->account.header_bytes_consumed, fd_ulong_min( data_sz, ssparse->acc_vec_bytes-ssparse->tar.file_bytes_consumed ) );
408 :
409 2241 : if( FD_UNLIKELY( !consume ) ) {
410 15 : if( FD_LIKELY( ssparse->tar.file_bytes_consumed==ssparse->acc_vec_bytes ) ) {
411 15 : ssparse->state = FD_SSPARSE_STATE_SCROLL_ACCOUNT_GARBAGE;
412 15 : return FD_SSPARSE_ADVANCE_AGAIN;
413 15 : } else {
414 0 : FD_LOG_WARNING(( "unexpected end of data while advancing account header, data_sz=%lu, file_bytes_consumed=%lu, acc_vec_bytes=%lu", data_sz, ssparse->tar.file_bytes_consumed, ssparse->acc_vec_bytes ));
415 0 : return FD_SSPARSE_ADVANCE_ERROR;
416 0 : }
417 15 : }
418 :
419 2226 : if( FD_UNLIKELY( consume<136UL ) ) {
420 2127 : fd_memcpy( ssparse->account.header+ssparse->account.header_bytes_consumed, data, consume );
421 2127 : } else if( ssparse->batch_enabled ) {
422 : /* fast path */
423 60 : int res = advance_account_batch( ssparse, data, data_sz, result );
424 60 : if( res==FD_SSPARSE_ADVANCE_ACCOUNT_BATCH ) return res;
425 : /* fall through and continue processing account header */
426 60 : }
427 :
428 2211 : ssparse->account.header_bytes_consumed += consume;
429 2211 : ssparse->tar.file_bytes_consumed += consume;
430 2211 : ssparse->bytes_consumed += consume;
431 2211 : result->bytes_consumed = consume;
432 :
433 2211 : if( FD_UNLIKELY( ssparse->account.header_bytes_consumed<136UL ) ) return FD_SSPARSE_ADVANCE_AGAIN;
434 :
435 105 : uchar const * hdr = ssparse->account.header;
436 105 : if( FD_LIKELY( consume==136UL ) ) hdr = data;
437 :
438 105 : result->account_header.data_len = fd_ulong_load_8_fast( hdr+8UL );
439 105 : if( FD_UNLIKELY( result->account_header.data_len>FD_RUNTIME_ACC_SZ_MAX ) ) {
440 3 : FD_LOG_WARNING(( "invalid account header data length %lu", result->account_header.data_len ));
441 3 : return FD_SSPARSE_ADVANCE_ERROR;
442 3 : }
443 :
444 102 : result->account_header.pubkey = hdr+16UL;
445 102 : result->account_header.lamports = fd_ulong_load_8_fast( hdr+48UL );
446 102 : result->account_header.rent_epoch = fd_ulong_load_8_fast( hdr+56UL );
447 102 : result->account_header.owner = hdr+64UL;
448 102 : result->account_header.executable = hdr[ 96UL ];
449 102 : if( FD_UNLIKELY( result->account_header.executable>1 ) ) {
450 3 : char pubkey_str[ FD_BASE58_ENCODED_32_SZ ];
451 3 : fd_base58_encode_32( result->account_header.pubkey, NULL, pubkey_str );
452 3 : FD_LOG_WARNING(( "invalid account header executable %d for account %s", result->account_header.executable, pubkey_str ));
453 3 : return FD_SSPARSE_ADVANCE_ERROR;
454 3 : }
455 99 : result->account_header.hash = hdr+104UL;
456 99 : result->account_header.slot = ssparse->slot;
457 :
458 99 : ssparse->account.data_len = result->account_header.data_len;
459 99 : ssparse->account.data_bytes_consumed = 0UL;
460 99 : ssparse->state = FD_SSPARSE_STATE_ACCOUNT_DATA;
461 :
462 99 : return FD_SSPARSE_ADVANCE_ACCOUNT_HEADER;
463 102 : }
464 :
465 : static int
466 : advance_account_data( fd_ssparse_t * ssparse,
467 : uchar const * data,
468 : ulong data_sz,
469 177 : fd_ssparse_advance_result_t * result ) {
470 177 : if( FD_UNLIKELY( ssparse->account.data_bytes_consumed==ssparse->account.data_len ) ) {
471 48 : ssparse->state = FD_SSPARSE_STATE_ACCOUNT_PADDING;
472 48 : return FD_SSPARSE_ADVANCE_AGAIN;
473 48 : }
474 :
475 129 : ulong consume = fd_ulong_min( data_sz, ssparse->acc_vec_bytes-ssparse->tar.file_bytes_consumed );
476 129 : if( FD_UNLIKELY( !consume ) ) {
477 3 : FD_LOG_WARNING(( "account data extends beyond append vec size" ));
478 3 : return FD_SSPARSE_ADVANCE_ERROR;
479 3 : }
480 :
481 126 : consume = fd_ulong_min( consume, ssparse->account.data_len-ssparse->account.data_bytes_consumed );
482 126 : if( FD_UNLIKELY( !consume ) ) {
483 0 : FD_LOG_WARNING(( "unexpected end of data while parsing account data, data_sz=%lu, data_bytes_consumed=%lu, data_len=%lu", data_sz, ssparse->account.data_bytes_consumed, ssparse->account.data_len ));
484 0 : return FD_SSPARSE_ADVANCE_ERROR;
485 0 : }
486 :
487 126 : ssparse->tar.file_bytes_consumed += consume;
488 126 : ssparse->bytes_consumed += consume;
489 126 : ssparse->account.data_bytes_consumed += consume;
490 126 : result->bytes_consumed = consume;
491 :
492 126 : result->account_data.data_sz = consume;
493 126 : result->account_data.data = data;
494 :
495 126 : FD_TEST( ssparse->account.data_bytes_consumed<=ssparse->account.data_len );
496 126 : if( FD_LIKELY( ssparse->account.data_bytes_consumed==ssparse->account.data_len ) ) {
497 45 : ssparse->state = FD_SSPARSE_STATE_ACCOUNT_PADDING;
498 45 : }
499 :
500 126 : return FD_SSPARSE_ADVANCE_ACCOUNT_DATA;
501 126 : }
502 :
503 : static int
504 : advance_account_padding( fd_ssparse_t * ssparse,
505 : uchar const * data,
506 : ulong data_sz,
507 120 : fd_ssparse_advance_result_t * result ) {
508 120 : (void)data;
509 :
510 120 : ulong pad_sz = fd_ulong_align_up( ssparse->tar.file_bytes_consumed, 8UL ) - ssparse->tar.file_bytes_consumed;
511 120 : pad_sz = fd_ulong_min( pad_sz, ssparse->acc_vec_bytes - ssparse->tar.file_bytes_consumed );
512 120 : if( FD_UNLIKELY( !pad_sz ) ) {
513 90 : if( FD_LIKELY( ssparse->tar.file_bytes_consumed==ssparse->acc_vec_bytes ) ) ssparse->state = FD_SSPARSE_STATE_SCROLL_TAR_HEADER;
514 12 : else ssparse->state = FD_SSPARSE_STATE_ACCOUNT_HEADER;
515 :
516 90 : ssparse->account.header_bytes_consumed = 0UL;
517 90 : return FD_SSPARSE_ADVANCE_AGAIN;
518 90 : }
519 :
520 30 : ulong consume = fd_ulong_min( data_sz, pad_sz );
521 30 : if( FD_UNLIKELY( !consume ) ) {
522 0 : FD_LOG_WARNING(( "unexpected end of data while parsing account padding, data_sz=%lu, file_bytes_consumed=%lu, acc_vec_bytes=%lu", data_sz, ssparse->tar.file_bytes_consumed, ssparse->acc_vec_bytes ));
523 0 : return FD_SSPARSE_ADVANCE_ERROR;
524 0 : }
525 :
526 30 : ssparse->tar.file_bytes_consumed += consume;
527 30 : ssparse->bytes_consumed += consume;
528 30 : result->bytes_consumed = consume;
529 :
530 30 : ulong remaining = fd_ulong_align_up( ssparse->tar.file_bytes_consumed, 8UL ) - ssparse->tar.file_bytes_consumed;
531 30 : if( FD_LIKELY( !remaining ) ) {
532 18 : ssparse->account.header_bytes_consumed = 0UL;
533 18 : ssparse->state = FD_SSPARSE_STATE_ACCOUNT_HEADER;
534 18 : }
535 30 : return FD_SSPARSE_ADVANCE_AGAIN;
536 30 : }
537 :
538 : static int
539 : advance_account_garbage( fd_ssparse_t * ssparse,
540 : uchar const * data,
541 : ulong data_sz,
542 483 : fd_ssparse_advance_result_t * result ) {
543 483 : (void)data;
544 483 : ulong rem = ssparse->tar.file_bytes-ssparse->tar.file_bytes_consumed;
545 483 : if( FD_UNLIKELY( !rem ) ) {
546 15 : ssparse->state = FD_SSPARSE_STATE_SCROLL_TAR_HEADER;
547 15 : return FD_SSPARSE_ADVANCE_AGAIN;
548 15 : }
549 :
550 468 : if( FD_UNLIKELY( !data_sz ) ) {
551 0 : FD_LOG_WARNING(( "unexpected end of data while parsing append vec garbage, data_sz=%lu, remaining_bytes=%lu", data_sz, rem ));
552 0 : return FD_SSPARSE_ADVANCE_ERROR;
553 0 : }
554 :
555 468 : ulong consume = fd_ulong_min( data_sz, rem );
556 468 : ssparse->tar.file_bytes_consumed += consume;
557 468 : ssparse->bytes_consumed += consume;
558 468 : result->bytes_consumed = consume;
559 :
560 468 : if( FD_LIKELY( ssparse->tar.file_bytes_consumed<ssparse->tar.file_bytes ) ) return FD_SSPARSE_ADVANCE_AGAIN;
561 :
562 15 : ssparse->state = FD_SSPARSE_STATE_SCROLL_TAR_HEADER;
563 15 : return FD_SSPARSE_ADVANCE_AGAIN;
564 468 : }
565 :
566 : int
567 : fd_ssparse_advance( fd_ssparse_t * ssparse,
568 : uchar const * data,
569 : ulong data_sz,
570 83667 : fd_ssparse_advance_result_t * result ) {
571 83667 : result->bytes_consumed = 0UL;
572 :
573 83667 : switch( ssparse->state ) {
574 48540 : case FD_SSPARSE_STATE_TAR_HEADER: return advance_tar( ssparse, data, data_sz, result );
575 31644 : case FD_SSPARSE_STATE_SCROLL_TAR_HEADER: return advance_next_tar( ssparse, data, data_sz, result );
576 213 : case FD_SSPARSE_STATE_VERSION: return advance_version( ssparse, data, data_sz, result );
577 138 : case FD_SSPARSE_STATE_MANIFEST: return advance_manifest( ssparse, data, data_sz, result );
578 2241 : case FD_SSPARSE_STATE_ACCOUNT_HEADER: return advance_account_header( ssparse, data, data_sz, result );
579 177 : case FD_SSPARSE_STATE_ACCOUNT_DATA: return advance_account_data( ssparse, data, data_sz, result );
580 120 : case FD_SSPARSE_STATE_ACCOUNT_PADDING: return advance_account_padding( ssparse, data, data_sz, result );
581 111 : case FD_SSPARSE_STATE_STATUS_CACHE: return advance_status_cache( ssparse, data, data_sz, result );
582 483 : case FD_SSPARSE_STATE_SCROLL_ACCOUNT_GARBAGE: return advance_account_garbage( ssparse, data, data_sz, result );
583 0 : default: FD_LOG_ERR(( "invalid state %d", ssparse->state ));
584 83667 : }
585 83667 : }
586 :
587 : void
588 : fd_ssparse_batch_enable( fd_ssparse_t * ssparse,
589 1368 : int enabled ) {
590 1368 : ssparse->batch_enabled = !!enabled;
591 1368 : }
592 :
593 : void
594 105 : fd_ssparse_appendvec_parse( fd_ssparse_t * ssparse ) {
595 105 : FD_TEST( ssparse->state==FD_SSPARSE_STATE_SCROLL_ACCOUNT_GARBAGE );
596 105 : FD_TEST( !ssparse->tar.file_bytes_consumed );
597 105 : ssparse->state = FD_SSPARSE_STATE_ACCOUNT_HEADER;
598 105 : }
|
