Line data Source code
1 : #include "fd_snapshot_loader.h"
2 : #include "fd_snapshot_base.h"
3 : #include "fd_snapshot_http.h"
4 :
5 : #include <errno.h>
6 : #include <fcntl.h>
7 : #include <netdb.h>
8 : #include <regex.h>
9 : #include <stdlib.h> /* strtoul */
10 : #include <unistd.h>
11 : #include <sys/types.h>
12 : #include <sys/socket.h>
13 : #include <netinet/in.h>
14 :
15 0 : #define FD_SNAPSHOT_LOADER_MAGIC (0xa78a73a69d33e6b1UL)
16 :
17 : struct fd_snapshot_loader {
18 : ulong magic;
19 :
20 : /* Source: HTTP */
21 :
22 : void * http_mem;
23 : fd_snapshot_http_t * http;
24 :
25 : /* Source: File I/O */
26 :
27 : int snapshot_fd;
28 : fd_io_istream_file_t vfile[1];
29 :
30 : /* Source I/O abstraction */
31 :
32 : fd_io_istream_obj_t vsrc;
33 :
34 : /* Zstandard decompressor */
35 :
36 : fd_zstd_dstream_t * zstd;
37 : fd_io_istream_zstd_t vzstd[1];
38 :
39 : /* Tar reader */
40 :
41 : fd_tar_reader_t tar[1];
42 : fd_tar_io_reader_t vtar[1];
43 :
44 : /* Downstream restore */
45 :
46 : fd_snapshot_restore_t * restore;
47 :
48 : /* Hash and slot numbers from filename */
49 :
50 : fd_snapshot_name_t name;
51 : };
52 :
53 : typedef struct fd_snapshot_loader fd_snapshot_loader_t;
54 :
55 : ulong
56 0 : fd_snapshot_loader_align( void ) {
57 0 : return fd_ulong_max( alignof(fd_snapshot_loader_t), fd_zstd_dstream_align() );
58 0 : }
59 :
60 : ulong
61 0 : fd_snapshot_loader_footprint( ulong zstd_window_sz ) {
62 0 : ulong l = FD_LAYOUT_INIT;
63 0 : l = FD_LAYOUT_APPEND( l, alignof(fd_snapshot_loader_t), sizeof(fd_snapshot_loader_t) );
64 0 : l = FD_LAYOUT_APPEND( l, fd_zstd_dstream_align(), fd_zstd_dstream_footprint( zstd_window_sz ) );
65 0 : l = FD_LAYOUT_APPEND( l, alignof(fd_snapshot_http_t), sizeof(fd_snapshot_http_t) );
66 : /* FIXME add test ensuring zstd dstream align > alignof loader */
67 0 : return FD_LAYOUT_FINI( l, fd_snapshot_loader_align() );
68 0 : }
69 :
70 : fd_snapshot_loader_t *
71 : fd_snapshot_loader_new( void * mem,
72 0 : ulong zstd_window_sz ) {
73 :
74 0 : if( FD_UNLIKELY( !mem ) ) {
75 0 : FD_LOG_WARNING(( "NULL mem" ));
76 0 : return NULL;
77 0 : }
78 :
79 0 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)mem, fd_snapshot_loader_align() ) ) ) {
80 0 : FD_LOG_WARNING(( "unaligned mem" ));
81 0 : return NULL;
82 0 : }
83 :
84 0 : FD_SCRATCH_ALLOC_INIT( l, mem );
85 0 : fd_snapshot_loader_t * loader = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_snapshot_loader_t), sizeof(fd_snapshot_loader_t) );
86 0 : void * zstd_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_zstd_dstream_align(), fd_zstd_dstream_footprint( zstd_window_sz ) );
87 0 : void * http_mem = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_snapshot_http_t), sizeof(fd_snapshot_http_t) );
88 0 : FD_SCRATCH_ALLOC_FINI( l, fd_snapshot_loader_align() );
89 :
90 0 : fd_memset( loader, 0, sizeof(fd_snapshot_loader_t) );
91 0 : loader->http_mem = http_mem;
92 0 : loader->zstd = fd_zstd_dstream_new( zstd_mem, zstd_window_sz );
93 :
94 0 : FD_COMPILER_MFENCE();
95 0 : loader->magic = FD_SNAPSHOT_LOADER_MAGIC;
96 0 : FD_COMPILER_MFENCE();
97 :
98 0 : return loader;
99 0 : }
100 :
101 : void *
102 0 : fd_snapshot_loader_delete( fd_snapshot_loader_t * loader ) {
103 :
104 0 : if( FD_UNLIKELY( !loader ) ) return NULL;
105 :
106 0 : if( FD_UNLIKELY( loader->magic != FD_SNAPSHOT_LOADER_MAGIC ) ) {
107 0 : FD_LOG_WARNING(( "invalid magic" ));
108 0 : return NULL;
109 0 : }
110 :
111 0 : fd_zstd_dstream_delete ( loader->zstd );
112 0 : fd_tar_io_reader_delete ( loader->vtar );
113 0 : fd_io_istream_zstd_delete( loader->vzstd );
114 0 : fd_io_istream_file_delete( loader->vfile );
115 0 : fd_snapshot_http_delete ( loader->http );
116 0 : fd_tar_reader_delete ( loader->tar );
117 :
118 0 : if( loader->snapshot_fd>=0 ) {
119 0 : if( FD_UNLIKELY( 0!=close( loader->snapshot_fd ) ) )
120 0 : FD_LOG_WARNING(( "close(%d) failed (%d-%s)", loader->snapshot_fd, errno, fd_io_strerror( errno ) ));
121 0 : loader->snapshot_fd = -1;
122 0 : }
123 :
124 0 : FD_COMPILER_MFENCE();
125 0 : loader->magic = 0UL;
126 0 : FD_COMPILER_MFENCE();
127 :
128 0 : return loader;
129 0 : }
130 :
131 : fd_snapshot_loader_t *
132 : fd_snapshot_loader_init( fd_snapshot_loader_t * d,
133 : fd_snapshot_restore_t * restore,
134 : fd_snapshot_src_t const * src,
135 : ulong base_slot,
136 0 : int validate_slot ) {
137 :
138 0 : d->restore = restore;
139 :
140 0 : switch( src->type ) {
141 0 : case FD_SNAPSHOT_SRC_FILE:
142 0 : d->snapshot_fd = open( src->file.path, O_RDONLY );
143 0 : if( FD_UNLIKELY( d->snapshot_fd<0 ) ) {
144 0 : FD_LOG_WARNING(( "open(%s) failed (%d-%s)", src->file.path, errno, fd_io_strerror( errno ) ));
145 0 : return NULL;
146 0 : }
147 :
148 0 : fd_snapshot_name_t * name = fd_snapshot_name_from_cstr( &d->name, src->file.path );
149 0 : if( FD_UNLIKELY( !name ) ) {
150 0 : return NULL;
151 0 : }
152 :
153 0 : if( FD_UNLIKELY( validate_slot && fd_snapshot_name_slot_validate( name, base_slot ) ) ) {
154 0 : return NULL;
155 0 : }
156 :
157 0 : if( FD_UNLIKELY( !fd_io_istream_file_new( d->vfile, d->snapshot_fd ) ) ) {
158 0 : FD_LOG_WARNING(( "Failed to create fd_io_istream_file_t" ));
159 0 : return NULL;
160 0 : }
161 :
162 0 : d->vsrc = fd_io_istream_file_virtual( d->vfile );
163 0 : break;
164 0 : case FD_SNAPSHOT_SRC_HTTP:
165 0 : d->http = fd_snapshot_http_new( d->http_mem, src->http.dest, src->http.ip4, src->http.port, src->snapshot_dir, &d->name );
166 0 : if( FD_UNLIKELY( !d->http ) ) {
167 0 : FD_LOG_WARNING(( "Failed to create fd_snapshot_http_t" ));
168 0 : return NULL;
169 0 : }
170 0 : fd_snapshot_http_set_path( d->http, src->http.path, src->http.path_len, validate_slot ? base_slot : ULONG_MAX );
171 :
172 0 : d->vsrc = fd_io_istream_snapshot_http_virtual( d->http );
173 0 : break;
174 0 : default:
175 0 : __builtin_unreachable();
176 0 : }
177 :
178 : /* Set up the snapshot reader */
179 :
180 0 : if( FD_UNLIKELY( !fd_tar_reader_new( d->tar, &fd_snapshot_restore_tar_vt, d->restore ) ) ) {
181 0 : FD_LOG_WARNING(( "Failed to create fd_tar_reader_t" ));
182 0 : return NULL;
183 0 : }
184 :
185 0 : fd_zstd_dstream_reset( d->zstd );
186 :
187 0 : if( FD_UNLIKELY( !fd_io_istream_zstd_new( d->vzstd, d->zstd, d->vsrc ) ) ) {
188 0 : FD_LOG_WARNING(( "Failed to create fd_io_istream_zstd_t" ));
189 0 : return NULL;
190 0 : }
191 :
192 0 : if( FD_UNLIKELY( !fd_tar_io_reader_new( d->vtar, d->tar, fd_io_istream_zstd_virtual( d->vzstd ) ) ) ) {
193 0 : FD_LOG_WARNING(( "Failed to create fd_tar_io_reader_t" ));
194 0 : return NULL;
195 0 : }
196 :
197 0 : return d;
198 0 : }
199 :
200 : int
201 0 : fd_snapshot_loader_advance( fd_snapshot_loader_t * dumper ) {
202 :
203 0 : fd_tar_io_reader_t * vtar = dumper->vtar;
204 :
205 0 : int untar_err = fd_tar_io_reader_advance( vtar );
206 0 : if( untar_err==0 ) {
207 : /* Ok */
208 0 : } else if( untar_err==MANIFEST_DONE ) {
209 : /* Finished reading the manifest for the first time. */
210 0 : return MANIFEST_DONE;
211 0 : } else if( untar_err<0 ) {
212 : /* EOF */
213 0 : return -1;
214 0 : } else {
215 0 : FD_LOG_WARNING(( "Failed to load snapshot (%d-%s)", untar_err, fd_io_strerror( untar_err ) ));
216 0 : return untar_err;
217 0 : }
218 :
219 0 : return 0;
220 0 : }
221 :
222 : fd_snapshot_src_t *
223 : fd_snapshot_src_parse( fd_snapshot_src_t * src,
224 : char * cstr,
225 0 : int src_type ) {
226 :
227 0 : fd_memset( src, 0, sizeof(fd_snapshot_src_t) );
228 :
229 0 : if( FD_LIKELY( src_type==FD_SNAPSHOT_SRC_HTTP ) ) {
230 0 : static char const url_regex[] = "^http://([^:/[:space:]]+)(:[[:digit:]]+)?(/.*)?$";
231 0 : regex_t url_re;
232 0 : FD_TEST( 0==regcomp( &url_re, url_regex, REG_EXTENDED ) );
233 0 : regmatch_t group[4] = {0};
234 0 : int url_re_res = regexec( &url_re, cstr, 4, group, 0 );
235 0 : regfree( &url_re );
236 0 : if( FD_UNLIKELY( url_re_res!=0 ) ) {
237 0 : FD_LOG_WARNING(( "Bad URL: %s", cstr ));
238 0 : return NULL;
239 0 : }
240 :
241 0 : regmatch_t * m_hostname = &group[1];
242 0 : regmatch_t * m_port = &group[2];
243 0 : regmatch_t * m_path = &group[3];
244 :
245 0 : src->type = FD_SNAPSHOT_SRC_HTTP;
246 0 : src->http.path = cstr + m_path->rm_so;
247 0 : src->http.path_len = (ulong)m_path->rm_eo - (ulong)m_path->rm_so;
248 :
249 : /* Resolve port to IPv4 address */
250 :
251 0 : if( m_port->rm_so==m_port->rm_eo ) {
252 0 : src->http.port = 80;
253 0 : } else {
254 0 : char port_cstr[7] = {0};
255 0 : strncpy( port_cstr, cstr + m_port->rm_so,
256 0 : fd_ulong_min( 7, (ulong)m_port->rm_eo - (ulong)m_port->rm_so ) );
257 0 : char * port = port_cstr + 1;
258 0 : char * end;
259 0 : ulong port_ul = strtoul( port, &end, 10 );
260 0 : if( FD_UNLIKELY( *end!='\0' ) ) {
261 0 : FD_LOG_WARNING(( "Bad port: %s", port ));
262 0 : return NULL;
263 0 : }
264 0 : if( FD_UNLIKELY( port_ul>65535 ) ) {
265 0 : FD_LOG_WARNING(( "Port out of range: %lu", port_ul ));
266 0 : return NULL;
267 0 : }
268 0 : src->http.port = (ushort)port_ul;
269 0 : }
270 :
271 : /* Resolve host to IPv4 address */
272 :
273 0 : int sep = cstr[ m_hostname->rm_eo ];
274 0 : cstr[ m_hostname->rm_eo ] = '\0';
275 0 : char * hostname = cstr + m_hostname->rm_so;
276 :
277 0 : strncpy( src->http.dest, hostname, sizeof(src->http.dest)-1 );
278 0 : src->http.dest[ sizeof(src->http.dest)-1 ] = '\0';
279 :
280 0 : struct sockaddr_in default_addr = {
281 0 : .sin_family = AF_INET,
282 0 : .sin_port = htons( 80 ),
283 0 : .sin_addr = { .s_addr = htonl( INADDR_ANY ) }
284 0 : };
285 0 : struct addrinfo hints = {
286 0 : .ai_family = AF_INET,
287 0 : .ai_socktype = SOCK_STREAM,
288 0 : .ai_addr = fd_type_pun( &default_addr ),
289 0 : .ai_addrlen = sizeof(struct sockaddr_in)
290 0 : };
291 0 : struct addrinfo * result = NULL;
292 0 : int lookup_res = getaddrinfo( hostname, NULL, &hints, &result );
293 0 : if( FD_UNLIKELY( lookup_res ) ) {
294 0 : FD_LOG_WARNING(( "getaddrinfo(%s) failed (%d-%s)", hostname, lookup_res, gai_strerror( lookup_res ) ));
295 0 : return NULL;
296 0 : }
297 :
298 0 : cstr[ m_hostname->rm_eo ] = (char)sep;
299 :
300 0 : for( struct addrinfo * rp = result; rp; rp = rp->ai_next ) {
301 0 : if( rp->ai_family==AF_INET ) {
302 0 : struct sockaddr_in * addr = (struct sockaddr_in *)rp->ai_addr;
303 0 : src->http.ip4 = addr->sin_addr.s_addr;
304 0 : freeaddrinfo( result );
305 0 : return src;
306 0 : }
307 0 : }
308 :
309 0 : FD_LOG_WARNING(( "Failed to resolve socket address for %s", hostname ));
310 0 : freeaddrinfo( result );
311 0 : return NULL;
312 0 : }
313 :
314 0 : if( FD_UNLIKELY( src_type==FD_SNAPSHOT_SRC_FILE ) ) {
315 0 : src->type = FD_SNAPSHOT_SRC_FILE;
316 0 : src->file.path = cstr;
317 0 : return src;
318 0 : }
319 :
320 0 : FD_LOG_ERR(( "unrecognized snapshot src type %d", src_type ));
321 0 : }
322 :
323 : /* fd_snapshot_src_parse_type_unknown determines the source from the
324 : given cstr.
325 :
326 : Should only be used for testing and dev. Production validators
327 : explicitly set the snapshot src type in the config file.
328 : */
329 :
330 : fd_snapshot_src_t *
331 : fd_snapshot_src_parse_type_unknown( fd_snapshot_src_t * src,
332 0 : char * cstr ) {
333 :
334 0 : if( 0==strncmp( cstr, "http://", 7 ) ) {
335 0 : return fd_snapshot_src_parse( src, cstr, FD_SNAPSHOT_SRC_HTTP );
336 0 : } else {
337 0 : return fd_snapshot_src_parse( src, cstr, FD_SNAPSHOT_SRC_FILE );
338 0 : }
339 :
340 0 : __builtin_unreachable();
341 0 : }
342 :
343 : fd_snapshot_name_t const * /* nullable */
344 0 : fd_snapshot_loader_get_name( fd_snapshot_loader_t const * loader ) {
345 0 : return &loader->name;
346 0 : }
|
