Line data Source code
1 : #include "fd_snapshot_loader.h"
2 : #include "fd_snapshot.h"
3 : #include "fd_snapshot_restore.h"
4 : #include "fd_snapshot_http.h"
5 :
6 : #include <errno.h>
7 : #include <fcntl.h>
8 : #include <netdb.h>
9 : #include <regex.h>
10 : #include <unistd.h>
11 : #include <sys/types.h>
12 : #include <sys/socket.h>
13 : #include <netinet/in.h>
14 :
15 : struct fd_snapshot_loader {
16 : ulong magic;
17 :
18 : /* Source: HTTP */
19 :
20 : void * http_mem;
21 : fd_snapshot_http_t * http;
22 :
23 : /* Source: File I/O */
24 :
25 : int snapshot_fd;
26 : fd_io_istream_file_t vfile[1];
27 :
28 : /* Source I/O abstraction */
29 :
30 : fd_io_istream_obj_t vsrc;
31 :
32 : /* Zstandard decompressor */
33 :
34 : fd_zstd_dstream_t * zstd;
35 : fd_io_istream_zstd_t vzstd[1];
36 :
37 : /* Tar reader */
38 :
39 : fd_tar_reader_t tar[1];
40 : fd_tar_io_reader_t vtar[1];
41 :
42 : /* Downstream restore */
43 :
44 : fd_snapshot_restore_t * restore;
45 :
46 : /* Hash and slot numbers from filename */
47 :
48 : fd_snapshot_name_t name;
49 : };
50 :
51 : typedef struct fd_snapshot_loader fd_snapshot_loader_t;
52 :
53 0 : #define FD_SNAPSHOT_LOADER_MAGIC (0xa78a73a69d33e6b1UL)
54 :
55 : ulong
56 0 : fd_snapshot_loader_align( void ) {
57 0 : return fd_ulong_max( alignof(fd_snapshot_loader_t), fd_zstd_dstream_align() );
58 0 : }
59 :
60 : ulong
61 0 : fd_snapshot_loader_footprint( ulong zstd_window_sz ) {
62 0 : ulong l = FD_LAYOUT_INIT;
63 0 : l = FD_LAYOUT_APPEND( l, alignof(fd_snapshot_loader_t), sizeof(fd_snapshot_loader_t) );
64 0 : l = FD_LAYOUT_APPEND( l, fd_zstd_dstream_align(), fd_zstd_dstream_footprint( zstd_window_sz ) );
65 0 : l = FD_LAYOUT_APPEND( l, alignof(fd_snapshot_http_t), sizeof(fd_snapshot_http_t) );
66 : /* FIXME add test ensuring zstd dstream align > alignof loader */
67 0 : return FD_LAYOUT_FINI( l, fd_snapshot_loader_align() );
68 0 : }
69 :
70 : fd_snapshot_loader_t *
71 : fd_snapshot_loader_new( void * mem,
72 0 : ulong zstd_window_sz ) {
73 :
74 0 : if( FD_UNLIKELY( !mem ) ) {
75 0 : FD_LOG_WARNING(( "NULL mem" ));
76 0 : return NULL;
77 0 : }
78 :
79 0 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)mem, fd_snapshot_loader_align() ) ) ) {
80 0 : FD_LOG_WARNING(( "unaligned mem" ));
81 0 : return NULL;
82 0 : }
83 :
84 0 : FD_SCRATCH_ALLOC_INIT( l, mem );
85 0 : fd_snapshot_loader_t * loader = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_snapshot_loader_t), sizeof(fd_snapshot_loader_t) );
86 0 : void * zstd_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_zstd_dstream_align(), fd_zstd_dstream_footprint( zstd_window_sz ) );
87 0 : void * http_mem = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_snapshot_http_t), sizeof(fd_snapshot_http_t) );
88 0 : FD_SCRATCH_ALLOC_FINI( l, fd_snapshot_loader_align() );
89 :
90 0 : fd_memset( loader, 0, sizeof(fd_snapshot_loader_t) );
91 0 : loader->http_mem = http_mem;
92 0 : loader->zstd = fd_zstd_dstream_new( zstd_mem, zstd_window_sz );
93 :
94 0 : FD_COMPILER_MFENCE();
95 0 : loader->magic = FD_SNAPSHOT_LOADER_MAGIC;
96 0 : FD_COMPILER_MFENCE();
97 :
98 0 : return loader;
99 0 : }
100 :
101 : void *
102 0 : fd_snapshot_loader_delete( fd_snapshot_loader_t * loader ) {
103 :
104 0 : if( FD_UNLIKELY( !loader ) ) return NULL;
105 :
106 0 : if( FD_UNLIKELY( loader->magic != FD_SNAPSHOT_LOADER_MAGIC ) ) {
107 0 : FD_LOG_WARNING(( "invalid magic" ));
108 0 : return NULL;
109 0 : }
110 :
111 0 : fd_zstd_dstream_delete ( loader->zstd );
112 0 : fd_tar_io_reader_delete ( loader->vtar );
113 0 : fd_io_istream_zstd_delete( loader->vzstd );
114 0 : fd_io_istream_file_delete( loader->vfile );
115 0 : fd_snapshot_http_delete ( loader->http );
116 0 : fd_tar_reader_delete ( loader->tar );
117 0 : fd_zstd_dstream_delete ( loader->zstd );
118 :
119 0 : if( loader->snapshot_fd>=0 ) {
120 0 : if( FD_UNLIKELY( 0!=close( loader->snapshot_fd ) ) )
121 0 : FD_LOG_WARNING(( "close(%d) failed (%d-%s)", loader->snapshot_fd, errno, fd_io_strerror( errno ) ));
122 0 : loader->snapshot_fd = -1;
123 0 : }
124 :
125 0 : FD_COMPILER_MFENCE();
126 0 : loader->magic = 0UL;
127 0 : FD_COMPILER_MFENCE();
128 :
129 0 : return loader;
130 0 : }
131 :
132 : fd_snapshot_loader_t *
133 : fd_snapshot_loader_init( fd_snapshot_loader_t * d,
134 : fd_snapshot_restore_t * restore,
135 : fd_snapshot_src_t const * src,
136 0 : ulong base_slot ) {
137 :
138 0 : d->restore = restore;
139 :
140 0 : switch( src->type ) {
141 0 : case FD_SNAPSHOT_SRC_FILE:
142 0 : d->snapshot_fd = open( src->file.path, O_RDONLY );
143 0 : if( FD_UNLIKELY( d->snapshot_fd<0 ) ) {
144 0 : FD_LOG_WARNING(( "open(%s) failed (%d-%s)", src->file.path, errno, fd_io_strerror( errno ) ));
145 0 : return NULL;
146 0 : }
147 :
148 0 : if( FD_UNLIKELY( !fd_snapshot_name_from_cstr( &d->name, src->file.path, base_slot ) ) ) {
149 0 : return NULL;
150 0 : }
151 :
152 0 : if( FD_UNLIKELY( !fd_io_istream_file_new( d->vfile, d->snapshot_fd ) ) ) {
153 0 : FD_LOG_WARNING(( "Failed to create fd_io_istream_file_t" ));
154 0 : return NULL;
155 0 : }
156 :
157 0 : d->vsrc = fd_io_istream_file_virtual( d->vfile );
158 0 : break;
159 0 : case FD_SNAPSHOT_SRC_HTTP:
160 0 : d->http = fd_snapshot_http_new( d->http_mem, src->http.dest, src->http.ip4, src->http.port, &d->name );
161 0 : if( FD_UNLIKELY( !d->http ) ) {
162 0 : FD_LOG_WARNING(( "Failed to create fd_snapshot_http_t" ));
163 0 : return NULL;
164 0 : }
165 0 : fd_snapshot_http_set_path( d->http, src->http.path, src->http.path_len, base_slot );
166 0 : d->http->hops = (ushort)3; /* TODO don't hardcode */
167 :
168 0 : d->vsrc = fd_io_istream_snapshot_http_virtual( d->http );
169 0 : break;
170 0 : default:
171 0 : __builtin_unreachable();
172 0 : }
173 :
174 : /* Set up the snapshot reader */
175 :
176 0 : if( FD_UNLIKELY( !fd_tar_reader_new( d->tar, &fd_snapshot_restore_tar_vt, d->restore ) ) ) {
177 0 : FD_LOG_WARNING(( "Failed to create fd_tar_reader_t" ));
178 0 : return NULL;
179 0 : }
180 :
181 0 : fd_zstd_dstream_reset( d->zstd );
182 :
183 0 : if( FD_UNLIKELY( !fd_io_istream_zstd_new( d->vzstd, d->zstd, d->vsrc ) ) ) {
184 0 : FD_LOG_WARNING(( "Failed to create fd_io_istream_zstd_t" ));
185 0 : return NULL;
186 0 : }
187 :
188 0 : if( FD_UNLIKELY( !fd_tar_io_reader_new( d->vtar, d->tar, fd_io_istream_zstd_virtual( d->vzstd ) ) ) ) {
189 0 : FD_LOG_WARNING(( "Failed to create fd_tar_io_reader_t" ));
190 0 : return NULL;
191 0 : }
192 :
193 0 : return d;
194 0 : }
195 :
196 : int
197 0 : fd_snapshot_loader_advance( fd_snapshot_loader_t * dumper ) {
198 :
199 0 : fd_tar_io_reader_t * vtar = dumper->vtar;
200 :
201 0 : int untar_err = fd_tar_io_reader_advance( vtar );
202 0 : if( untar_err==0 ) { /* ok */ }
203 0 : else if( untar_err<0 ) { /* EOF */ return -1; }
204 0 : else {
205 0 : FD_LOG_WARNING(( "Failed to load snapshot (%d-%s)", untar_err, fd_io_strerror( untar_err ) ));
206 0 : return untar_err;
207 0 : }
208 :
209 0 : return 0;
210 0 : }
211 :
212 : /* fd_snapshot_src_parse determines the source from the given cstr. */
213 :
214 : fd_snapshot_src_t *
215 : fd_snapshot_src_parse( fd_snapshot_src_t * src,
216 0 : char * cstr ) {
217 :
218 0 : fd_memset( src, 0, sizeof(fd_snapshot_src_t) );
219 :
220 0 : if( 0==strncmp( cstr, "http://", 7 ) ) {
221 0 : static char const url_regex[] = "^http://([^:/[:space:]]+)(:[[:digit:]]+)?(/.*)?$";
222 0 : regex_t url_re;
223 0 : FD_TEST( 0==regcomp( &url_re, url_regex, REG_EXTENDED ) );
224 0 : regmatch_t group[4] = {0};
225 0 : int url_re_res = regexec( &url_re, cstr, 4, group, 0 );
226 0 : regfree( &url_re );
227 0 : if( FD_UNLIKELY( url_re_res!=0 ) ) {
228 0 : FD_LOG_WARNING(( "Bad URL: %s", cstr ));
229 0 : return NULL;
230 0 : }
231 :
232 0 : regmatch_t * m_hostname = &group[1];
233 0 : regmatch_t * m_port = &group[2];
234 0 : regmatch_t * m_path = &group[3];
235 :
236 0 : src->type = FD_SNAPSHOT_SRC_HTTP;
237 0 : src->http.path = cstr + m_path->rm_so;
238 0 : src->http.path_len = (ulong)m_path->rm_eo - (ulong)m_path->rm_so;
239 :
240 : /* Resolve port to IPv4 address */
241 :
242 0 : if( m_port->rm_so==m_port->rm_eo ) {
243 0 : src->http.port = 80;
244 0 : } else {
245 0 : char port_cstr[7] = {0};
246 0 : strncpy( port_cstr, cstr + m_port->rm_so,
247 0 : fd_ulong_min( 7, (ulong)m_port->rm_eo - (ulong)m_port->rm_so ) );
248 0 : char * port = port_cstr + 1;
249 0 : char * end;
250 0 : ulong port_ul = strtoul( port, &end, 10 );
251 0 : if( FD_UNLIKELY( *end!='\0' ) ) {
252 0 : FD_LOG_WARNING(( "Bad port: %s", port ));
253 0 : return NULL;
254 0 : }
255 0 : if( FD_UNLIKELY( port_ul>65535 ) ) {
256 0 : FD_LOG_WARNING(( "Port out of range: %lu", port_ul ));
257 0 : return NULL;
258 0 : }
259 0 : src->http.port = (ushort)port_ul;
260 0 : }
261 :
262 : /* Resolve host to IPv4 address */
263 :
264 0 : int sep = cstr[ m_hostname->rm_eo ];
265 0 : cstr[ m_hostname->rm_eo ] = '\0';
266 0 : char * hostname = cstr + m_hostname->rm_so;
267 :
268 0 : strncpy( src->http.dest, hostname, sizeof(src->http.dest)-1 );
269 0 : src->http.dest[ sizeof(src->http.dest)-1 ] = '\0';
270 :
271 0 : struct sockaddr_in default_addr = {
272 0 : .sin_family = AF_INET,
273 0 : .sin_port = htons( 80 ),
274 0 : .sin_addr = { .s_addr = htonl( INADDR_ANY ) }
275 0 : };
276 0 : struct addrinfo hints = {
277 0 : .ai_family = AF_INET,
278 0 : .ai_socktype = SOCK_STREAM,
279 0 : .ai_addr = fd_type_pun( &default_addr ),
280 0 : .ai_addrlen = sizeof(struct sockaddr_in)
281 0 : };
282 0 : struct addrinfo * result = NULL;
283 0 : int lookup_res = getaddrinfo( hostname, NULL, &hints, &result );
284 0 : if( FD_UNLIKELY( lookup_res ) ) {
285 0 : FD_LOG_WARNING(( "getaddrinfo(%s) failed (%d-%s)", hostname, lookup_res, gai_strerror( lookup_res ) ));
286 0 : return NULL;
287 0 : }
288 :
289 0 : cstr[ m_hostname->rm_eo ] = (char)sep;
290 :
291 0 : for( struct addrinfo * rp = result; rp; rp = rp->ai_next ) {
292 0 : if( rp->ai_family==AF_INET ) {
293 0 : struct sockaddr_in * addr = (struct sockaddr_in *)rp->ai_addr;
294 0 : src->http.ip4 = addr->sin_addr.s_addr;
295 0 : freeaddrinfo( result );
296 0 : return src;
297 0 : }
298 0 : }
299 :
300 0 : FD_LOG_WARNING(( "Failed to resolve socket address for %s", hostname ));
301 0 : freeaddrinfo( result );
302 0 : return NULL;
303 0 : } else if( 0==strncmp( cstr, "archive:", sizeof("archive:")-1 ) ) {
304 0 : src->type = FD_SNAPSHOT_SRC_ARCHIVE;
305 0 : src->file.path = cstr + (sizeof("archive:")-1);
306 0 : return src;
307 0 : } else {
308 0 : src->type = FD_SNAPSHOT_SRC_FILE;
309 0 : src->file.path = cstr;
310 0 : return src;
311 0 : }
312 :
313 0 : __builtin_unreachable();
314 0 : }
315 :
316 : fd_snapshot_name_t const * /* nullable */
317 0 : fd_snapshot_loader_get_name( fd_snapshot_loader_t const * loader ) {
318 0 : return &loader->name;
319 0 : }
|
