Line data Source code
1 : #include "fd_snapshot_loader.h"
2 : #include "fd_snapshot.h"
3 : #include "fd_snapshot_restore.h"
4 : #include "fd_snapshot_http.h"
5 :
6 : #include <errno.h>
7 : #include <fcntl.h>
8 : #include <netdb.h>
9 : #include <regex.h>
10 : #include <unistd.h>
11 : #include <sys/types.h>
12 : #include <sys/socket.h>
13 : #include <netinet/in.h>
14 :
15 : struct fd_snapshot_loader {
16 : ulong magic;
17 :
18 : /* Source: HTTP */
19 :
20 : void * http_mem;
21 : fd_snapshot_http_t * http;
22 :
23 : /* Source: File I/O */
24 :
25 : int snapshot_fd;
26 : fd_io_istream_file_t vfile[1];
27 :
28 : /* Source I/O abstraction */
29 :
30 : fd_io_istream_obj_t vsrc;
31 :
32 : /* Zstandard decompressor */
33 :
34 : fd_zstd_dstream_t * zstd;
35 : fd_io_istream_zstd_t vzstd[1];
36 :
37 : /* Tar reader */
38 :
39 : fd_tar_reader_t tar[1];
40 : fd_tar_io_reader_t vtar[1];
41 :
42 : /* Downstream restore */
43 :
44 : fd_snapshot_restore_t * restore;
45 :
46 : /* Hash and slot numbers from filename */
47 :
48 : fd_snapshot_name_t name;
49 : };
50 :
51 : typedef struct fd_snapshot_loader fd_snapshot_loader_t;
52 :
53 0 : #define FD_SNAPSHOT_LOADER_MAGIC (0xa78a73a69d33e6b1UL)
54 :
55 : ulong
56 0 : fd_snapshot_loader_align( void ) {
57 0 : return fd_ulong_max( alignof(fd_snapshot_loader_t), fd_zstd_dstream_align() );
58 0 : }
59 :
60 : ulong
61 0 : fd_snapshot_loader_footprint( ulong zstd_window_sz ) {
62 0 : ulong l = FD_LAYOUT_INIT;
63 0 : l = FD_LAYOUT_APPEND( l, alignof(fd_snapshot_loader_t), sizeof(fd_snapshot_loader_t) );
64 0 : l = FD_LAYOUT_APPEND( l, fd_zstd_dstream_align(), fd_zstd_dstream_footprint( zstd_window_sz ) );
65 0 : l = FD_LAYOUT_APPEND( l, alignof(fd_snapshot_http_t), sizeof(fd_snapshot_http_t) );
66 : /* FIXME add test ensuring zstd dstream align > alignof loader */
67 0 : return FD_LAYOUT_FINI( l, fd_snapshot_loader_align() );
68 0 : }
69 :
70 : fd_snapshot_loader_t *
71 : fd_snapshot_loader_new( void * mem,
72 0 : ulong zstd_window_sz ) {
73 :
74 0 : if( FD_UNLIKELY( !mem ) ) {
75 0 : FD_LOG_WARNING(( "NULL mem" ));
76 0 : return NULL;
77 0 : }
78 :
79 0 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)mem, fd_snapshot_loader_align() ) ) ) {
80 0 : FD_LOG_WARNING(( "unaligned mem" ));
81 0 : return NULL;
82 0 : }
83 :
84 0 : FD_SCRATCH_ALLOC_INIT( l, mem );
85 0 : fd_snapshot_loader_t * loader = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_snapshot_loader_t), sizeof(fd_snapshot_loader_t) );
86 0 : void * zstd_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_zstd_dstream_align(), fd_zstd_dstream_footprint( zstd_window_sz ) );
87 0 : void * http_mem = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_snapshot_http_t), sizeof(fd_snapshot_http_t) );
88 0 : FD_SCRATCH_ALLOC_FINI( l, fd_snapshot_loader_align() );
89 :
90 0 : fd_memset( loader, 0, sizeof(fd_snapshot_loader_t) );
91 0 : loader->http_mem = http_mem;
92 0 : loader->zstd = fd_zstd_dstream_new( zstd_mem, zstd_window_sz );
93 :
94 0 : FD_COMPILER_MFENCE();
95 0 : loader->magic = FD_SNAPSHOT_LOADER_MAGIC;
96 0 : FD_COMPILER_MFENCE();
97 :
98 0 : return loader;
99 0 : }
100 :
101 : void *
102 0 : fd_snapshot_loader_delete( fd_snapshot_loader_t * loader ) {
103 :
104 0 : if( FD_UNLIKELY( !loader ) ) return NULL;
105 :
106 0 : if( FD_UNLIKELY( loader->magic != FD_SNAPSHOT_LOADER_MAGIC ) ) {
107 0 : FD_LOG_WARNING(( "invalid magic" ));
108 0 : return NULL;
109 0 : }
110 :
111 0 : fd_zstd_dstream_delete ( loader->zstd );
112 0 : fd_tar_io_reader_delete ( loader->vtar );
113 0 : fd_io_istream_zstd_delete( loader->vzstd );
114 0 : fd_io_istream_file_delete( loader->vfile );
115 0 : fd_snapshot_http_delete ( loader->http );
116 0 : fd_tar_reader_delete ( loader->tar );
117 :
118 0 : if( loader->snapshot_fd>=0 ) {
119 0 : if( FD_UNLIKELY( 0!=close( loader->snapshot_fd ) ) )
120 0 : FD_LOG_WARNING(( "close(%d) failed (%d-%s)", loader->snapshot_fd, errno, fd_io_strerror( errno ) ));
121 0 : loader->snapshot_fd = -1;
122 0 : }
123 :
124 0 : FD_COMPILER_MFENCE();
125 0 : loader->magic = 0UL;
126 0 : FD_COMPILER_MFENCE();
127 :
128 0 : return loader;
129 0 : }
130 :
131 : fd_snapshot_loader_t *
132 : fd_snapshot_loader_init( fd_snapshot_loader_t * d,
133 : fd_snapshot_restore_t * restore,
134 : fd_snapshot_src_t const * src,
135 0 : ulong base_slot ) {
136 :
137 0 : d->restore = restore;
138 :
139 0 : switch( src->type ) {
140 0 : case FD_SNAPSHOT_SRC_FILE:
141 0 : d->snapshot_fd = open( src->file.path, O_RDONLY );
142 0 : if( FD_UNLIKELY( d->snapshot_fd<0 ) ) {
143 0 : FD_LOG_WARNING(( "open(%s) failed (%d-%s)", src->file.path, errno, fd_io_strerror( errno ) ));
144 0 : return NULL;
145 0 : }
146 :
147 0 : if( FD_UNLIKELY( !fd_snapshot_name_from_cstr( &d->name, src->file.path, base_slot ) ) ) {
148 0 : return NULL;
149 0 : }
150 :
151 0 : if( FD_UNLIKELY( !fd_io_istream_file_new( d->vfile, d->snapshot_fd ) ) ) {
152 0 : FD_LOG_WARNING(( "Failed to create fd_io_istream_file_t" ));
153 0 : return NULL;
154 0 : }
155 :
156 0 : d->vsrc = fd_io_istream_file_virtual( d->vfile );
157 0 : break;
158 0 : case FD_SNAPSHOT_SRC_HTTP:
159 0 : d->http = fd_snapshot_http_new( d->http_mem, src->http.dest, src->http.ip4, src->http.port, &d->name );
160 0 : if( FD_UNLIKELY( !d->http ) ) {
161 0 : FD_LOG_WARNING(( "Failed to create fd_snapshot_http_t" ));
162 0 : return NULL;
163 0 : }
164 0 : fd_snapshot_http_set_path( d->http, src->http.path, src->http.path_len, base_slot );
165 0 : d->http->hops = (ushort)3; /* TODO don't hardcode */
166 :
167 0 : d->vsrc = fd_io_istream_snapshot_http_virtual( d->http );
168 0 : break;
169 0 : default:
170 0 : __builtin_unreachable();
171 0 : }
172 :
173 : /* Set up the snapshot reader */
174 :
175 0 : if( FD_UNLIKELY( !fd_tar_reader_new( d->tar, &fd_snapshot_restore_tar_vt, d->restore ) ) ) {
176 0 : FD_LOG_WARNING(( "Failed to create fd_tar_reader_t" ));
177 0 : return NULL;
178 0 : }
179 :
180 0 : fd_zstd_dstream_reset( d->zstd );
181 :
182 0 : if( FD_UNLIKELY( !fd_io_istream_zstd_new( d->vzstd, d->zstd, d->vsrc ) ) ) {
183 0 : FD_LOG_WARNING(( "Failed to create fd_io_istream_zstd_t" ));
184 0 : return NULL;
185 0 : }
186 :
187 0 : if( FD_UNLIKELY( !fd_tar_io_reader_new( d->vtar, d->tar, fd_io_istream_zstd_virtual( d->vzstd ) ) ) ) {
188 0 : FD_LOG_WARNING(( "Failed to create fd_tar_io_reader_t" ));
189 0 : return NULL;
190 0 : }
191 :
192 0 : return d;
193 0 : }
194 :
195 : int
196 0 : fd_snapshot_loader_advance( fd_snapshot_loader_t * dumper ) {
197 :
198 0 : fd_tar_io_reader_t * vtar = dumper->vtar;
199 :
200 0 : int untar_err = fd_tar_io_reader_advance( vtar );
201 0 : if( untar_err==0 ) { /* ok */ }
202 0 : else if( untar_err<0 ) { /* EOF */ return -1; }
203 0 : else {
204 0 : FD_LOG_WARNING(( "Failed to load snapshot (%d-%s)", untar_err, fd_io_strerror( untar_err ) ));
205 0 : return untar_err;
206 0 : }
207 :
208 0 : return 0;
209 0 : }
210 :
211 : /* fd_snapshot_src_parse determines the source from the given cstr. */
212 :
213 : fd_snapshot_src_t *
214 : fd_snapshot_src_parse( fd_snapshot_src_t * src,
215 0 : char * cstr ) {
216 :
217 0 : fd_memset( src, 0, sizeof(fd_snapshot_src_t) );
218 :
219 0 : if( 0==strncmp( cstr, "http://", 7 ) ) {
220 0 : static char const url_regex[] = "^http://([^:/[:space:]]+)(:[[:digit:]]+)?(/.*)?$";
221 0 : regex_t url_re;
222 0 : FD_TEST( 0==regcomp( &url_re, url_regex, REG_EXTENDED ) );
223 0 : regmatch_t group[4] = {0};
224 0 : int url_re_res = regexec( &url_re, cstr, 4, group, 0 );
225 0 : regfree( &url_re );
226 0 : if( FD_UNLIKELY( url_re_res!=0 ) ) {
227 0 : FD_LOG_WARNING(( "Bad URL: %s", cstr ));
228 0 : return NULL;
229 0 : }
230 :
231 0 : regmatch_t * m_hostname = &group[1];
232 0 : regmatch_t * m_port = &group[2];
233 0 : regmatch_t * m_path = &group[3];
234 :
235 0 : src->type = FD_SNAPSHOT_SRC_HTTP;
236 0 : src->http.path = cstr + m_path->rm_so;
237 0 : src->http.path_len = (ulong)m_path->rm_eo - (ulong)m_path->rm_so;
238 :
239 : /* Resolve port to IPv4 address */
240 :
241 0 : if( m_port->rm_so==m_port->rm_eo ) {
242 0 : src->http.port = 80;
243 0 : } else {
244 0 : char port_cstr[7] = {0};
245 0 : strncpy( port_cstr, cstr + m_port->rm_so,
246 0 : fd_ulong_min( 7, (ulong)m_port->rm_eo - (ulong)m_port->rm_so ) );
247 0 : char * port = port_cstr + 1;
248 0 : char * end;
249 0 : ulong port_ul = strtoul( port, &end, 10 );
250 0 : if( FD_UNLIKELY( *end!='\0' ) ) {
251 0 : FD_LOG_WARNING(( "Bad port: %s", port ));
252 0 : return NULL;
253 0 : }
254 0 : if( FD_UNLIKELY( port_ul>65535 ) ) {
255 0 : FD_LOG_WARNING(( "Port out of range: %lu", port_ul ));
256 0 : return NULL;
257 0 : }
258 0 : src->http.port = (ushort)port_ul;
259 0 : }
260 :
261 : /* Resolve host to IPv4 address */
262 :
263 0 : int sep = cstr[ m_hostname->rm_eo ];
264 0 : cstr[ m_hostname->rm_eo ] = '\0';
265 0 : char * hostname = cstr + m_hostname->rm_so;
266 :
267 0 : strncpy( src->http.dest, hostname, sizeof(src->http.dest)-1 );
268 0 : src->http.dest[ sizeof(src->http.dest)-1 ] = '\0';
269 :
270 0 : struct sockaddr_in default_addr = {
271 0 : .sin_family = AF_INET,
272 0 : .sin_port = htons( 80 ),
273 0 : .sin_addr = { .s_addr = htonl( INADDR_ANY ) }
274 0 : };
275 0 : struct addrinfo hints = {
276 0 : .ai_family = AF_INET,
277 0 : .ai_socktype = SOCK_STREAM,
278 0 : .ai_addr = fd_type_pun( &default_addr ),
279 0 : .ai_addrlen = sizeof(struct sockaddr_in)
280 0 : };
281 0 : struct addrinfo * result = NULL;
282 0 : int lookup_res = getaddrinfo( hostname, NULL, &hints, &result );
283 0 : if( FD_UNLIKELY( lookup_res ) ) {
284 0 : FD_LOG_WARNING(( "getaddrinfo(%s) failed (%d-%s)", hostname, lookup_res, gai_strerror( lookup_res ) ));
285 0 : return NULL;
286 0 : }
287 :
288 0 : cstr[ m_hostname->rm_eo ] = (char)sep;
289 :
290 0 : for( struct addrinfo * rp = result; rp; rp = rp->ai_next ) {
291 0 : if( rp->ai_family==AF_INET ) {
292 0 : struct sockaddr_in * addr = (struct sockaddr_in *)rp->ai_addr;
293 0 : src->http.ip4 = addr->sin_addr.s_addr;
294 0 : freeaddrinfo( result );
295 0 : return src;
296 0 : }
297 0 : }
298 :
299 0 : FD_LOG_WARNING(( "Failed to resolve socket address for %s", hostname ));
300 0 : freeaddrinfo( result );
301 0 : return NULL;
302 0 : } else {
303 0 : src->type = FD_SNAPSHOT_SRC_FILE;
304 0 : src->file.path = cstr;
305 0 : return src;
306 0 : }
307 :
308 0 : __builtin_unreachable();
309 0 : }
310 :
311 : fd_snapshot_name_t const * /* nullable */
312 0 : fd_snapshot_loader_get_name( fd_snapshot_loader_t const * loader ) {
313 0 : return &loader->name;
314 0 : }
|
