Line data Source code
1 : #define _GNU_SOURCE
2 : #include "fd_sshttp.h"
3 : #include "fd_ssarchive.h"
4 :
5 : #include "../../../waltz/http/picohttpparser.h"
6 : #include "../../../util/log/fd_log.h"
7 : #include "../../../flamenco/types/fd_types_custom.h"
8 :
9 : #include <unistd.h>
10 : #include <errno.h>
11 : #include <stdlib.h>
12 :
13 : #include <sys/socket.h>
14 : #include <netinet/tcp.h>
15 : #include <netinet/in.h>
16 :
17 0 : #define FD_SSHTTP_STATE_INIT (0) /* start */
18 0 : #define FD_SSHTTP_STATE_REQ (1) /* sending request */
19 0 : #define FD_SSHTTP_STATE_RESP (2) /* receiving response headers */
20 0 : #define FD_SSHTTP_STATE_DL (3) /* downloading response body */
21 :
22 : struct fd_sshttp_private {
23 : int state;
24 : long deadline;
25 : int full;
26 :
27 : int hops;
28 :
29 : fd_ip4_port_t addr;
30 : int sockfd;
31 :
32 : char request[ 4096UL ];
33 : ulong request_len;
34 : ulong request_sent;
35 :
36 : ulong response_len;
37 : char response[ USHORT_MAX ];
38 :
39 : char full_snapshot_name[ PATH_MAX ];
40 : char incremental_snapshot_name[ PATH_MAX ];
41 :
42 : ulong content_len;
43 : ulong content_read;
44 :
45 : ulong magic;
46 : };
47 :
48 : FD_FN_CONST ulong
49 0 : fd_sshttp_align( void ) {
50 0 : return FD_SSHTTP_ALIGN;
51 0 : }
52 :
53 : FD_FN_CONST ulong
54 0 : fd_sshttp_footprint( void ) {
55 0 : ulong l;
56 0 : l = FD_LAYOUT_INIT;
57 0 : l = FD_LAYOUT_APPEND( l, FD_SSHTTP_ALIGN, sizeof(fd_sshttp_t) );
58 0 : return FD_LAYOUT_FINI( l, FD_SSHTTP_ALIGN );
59 0 : }
60 :
61 : void *
62 0 : fd_sshttp_new( void * shmem ) {
63 0 : if( FD_UNLIKELY( !shmem ) ) {
64 0 : FD_LOG_WARNING(( "NULL shmem" ));
65 0 : return NULL;
66 0 : }
67 :
68 0 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)shmem, fd_sshttp_align() ) ) ) {
69 0 : FD_LOG_WARNING(( "unaligned shmem" ));
70 0 : return NULL;
71 0 : }
72 :
73 0 : FD_SCRATCH_ALLOC_INIT( l, shmem );
74 0 : fd_sshttp_t * sshttp = FD_SCRATCH_ALLOC_APPEND( l, FD_SSHTTP_ALIGN, sizeof(fd_sshttp_t) );
75 :
76 0 : sshttp->state = FD_SSHTTP_STATE_INIT;
77 0 : sshttp->full_snapshot_name[ 0 ] = '\0';
78 0 : sshttp->incremental_snapshot_name[ 0 ] = '\0';
79 :
80 0 : FD_COMPILER_MFENCE();
81 0 : FD_VOLATILE( sshttp->magic ) = FD_SSHTTP_MAGIC;
82 0 : FD_COMPILER_MFENCE();
83 :
84 0 : return (void *)sshttp;
85 0 : }
86 :
87 : fd_sshttp_t *
88 0 : fd_sshttp_join( void * shhttp ) {
89 0 : if( FD_UNLIKELY( !shhttp ) ) {
90 0 : FD_LOG_WARNING(( "NULL shhttp" ));
91 0 : return NULL;
92 0 : }
93 :
94 0 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)shhttp, fd_sshttp_align() ) ) ) {
95 0 : FD_LOG_WARNING(( "misaligned shhttp" ));
96 0 : return NULL;
97 0 : }
98 :
99 0 : fd_sshttp_t * sshttp = (fd_sshttp_t *)shhttp;
100 :
101 0 : if( FD_UNLIKELY( sshttp->magic!=FD_SSHTTP_MAGIC ) ) {
102 0 : FD_LOG_WARNING(( "bad magic" ));
103 0 : return NULL;
104 0 : }
105 :
106 0 : return sshttp;
107 0 : }
108 :
109 : void
110 : fd_sshttp_init( fd_sshttp_t * http,
111 : fd_ip4_port_t addr,
112 : char const * path,
113 : ulong path_len,
114 0 : long now ) {
115 0 : FD_TEST( http->state==FD_SSHTTP_STATE_INIT );
116 :
117 0 : http->hops = 4UL;
118 :
119 0 : http->request_sent = 0UL;
120 0 : FD_TEST( fd_cstr_printf_check( http->request, sizeof(http->request), &http->request_len,
121 0 : "GET %.*s HTTP/1.1\r\n"
122 0 : "User-Agent: Firedancer\r\n"
123 0 : "Accept: */*\r\n"
124 0 : "Accept-Encoding: identity\r\n"
125 0 : "Host: " FD_IP4_ADDR_FMT "\r\n\r\n",
126 0 : (int)path_len, path, FD_IP4_ADDR_FMT_ARGS( addr.addr ) ) );
127 :
128 : /* TODO: Figure out recv coalescing properly and switch stream back
129 : to non-blocking. */
130 0 : http->addr = addr;
131 0 : http->sockfd = socket( AF_INET, SOCK_STREAM, 0 );
132 0 : if( FD_UNLIKELY( -1==http->sockfd ) ) FD_LOG_ERR(( "socket() failed (%d-%s)", errno, fd_io_strerror( errno ) ));
133 :
134 0 : struct timeval timeout = {
135 0 : .tv_sec = 0UL,
136 0 : .tv_usec = (ulong)10e3,
137 0 : };
138 :
139 0 : if( FD_UNLIKELY( -1==setsockopt( http->sockfd, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof(timeout) ) ) ) FD_LOG_ERR(("setsockopt SO_RCVTIMEO failed (%d-%s)", errno, fd_io_strerror(errno)));
140 :
141 0 : struct sockaddr_in addr_in = {
142 0 : .sin_family = AF_INET,
143 0 : .sin_port = fd_ushort_bswap( addr.port ),
144 0 : .sin_addr = { .s_addr = addr.addr }
145 0 : };
146 :
147 0 : if( FD_UNLIKELY( -1==connect( http->sockfd, fd_type_pun_const( &addr_in ), sizeof(addr_in) ) ) ) {
148 0 : if( FD_UNLIKELY( errno!=EINPROGRESS ) ) {
149 0 : if( FD_UNLIKELY( -1==close( http->sockfd ) ) ) FD_LOG_ERR(( "close() failed (%d-%s)", errno, fd_io_strerror( errno ) ));
150 0 : return;
151 0 : }
152 0 : }
153 :
154 0 : http->state = FD_SSHTTP_STATE_REQ;
155 0 : http->deadline = now + 500L*1000L*1000L;
156 0 : }
157 :
158 : void
159 0 : fd_sshttp_cancel( fd_sshttp_t * http ) {
160 0 : if( FD_LIKELY( http->state!=FD_SSHTTP_STATE_INIT && -1!=http->sockfd ) ) {
161 0 : if( FD_UNLIKELY( -1==close( http->sockfd ) ) ) FD_LOG_ERR(( "close() failed (%d-%s)", errno, fd_io_strerror( errno ) ));
162 0 : http->sockfd = -1;
163 0 : }
164 0 : http->state = FD_SSHTTP_STATE_INIT;
165 0 : }
166 :
167 : static int
168 : send_request( fd_sshttp_t * http,
169 0 : long now ) {
170 0 : if( FD_UNLIKELY( now>http->deadline ) ) {
171 0 : fd_sshttp_cancel( http );
172 0 : return FD_SSHTTP_ADVANCE_ERROR;
173 0 : }
174 :
175 0 : long sent = sendto( http->sockfd, http->request+http->request_sent, http->request_len-http->request_sent, 0, NULL, 0 );
176 0 : if( FD_UNLIKELY( -1==sent && errno==EAGAIN ) ) return FD_SSHTTP_ADVANCE_AGAIN;
177 0 : else if( FD_UNLIKELY( -1==sent ) ) {
178 0 : fd_sshttp_cancel( http );
179 0 : return FD_SSHTTP_ADVANCE_ERROR;
180 0 : }
181 :
182 0 : http->request_sent += (ulong)sent;
183 0 : if( FD_UNLIKELY( http->request_sent==http->request_len ) ) {
184 0 : http->state = FD_SSHTTP_STATE_RESP;
185 0 : http->response_len = 0UL;
186 0 : http->deadline = now + 500L*1000L*1000L;
187 0 : }
188 :
189 0 : return FD_SSHTTP_ADVANCE_AGAIN;
190 0 : }
191 :
192 : static int
193 : follow_redirect( fd_sshttp_t * http,
194 : struct phr_header * headers,
195 : ulong header_cnt,
196 0 : long now ) {
197 0 : if( FD_UNLIKELY( !http->hops ) ) {
198 0 : FD_LOG_WARNING(( "too many redirects" ));
199 0 : fd_sshttp_cancel( http );
200 0 : return FD_SSHTTP_ADVANCE_ERROR;
201 0 : }
202 :
203 0 : http->hops--;
204 :
205 0 : ulong location_len;
206 0 : char const * location = NULL;
207 :
208 0 : for( ulong i=0UL; i<header_cnt; i++ ) {
209 0 : if( FD_UNLIKELY( !strncasecmp( headers[ i ].name, "location", headers[ i ].name_len ) ) ) {
210 0 : if( FD_UNLIKELY( !headers [ i ].value_len || headers[ i ].value[ 0 ]!='/' ) ) {
211 0 : FD_LOG_WARNING(( "invalid location header `%.*s`", (int)headers[ i ].value_len, headers[ i ].value ));
212 0 : fd_sshttp_cancel( http );
213 0 : return FD_SSHTTP_ADVANCE_ERROR;
214 0 : }
215 :
216 0 : location_len = headers[ i ].value_len;
217 0 : location = headers[ i ].value;
218 :
219 0 : if( FD_UNLIKELY( location_len>=PATH_MAX-1UL ) ) {
220 0 : fd_sshttp_cancel( http );
221 0 : return FD_SSHTTP_ADVANCE_ERROR;
222 0 : }
223 :
224 0 : char snapshot_name[ PATH_MAX ];
225 0 : fd_memcpy( snapshot_name, location+1UL, location_len-1UL );
226 0 : snapshot_name[ location_len-1UL ] = '\0';
227 :
228 0 : ulong full_entry_slot, incremental_entry_slot;
229 0 : uchar decoded_hash[ FD_HASH_FOOTPRINT ];
230 0 : int err = fd_ssarchive_parse_filename( snapshot_name, &full_entry_slot, &incremental_entry_slot, decoded_hash );
231 :
232 0 : if( FD_UNLIKELY( err ) ) {
233 0 : FD_LOG_WARNING(( "unrecognized snapshot file `%s` in redirect location header", snapshot_name ));
234 0 : fd_sshttp_cancel( http );
235 0 : return FD_SSHTTP_ADVANCE_ERROR;
236 0 : }
237 :
238 0 : char encoded_hash[ FD_BASE58_ENCODED_32_SZ ];
239 0 : fd_base58_encode_32( decoded_hash, NULL, encoded_hash );
240 :
241 0 : if( FD_LIKELY( incremental_entry_slot!=ULONG_MAX ) ) {
242 0 : FD_TEST( fd_cstr_printf_check( http->incremental_snapshot_name, PATH_MAX, NULL, "incremental-snapshot-%lu-%lu-%s.tar.zst", full_entry_slot, incremental_entry_slot, encoded_hash ) );
243 0 : } else {
244 0 : FD_TEST( fd_cstr_printf_check( http->full_snapshot_name, PATH_MAX, NULL, "snapshot-%lu-%s.tar.zst", full_entry_slot, encoded_hash ) );
245 0 : }
246 0 : break;
247 0 : }
248 0 : }
249 :
250 0 : if( FD_UNLIKELY( !location ) ) {
251 0 : FD_LOG_WARNING(( "no location header in redirect response" ));
252 0 : fd_sshttp_cancel( http );
253 0 : return FD_SSHTTP_ADVANCE_ERROR;
254 0 : }
255 :
256 0 : if( FD_UNLIKELY( !fd_cstr_printf_check( http->request, sizeof(http->request), &http->request_len,
257 0 : "GET %.*s HTTP/1.1\r\n"
258 0 : "User-Agent: Firedancer\r\n"
259 0 : "Accept: */*\r\n"
260 0 : "Accept-Encoding: identity\r\n"
261 0 : "Host: " FD_IP4_ADDR_FMT "\r\n\r\n",
262 0 : (int)location_len, location, FD_IP4_ADDR_FMT_ARGS( http->addr.addr ) ) ) ) {
263 0 : FD_LOG_WARNING(( "location header too long `%.*s`", (int)location_len, location ));
264 0 : fd_sshttp_cancel( http );
265 0 : return FD_SSHTTP_ADVANCE_ERROR;
266 0 : }
267 :
268 0 : FD_LOG_NOTICE(( "following redirect to http://" FD_IP4_ADDR_FMT ":%hu%.*s",
269 0 : FD_IP4_ADDR_FMT_ARGS( http->addr.addr ), http->addr.port,
270 0 : (int)headers[ 0 ].value_len, headers[ 0 ].value ));
271 :
272 0 : fd_sshttp_cancel( http );
273 0 : fd_sshttp_init( http, http->addr, location, location_len, now );
274 :
275 0 : return FD_SSHTTP_ADVANCE_AGAIN;
276 0 : }
277 :
278 : static int
279 : read_response( fd_sshttp_t * http,
280 : ulong * data_len,
281 : uchar * data,
282 0 : long now ) {
283 0 : if( FD_UNLIKELY( now>http->deadline ) ) {
284 0 : FD_LOG_WARNING(( "timeout reading response" ));
285 0 : fd_sshttp_cancel( http );
286 0 : return FD_SSHTTP_ADVANCE_ERROR;
287 0 : }
288 :
289 0 : long read = recvfrom( http->sockfd, http->response+http->response_len, sizeof(http->response)-http->response_len, 0, NULL, NULL );
290 0 : if( FD_UNLIKELY( -1==read && errno==EAGAIN ) ) return 0;
291 0 : else if( FD_UNLIKELY( -1==read ) ) {
292 0 : FD_LOG_WARNING(( "recv() failed (%d-%s)", errno, fd_io_strerror( errno ) ));
293 0 : fd_sshttp_cancel( http );
294 0 : return FD_SSHTTP_ADVANCE_ERROR;
295 0 : }
296 :
297 0 : http->response_len += (ulong)read;
298 :
299 0 : int minor_version;
300 0 : int status;
301 0 : const char * message;
302 0 : ulong message_len;
303 0 : struct phr_header headers[ 128UL ];
304 0 : ulong header_cnt = 128UL;
305 0 : int parsed = phr_parse_response( http->response,
306 0 : http->response_len,
307 0 : &minor_version,
308 0 : &status,
309 0 : &message,
310 0 : &message_len,
311 0 : headers,
312 0 : &header_cnt,
313 0 : http->response_len - (ulong)read );
314 0 : if( FD_UNLIKELY( parsed==-1 ) ) {
315 0 : FD_LOG_WARNING(( "malformed response body" ));
316 0 : fd_sshttp_cancel( http );
317 0 : return FD_SSHTTP_ADVANCE_ERROR;
318 0 : } else if( parsed==-2 ) {
319 0 : return FD_SSHTTP_ADVANCE_AGAIN;
320 0 : }
321 :
322 0 : int is_redirect = (status==301) | (status==302) | (status==303) | (status==304) | (status==307) | (status==308);
323 0 : if( FD_UNLIKELY( is_redirect ) ) {
324 0 : return follow_redirect( http, headers, header_cnt, now );
325 0 : }
326 :
327 0 : if( FD_UNLIKELY( status!=200 ) ) {
328 0 : FD_LOG_WARNING(( "unexpected response status %d", status ));
329 0 : fd_sshttp_cancel( http );
330 0 : return FD_SSHTTP_ADVANCE_ERROR;
331 0 : }
332 :
333 0 : http->content_read = 0UL;
334 0 : http->content_len = ULONG_MAX;
335 0 : for( ulong i=0UL; i<header_cnt; i++ ) {
336 0 : if( FD_LIKELY( headers[i].name_len!=14UL ) ) continue;
337 0 : if( FD_LIKELY( strncasecmp( headers[i].name, "content-length", 14UL ) ) ) continue;
338 :
339 0 : http->content_len = strtoul( headers[i].value, NULL, 10 );
340 0 : break;
341 0 : }
342 :
343 0 : if( FD_UNLIKELY( http->content_len==ULONG_MAX ) ) {
344 0 : FD_LOG_WARNING(( "no content-length header in response" ));
345 0 : fd_sshttp_cancel( http );
346 0 : return FD_SSHTTP_ADVANCE_ERROR;
347 0 : }
348 :
349 0 : http->state = FD_SSHTTP_STATE_DL;
350 0 : if( FD_UNLIKELY( (ulong)parsed<http->response_len ) ) {
351 0 : if( FD_UNLIKELY( *data_len<http->response_len-(ulong)parsed ) ) FD_LOG_ERR(( "data buffer too small %lu %lu %lu", *data_len, http->response_len, (ulong)parsed ));
352 0 : FD_TEST( *data_len>=http->response_len-(ulong)parsed );
353 0 : *data_len = http->response_len - (ulong)parsed;
354 0 : fd_memcpy( data, http->response+parsed, *data_len );
355 0 : http->content_read += *data_len;
356 0 : return FD_SSHTTP_ADVANCE_DATA;
357 0 : } else {
358 0 : FD_TEST( http->response_len==(ulong)parsed );
359 0 : return FD_SSHTTP_ADVANCE_AGAIN;
360 0 : }
361 0 : }
362 :
363 : static int
364 : read_body( fd_sshttp_t * http,
365 : ulong * data_len,
366 0 : uchar * data ) {
367 0 : if( FD_UNLIKELY( http->content_read>=http->content_len ) ) {
368 0 : fd_sshttp_cancel( http );
369 0 : http->state = FD_SSHTTP_STATE_INIT;
370 0 : return FD_SSHTTP_ADVANCE_DONE;
371 0 : }
372 :
373 0 : FD_TEST( http->content_read<http->content_len );
374 0 : long read = recvfrom( http->sockfd, data, fd_ulong_min( *data_len, http->content_len-http->content_read ), 0, NULL, NULL );
375 0 : if( FD_UNLIKELY( -1==read && errno==EAGAIN ) ) return FD_SSHTTP_ADVANCE_AGAIN;
376 0 : else if( FD_UNLIKELY( -1==read ) ) {
377 0 : fd_sshttp_cancel( http );
378 0 : return FD_SSHTTP_ADVANCE_ERROR;
379 0 : }
380 :
381 0 : if( FD_UNLIKELY( !read ) ) return FD_SSHTTP_ADVANCE_AGAIN;
382 :
383 0 : *data_len = (ulong)read;
384 0 : http->content_read += (ulong)read;
385 :
386 0 : return FD_SSHTTP_ADVANCE_DATA;
387 0 : }
388 :
389 : void
390 : fd_sshttp_snapshot_names( fd_sshttp_t const * http,
391 : char const ** full_snapshot_name,
392 0 : char const ** incremental_snapshot_name ) {
393 0 : *full_snapshot_name = http->full_snapshot_name;
394 0 : *incremental_snapshot_name = http->incremental_snapshot_name;
395 0 : }
396 :
397 : ulong
398 0 : fd_sshttp_content_len( fd_sshttp_t const * http ) {
399 0 : return http->content_len;
400 0 : }
401 :
402 : int
403 : fd_sshttp_advance( fd_sshttp_t * http,
404 : ulong * data_len,
405 : uchar * data,
406 0 : long now ) {
407 : /* TODO: Add timeouts ... */
408 :
409 0 : switch( http->state ) {
410 0 : case FD_SSHTTP_STATE_INIT: return FD_SSHTTP_ADVANCE_AGAIN;
411 0 : case FD_SSHTTP_STATE_REQ: return send_request( http, now );
412 0 : case FD_SSHTTP_STATE_RESP: return read_response( http, data_len, data, now );
413 0 : case FD_SSHTTP_STATE_DL: return read_body( http, data_len, data );
414 0 : default: return 0;
415 0 : }
416 0 : }
|
