Line data    Source code

       1             : #define _GNU_SOURCE
       2             : #include "fd_sshttp.h"
       3             : #include "fd_ssarchive.h"
       4             : 
       5             : #include "../../../waltz/http/picohttpparser.h"
       6             : #include "../../../util/log/fd_log.h"
       7             : #include "../../../flamenco/types/fd_types_custom.h"
       8             : 
       9             : #include <unistd.h>
      10             : #include <errno.h>
      11             : #include <stdlib.h>
      12             : 
      13             : #include <sys/socket.h>
      14             : #include <netinet/tcp.h>
      15             : #include <netinet/in.h>
      16             : 
      17           0 : #define FD_SSHTTP_STATE_INIT  (0) /* start */
      18           0 : #define FD_SSHTTP_STATE_REQ   (1) /* sending request */
      19           0 : #define FD_SSHTTP_STATE_RESP  (2) /* receiving response headers */
      20           0 : #define FD_SSHTTP_STATE_DL    (3) /* downloading response body */
      21             : 
      22             : struct fd_sshttp_private {
      23             :   int  state;
      24             :   long deadline;
      25             :   int  full;
      26             : 
      27             :   int   hops;
      28             : 
      29             :   fd_ip4_port_t addr;
      30             :   int           sockfd;
      31             : 
      32             :   char  request[ 4096UL ];
      33             :   ulong request_len;
      34             :   ulong request_sent;
      35             : 
      36             :   ulong response_len;
      37             :   char  response[ USHORT_MAX ];
      38             : 
      39             :   char  full_snapshot_name[ PATH_MAX ];
      40             :   char  incremental_snapshot_name[ PATH_MAX ];
      41             : 
      42             :   ulong content_len;
      43             :   ulong content_read;
      44             : 
      45             :   ulong magic;
      46             : };
      47             : 
      48             : FD_FN_CONST ulong
      49           0 : fd_sshttp_align( void ) {
      50           0 :   return FD_SSHTTP_ALIGN;
      51           0 : }
      52             : 
      53             : FD_FN_CONST ulong
      54           0 : fd_sshttp_footprint( void ) {
      55           0 :   ulong l;
      56           0 :   l = FD_LAYOUT_INIT;
      57           0 :   l = FD_LAYOUT_APPEND( l, FD_SSHTTP_ALIGN, sizeof(fd_sshttp_t) );
      58           0 :   return FD_LAYOUT_FINI( l, FD_SSHTTP_ALIGN );
      59           0 : }
      60             : 
      61             : void *
      62           0 : fd_sshttp_new( void * shmem ) {
      63           0 :   if( FD_UNLIKELY( !shmem ) ) {
      64           0 :     FD_LOG_WARNING(( "NULL shmem" ));
      65           0 :     return NULL;
      66           0 :   }
      67             : 
      68           0 :   if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)shmem, fd_sshttp_align() ) ) ) {
      69           0 :     FD_LOG_WARNING(( "unaligned shmem" ));
      70           0 :     return NULL;
      71           0 :   }
      72             : 
      73           0 :   FD_SCRATCH_ALLOC_INIT( l, shmem );
      74           0 :   fd_sshttp_t * sshttp = FD_SCRATCH_ALLOC_APPEND( l, FD_SSHTTP_ALIGN, sizeof(fd_sshttp_t) );
      75             : 
      76           0 :   sshttp->state = FD_SSHTTP_STATE_INIT;
      77           0 :   sshttp->full_snapshot_name[ 0 ] = '\0';
      78           0 :   sshttp->incremental_snapshot_name[ 0 ] = '\0';
      79             : 
      80           0 :   FD_COMPILER_MFENCE();
      81           0 :   FD_VOLATILE( sshttp->magic ) = FD_SSHTTP_MAGIC;
      82           0 :   FD_COMPILER_MFENCE();
      83             : 
      84           0 :   return (void *)sshttp;
      85           0 : }
      86             : 
      87             : fd_sshttp_t *
      88           0 : fd_sshttp_join( void * shhttp ) {
      89           0 :   if( FD_UNLIKELY( !shhttp ) ) {
      90           0 :     FD_LOG_WARNING(( "NULL shhttp" ));
      91           0 :     return NULL;
      92           0 :   }
      93             : 
      94           0 :   if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)shhttp, fd_sshttp_align() ) ) ) {
      95           0 :     FD_LOG_WARNING(( "misaligned shhttp" ));
      96           0 :     return NULL;
      97           0 :   }
      98             : 
      99           0 :   fd_sshttp_t * sshttp = (fd_sshttp_t *)shhttp;
     100             : 
     101           0 :   if( FD_UNLIKELY( sshttp->magic!=FD_SSHTTP_MAGIC ) ) {
     102           0 :     FD_LOG_WARNING(( "bad magic" ));
     103           0 :     return NULL;
     104           0 :   }
     105             : 
     106           0 :   return sshttp;
     107           0 : }
     108             : 
     109             : void
     110             : fd_sshttp_init( fd_sshttp_t * http,
     111             :                 fd_ip4_port_t addr,
     112             :                 char const *  path,
     113             :                 ulong         path_len,
     114           0 :                 long          now ) {
     115           0 :   FD_TEST( http->state==FD_SSHTTP_STATE_INIT );
     116             : 
     117           0 :   http->hops = 4UL;
     118             : 
     119           0 :   http->request_sent = 0UL;
     120           0 :   FD_TEST( fd_cstr_printf_check( http->request, sizeof(http->request), &http->request_len,
     121           0 :     "GET %.*s HTTP/1.1\r\n"
     122           0 :     "User-Agent: Firedancer\r\n"
     123           0 :     "Accept: */*\r\n"
     124           0 :     "Accept-Encoding: identity\r\n"
     125           0 :     "Host: " FD_IP4_ADDR_FMT "\r\n\r\n",
     126           0 :     (int)path_len, path, FD_IP4_ADDR_FMT_ARGS( addr.addr ) ) );
     127             : 
     128           0 :   http->addr = addr;
     129           0 :   http->sockfd = socket( AF_INET, SOCK_STREAM|SOCK_NONBLOCK, 0 );
     130           0 :   if( FD_UNLIKELY( -1==http->sockfd ) ) FD_LOG_ERR(( "socket() failed (%d-%s)", errno, fd_io_strerror( errno ) ));
     131             : 
     132           0 :   int optval = 1;
     133           0 :   if( FD_UNLIKELY( -1==setsockopt( http->sockfd, SOL_TCP, TCP_NODELAY, &optval, sizeof(int) ) ) ) {
     134           0 :     FD_LOG_ERR(( "setsockopt() failed (%d-%s)", errno, fd_io_strerror( errno ) ));
     135           0 :   }
     136             : 
     137           0 :   struct sockaddr_in addr_in = {
     138           0 :     .sin_family = AF_INET,
     139           0 :     .sin_port   = fd_ushort_bswap( addr.port ),
     140           0 :     .sin_addr   = { .s_addr = addr.addr }
     141           0 :   };
     142             : 
     143           0 :   if( FD_UNLIKELY( -1==connect( http->sockfd, fd_type_pun_const( &addr_in ), sizeof(addr_in) ) ) ) {
     144           0 :     if( FD_UNLIKELY( errno!=EINPROGRESS ) ) {
     145           0 :       if( FD_UNLIKELY( -1==close( http->sockfd ) ) ) FD_LOG_ERR(( "close() failed (%d-%s)", errno, fd_io_strerror( errno ) ));
     146           0 :       return;
     147           0 :     }
     148           0 :   }
     149             : 
     150           0 :   http->state    = FD_SSHTTP_STATE_REQ;
     151           0 :   http->deadline = now + 500L*1000L*1000L;
     152           0 : }
     153             : 
     154             : void
     155           0 : fd_sshttp_cancel( fd_sshttp_t * http ) {
     156           0 :   if( FD_LIKELY( http->state!=FD_SSHTTP_STATE_INIT && -1!=http->sockfd ) ) {
     157           0 :     if( FD_UNLIKELY( -1==close( http->sockfd ) ) ) FD_LOG_ERR(( "close() failed (%d-%s)", errno, fd_io_strerror( errno ) ));
     158           0 :     http->sockfd = -1;
     159           0 :   }
     160           0 :   http->state = FD_SSHTTP_STATE_INIT;
     161           0 : }
     162             : 
     163             : static int
     164             : send_request( fd_sshttp_t * http,
     165           0 :               long          now ) {
     166           0 :   if( FD_UNLIKELY( now>http->deadline ) ) {
     167           0 :     fd_sshttp_cancel( http );
     168           0 :     return FD_SSHTTP_ADVANCE_ERROR;
     169           0 :   }
     170             : 
     171           0 :   long sent = sendto( http->sockfd, http->request+http->request_sent, http->request_len-http->request_sent, 0, NULL, 0 );
     172           0 :   if( FD_UNLIKELY( -1==sent && errno==EAGAIN ) ) return FD_SSHTTP_ADVANCE_AGAIN;
     173           0 :   else if( FD_UNLIKELY( -1==sent ) ) {
     174           0 :     fd_sshttp_cancel( http );
     175           0 :     return FD_SSHTTP_ADVANCE_ERROR;
     176           0 :   }
     177             : 
     178           0 :   http->request_sent += (ulong)sent;
     179           0 :   if( FD_UNLIKELY( http->request_sent==http->request_len ) ) {
     180           0 :     http->state = FD_SSHTTP_STATE_RESP;
     181           0 :     http->response_len = 0UL;
     182           0 :     http->deadline = now + 500L*1000L*1000L;
     183           0 :   }
     184             : 
     185           0 :   return FD_SSHTTP_ADVANCE_AGAIN;
     186           0 : }
     187             : 
     188             : static int
     189             : follow_redirect( fd_sshttp_t *        http,
     190             :                   struct phr_header * headers,
     191             :                   ulong               header_cnt,
     192           0 :                   long                now ) {
     193           0 :   if( FD_UNLIKELY( !http->hops ) ) {
     194           0 :     FD_LOG_WARNING(( "too many redirects" ));
     195           0 :     fd_sshttp_cancel( http );
     196           0 :     return FD_SSHTTP_ADVANCE_ERROR;
     197           0 :   }
     198             : 
     199           0 :   http->hops--;
     200             : 
     201           0 :   ulong        location_len;
     202           0 :   char const * location = NULL;
     203             : 
     204           0 :   for( ulong i=0UL; i<header_cnt; i++ ) {
     205           0 :     if( FD_UNLIKELY( !strncasecmp( headers[ i ].name, "location", headers[ i ].name_len ) ) ) {
     206           0 :       if( FD_UNLIKELY( !headers [ i ].value_len || headers[ i ].value[ 0 ]!='/' ) ) {
     207           0 :         FD_LOG_WARNING(( "invalid location header `%.*s`", (int)headers[ i ].value_len, headers[ i ].value ));
     208           0 :         fd_sshttp_cancel( http );
     209           0 :         return FD_SSHTTP_ADVANCE_ERROR;
     210           0 :       }
     211             : 
     212           0 :       location_len = headers[ i ].value_len;
     213           0 :       location     = headers[ i ].value;
     214             : 
     215           0 :       if( FD_UNLIKELY( location_len>=PATH_MAX-1UL ) ) {
     216           0 :         fd_sshttp_cancel( http );
     217           0 :         return FD_SSHTTP_ADVANCE_ERROR;
     218           0 :       }
     219             : 
     220           0 :       char snapshot_name[ PATH_MAX ];
     221           0 :       fd_memcpy( snapshot_name, location+1UL, location_len-1UL );
     222           0 :       snapshot_name[ location_len-1UL ] = '\0';
     223             : 
     224           0 :       ulong full_entry_slot, incremental_entry_slot;
     225           0 :       uchar decoded_hash[ FD_HASH_FOOTPRINT ];
     226           0 :       int err = fd_ssarchive_parse_filename( snapshot_name, &full_entry_slot, &incremental_entry_slot, decoded_hash );
     227             : 
     228           0 :       if( FD_UNLIKELY( err ) ) {
     229           0 :         FD_LOG_WARNING(( "unrecognized snapshot file `%s` in redirect location header", snapshot_name ));
     230           0 :         fd_sshttp_cancel( http );
     231           0 :         return FD_SSHTTP_ADVANCE_ERROR;
     232           0 :       }
     233             : 
     234           0 :       char encoded_hash[ FD_BASE58_ENCODED_32_SZ ];
     235           0 :       fd_base58_encode_32( decoded_hash, NULL, encoded_hash );
     236             : 
     237           0 :       if( FD_LIKELY( incremental_entry_slot!=ULONG_MAX ) ) {
     238           0 :         FD_TEST( fd_cstr_printf_check( http->incremental_snapshot_name, PATH_MAX, NULL, "incremental-snapshot-%lu-%lu-%s.tar.zst", full_entry_slot, incremental_entry_slot, encoded_hash ) );
     239           0 :       } else {
     240           0 :         FD_TEST( fd_cstr_printf_check( http->full_snapshot_name, PATH_MAX, NULL, "snapshot-%lu-%s.tar.zst", full_entry_slot, encoded_hash ) );
     241           0 :       }
     242           0 :       break;
     243           0 :     }
     244           0 :   }
     245             : 
     246           0 :   if( FD_UNLIKELY( !location ) ) {
     247           0 :     FD_LOG_WARNING(( "no location header in redirect response" ));
     248           0 :     fd_sshttp_cancel( http );
     249           0 :     return FD_SSHTTP_ADVANCE_ERROR;
     250           0 :   }
     251             : 
     252           0 :   if( FD_UNLIKELY( !fd_cstr_printf_check( http->request, sizeof(http->request), &http->request_len,
     253           0 :     "GET %.*s HTTP/1.1\r\n"
     254           0 :     "User-Agent: Firedancer\r\n"
     255           0 :     "Accept: */*\r\n"
     256           0 :     "Accept-Encoding: identity\r\n"
     257           0 :     "Host: " FD_IP4_ADDR_FMT "\r\n\r\n",
     258           0 :     (int)location_len, location, FD_IP4_ADDR_FMT_ARGS( http->addr.addr ) ) ) ) {
     259           0 :     FD_LOG_WARNING(( "location header too long `%.*s`", (int)location_len, location ));
     260           0 :     fd_sshttp_cancel( http );
     261           0 :     return FD_SSHTTP_ADVANCE_ERROR;
     262           0 :   }
     263             : 
     264           0 :   FD_LOG_NOTICE(( "following redirect to http://" FD_IP4_ADDR_FMT ":%hu%.*s",
     265           0 :                   FD_IP4_ADDR_FMT_ARGS( http->addr.addr ), http->addr.port,
     266           0 :                   (int)headers[ 0 ].value_len, headers[ 0 ].value ));
     267             : 
     268           0 :   fd_sshttp_cancel( http );
     269           0 :   fd_sshttp_init( http, http->addr, location, location_len, now );
     270             : 
     271           0 :   return FD_SSHTTP_ADVANCE_AGAIN;
     272           0 : }
     273             : 
     274             : static int
     275             : read_response( fd_sshttp_t * http,
     276             :                ulong *       data_len,
     277             :                uchar *       data,
     278           0 :                long          now ) {
     279           0 :   if( FD_UNLIKELY( now>http->deadline ) ) {
     280           0 :     FD_LOG_WARNING(( "timeout reading response" ));
     281           0 :     fd_sshttp_cancel( http );
     282           0 :     return FD_SSHTTP_ADVANCE_ERROR;
     283           0 :   }
     284             : 
     285           0 :   long read = recvfrom( http->sockfd, http->response+http->response_len, sizeof(http->response)-http->response_len, 0, NULL, NULL );
     286           0 :   if( FD_UNLIKELY( -1==read && errno==EAGAIN ) ) return 0;
     287           0 :   else if( FD_UNLIKELY( -1==read ) ) {
     288           0 :     FD_LOG_WARNING(( "recv() failed (%d-%s)", errno, fd_io_strerror( errno ) ));
     289           0 :     fd_sshttp_cancel( http );
     290           0 :     return FD_SSHTTP_ADVANCE_ERROR;
     291           0 :   }
     292             : 
     293           0 :   http->response_len += (ulong)read;
     294             : 
     295           0 :   int               minor_version;
     296           0 :   int               status;
     297           0 :   const char *      message;
     298           0 :   ulong             message_len;
     299           0 :   struct phr_header headers[ 128UL ];
     300           0 :   ulong             header_cnt = 128UL;
     301           0 :   int parsed = phr_parse_response( http->response,
     302           0 :                                     http->response_len,
     303           0 :                                     &minor_version,
     304           0 :                                     &status,
     305           0 :                                     &message,
     306           0 :                                     &message_len,
     307           0 :                                     headers,
     308           0 :                                     &header_cnt,
     309           0 :                                     http->response_len - (ulong)read );
     310           0 :   if( FD_UNLIKELY( parsed==-1 ) ) {
     311           0 :     FD_LOG_WARNING(( "malformed response body" ));
     312           0 :     fd_sshttp_cancel( http );
     313           0 :     return FD_SSHTTP_ADVANCE_ERROR;
     314           0 :   } else if( parsed==-2 ) {
     315           0 :     return FD_SSHTTP_ADVANCE_AGAIN;
     316           0 :   }
     317             : 
     318           0 :   int is_redirect = (status==301) | (status==302) | (status==303) | (status==304) | (status==307) | (status==308);
     319           0 :   if( FD_UNLIKELY( is_redirect ) ) {
     320           0 :     return follow_redirect( http, headers, header_cnt, now );
     321           0 :   }
     322             : 
     323           0 :   if( FD_UNLIKELY( status!=200 ) ) {
     324           0 :     FD_LOG_WARNING(( "unexpected response status %d", status ));
     325           0 :     fd_sshttp_cancel( http );
     326           0 :     return FD_SSHTTP_ADVANCE_ERROR;
     327           0 :   }
     328             : 
     329           0 :   http->content_read = 0UL;
     330           0 :   http->content_len = ULONG_MAX;
     331           0 :   for( ulong i=0UL; i<header_cnt; i++ ) {
     332           0 :     if( FD_LIKELY( headers[i].name_len!=14UL ) ) continue;
     333           0 :     if( FD_LIKELY( strncasecmp( headers[i].name, "content-length", 14UL ) ) ) continue;
     334             : 
     335           0 :     http->content_len = strtoul( headers[i].value, NULL, 10 );
     336           0 :     break;
     337           0 :   }
     338             : 
     339           0 :   if( FD_UNLIKELY( http->content_len==ULONG_MAX ) ) {
     340           0 :     FD_LOG_WARNING(( "no content-length header in response" ));
     341           0 :     fd_sshttp_cancel( http );
     342           0 :     return FD_SSHTTP_ADVANCE_ERROR;
     343           0 :   }
     344             : 
     345           0 :   http->state = FD_SSHTTP_STATE_DL;
     346           0 :   if( FD_UNLIKELY( (ulong)parsed<http->response_len ) ) {
     347           0 :     if( FD_UNLIKELY( *data_len<http->response_len-(ulong)parsed ) ) FD_LOG_ERR(( "data buffer too small %lu %lu %lu", *data_len, http->response_len, (ulong)parsed ));
     348           0 :     FD_TEST( *data_len>=http->response_len-(ulong)parsed );
     349           0 :     *data_len = http->response_len - (ulong)parsed;
     350           0 :     fd_memcpy( data, http->response+parsed, *data_len );
     351           0 :     http->content_read += *data_len;
     352           0 :     return FD_SSHTTP_ADVANCE_DATA;
     353           0 :   } else {
     354           0 :     FD_TEST( http->response_len==(ulong)parsed );
     355           0 :     return FD_SSHTTP_ADVANCE_AGAIN;
     356           0 :   }
     357           0 : }
     358             : 
     359             : static int
     360             : read_body( fd_sshttp_t * http,
     361             :            ulong *       data_len,
     362           0 :            uchar *       data ) {
     363           0 :   if( FD_UNLIKELY( http->content_read>=http->content_len ) ) {
     364           0 :     fd_sshttp_cancel( http );
     365           0 :     http->state = FD_SSHTTP_STATE_INIT;
     366           0 :     return FD_SSHTTP_ADVANCE_DONE;
     367           0 :   }
     368             : 
     369           0 :   FD_TEST( http->content_read<http->content_len );
     370           0 :   long read = recvfrom( http->sockfd, data, fd_ulong_min( *data_len, http->content_len-http->content_read ), 0, NULL, NULL );
     371           0 :   if( FD_UNLIKELY( -1==read && errno==EAGAIN ) ) return FD_SSHTTP_ADVANCE_AGAIN;
     372           0 :   else if( FD_UNLIKELY( -1==read ) ) {
     373           0 :     fd_sshttp_cancel( http );
     374           0 :     return FD_SSHTTP_ADVANCE_ERROR;
     375           0 :   }
     376             : 
     377           0 :   if( FD_UNLIKELY( !read ) ) return FD_SSHTTP_ADVANCE_AGAIN;
     378             : 
     379           0 :   *data_len = (ulong)read;
     380           0 :   http->content_read += (ulong)read;
     381             : 
     382           0 :   return FD_SSHTTP_ADVANCE_DATA;
     383           0 : }
     384             : 
     385             : void
     386             : fd_sshttp_snapshot_names( fd_sshttp_t const * http,
     387             :                           char const **       full_snapshot_name,
     388           0 :                           char const **       incremental_snapshot_name ) {
     389           0 :   *full_snapshot_name        = http->full_snapshot_name;
     390           0 :   *incremental_snapshot_name = http->incremental_snapshot_name;
     391           0 : }
     392             : 
     393             : ulong
     394           0 : fd_sshttp_content_len( fd_sshttp_t const * http ) {
     395           0 :   return http->content_len;
     396           0 : }
     397             : 
     398             : int
     399             : fd_sshttp_advance( fd_sshttp_t * http,
     400             :                    ulong *       data_len,
     401             :                    uchar *       data,
     402           0 :                    long          now ) {
     403             :   /* TODO: Add timeouts ... */
     404             : 
     405           0 :   switch( http->state ) {
     406           0 :     case FD_SSHTTP_STATE_INIT: return FD_SSHTTP_ADVANCE_AGAIN;
     407           0 :     case FD_SSHTTP_STATE_REQ: return send_request( http, now );
     408           0 :     case FD_SSHTTP_STATE_RESP: return read_response( http, data_len, data, now );
     409           0 :     case FD_SSHTTP_STATE_DL: return read_body( http, data_len, data );
     410           0 :     default: return 0;
     411           0 :   }
     412           0 : }