Line data    Source code

       1             : #include "../fd_flamenco.h"
       2             : #include "fd_solcap.pb.h"
       3             : #include "fd_solcap_proto.h"
       4             : #include "fd_solcap_writer.h"
       5             : #include "../../ballet/base58/fd_base58.h"
       6             : #include "../../ballet/base64/fd_base64.h"
       7             : #include "../../ballet/json/cJSON.h"
       8             : 
       9             : #include <errno.h>
      10             : #include <fcntl.h>
      11             : #include <stdio.h>
      12             : #include <unistd.h>
      13             : #include <sys/stat.h>
      14             : #include <math.h>
      15             : #include <dirent.h>
      16             : 
      17             : #ifndef DT_REG
      18           0 : #define DT_REG (8UL)
      19             : #endif
      20             : 
      21             : static int
      22           0 : usage( void ) {
      23           0 :   fprintf( stderr,
      24           0 :     "Usage: fd_solcap_import [options] {IN_DIR} {OUT_FILE}\n"
      25           0 :     "\n"
      26           0 :     "Imports a runtime capture directory from JSON.\n"
      27           0 :     "\n"
      28           0 :     "Options:\n"
      29           0 :     "  --page-sz      {gigantic|huge|normal}    Page size\n"
      30           0 :     "  --page-cnt     {count}                   Page count\n"
      31           0 :     "  --scratch-mb   1024                      Scratch mem MiB\n"
      32           0 :     "\n" );
      33           0 :   return 0;
      34           0 : }
      35             : 
      36             : /* fd_alloc wrapper */
      37             : 
      38             : static fd_alloc_t * current_alloc;
      39           0 : static void * my_malloc( ulong sz ) { return fd_alloc_malloc( current_alloc, 1UL, sz ); }
      40           0 : static void   my_free  ( void * p ) {        fd_alloc_free  ( current_alloc, p       ); }
      41             : 
      42             : static cJSON *
      43             : read_json_file( fd_wksp_t *  wksp,
      44             :                 fd_alloc_t * alloc,
      45           0 :                 char const * path ) {
      46             : 
      47             :   /* Wire up fd_alloc with cJSON */
      48             : 
      49           0 :   current_alloc = alloc;
      50           0 :   cJSON_Hooks hooks = {
      51           0 :     .malloc_fn = my_malloc,
      52           0 :     .free_fn   = my_free
      53           0 :   };
      54           0 :   cJSON_InitHooks( &hooks );
      55             : 
      56             :   /* Open file */
      57             : 
      58           0 :   int fd;
      59           0 :   fd = open( path, O_RDONLY );
      60           0 :   if( FD_UNLIKELY( fd<0 ) ) {
      61           0 :     FD_LOG_WARNING(( "open(%s) failed (%d-%s)", path, errno, strerror( errno ) ));
      62           0 :     return NULL;
      63           0 :   }
      64             : 
      65             :   /* Figure out file size */
      66             : 
      67           0 :   struct stat stat;
      68           0 :   if( FD_UNLIKELY( fstat( fd, &stat )<0 ) ) {
      69           0 :     FD_LOG_WARNING(( "fstat(%s) failed (%d-%s)", path, errno, strerror( errno ) ));
      70           0 :     close( fd );
      71           0 :     return NULL;
      72           0 :   }
      73             : 
      74             :   /* Allocate buffer to store file content */
      75             : 
      76           0 :   char * buf = fd_wksp_alloc_laddr( wksp, 1UL, (ulong)stat.st_size, 1UL );
      77           0 :   if( FD_UNLIKELY( !buf ) ) {
      78           0 :     FD_LOG_WARNING(( "Failed to alloc memory region to fit file content" ));
      79           0 :     close( fd );
      80           0 :     return NULL;
      81           0 :   }
      82             : 
      83             :   /* Copy file content to memory */
      84             : 
      85           0 :   char * cursor = buf;
      86           0 :   ulong rem = (ulong)stat.st_size;
      87           0 :   while( rem>0UL ) {
      88           0 :     long n = read( fd, cursor, rem );
      89           0 :     if( FD_UNLIKELY( n<0L ) ) {
      90           0 :       FD_LOG_WARNING(( "read(%s) failed (%d-%s)", path, errno, strerror( errno ) ));
      91           0 :       close( fd );
      92           0 :       return NULL;
      93           0 :     }
      94           0 :     if( FD_UNLIKELY( n==0L ) ) {
      95           0 :       FD_LOG_WARNING(( "read(%s) failed: unexpected EOF", path ));
      96           0 :       close( fd );
      97           0 :       return NULL;
      98           0 :     }
      99             : 
     100           0 :     cursor += (ulong)n;
     101           0 :     rem    -= (ulong)n;
     102           0 :   }
     103             : 
     104             :   /* Call parser */
     105             : 
     106           0 :   cJSON * json = cJSON_ParseWithLength( buf, (ulong)stat.st_size );
     107             : 
     108             :   /* Clean up */
     109             : 
     110           0 :   fd_wksp_free_laddr( buf );
     111           0 :   close( fd );
     112           0 :   return json;
     113           0 : }
     114             : 
     115             : /* unmarshal_hash interprets given JSON node as a string containing
     116             :    the Base58 encoding of 32 bytes.  Copies the bytes out to out_buf.
     117             :    Returns NULL if json is NULL, json is not string, or is not a valid
     118             :    32-byte Base58 encoding.  Returns out_buf on success. */
     119             : 
     120             : static uchar *
     121             : unmarshal_hash( cJSON const * json,
     122           0 :                 uchar         out_buf[static 32] ) {
     123             : 
     124           0 :   char const * str = cJSON_GetStringValue( json );
     125           0 :   if( FD_UNLIKELY( !str ) ) return NULL;
     126             : 
     127           0 :   return fd_base58_decode_32( str, out_buf );
     128           0 : }
     129             : 
     130             : /* unmarshal_bank_preimage reads top-level bank preimage information
     131             :    from given JSON dictionary.   Copies values into given out struct.
     132             :    Aborts application via error log on failure. */
     133             : 
     134             : static void
     135             : unmarshal_bank_preimage( cJSON const *            json,
     136           0 :                          fd_solcap_BankPreimage * out ) {
     137             : 
     138           0 :   cJSON * head = (cJSON *)json;
     139             : 
     140           0 :   cJSON * slot = cJSON_GetObjectItem( head, "slot" );
     141           0 :   out->slot = slot ? slot->valueulong : 0UL;
     142             : 
     143           0 :   FD_TEST( unmarshal_hash( cJSON_GetObjectItem( head, "bank_hash"           ), out->bank_hash          ) );
     144           0 :   FD_TEST( unmarshal_hash( cJSON_GetObjectItem( head, "parent_bank_hash"    ), out->prev_bank_hash     ) );
     145           0 :   FD_TEST( unmarshal_hash( cJSON_GetObjectItem( head, "accounts_delta_hash" ), out->account_delta_hash ) );
     146           0 :   FD_TEST( unmarshal_hash( cJSON_GetObjectItem( head, "last_blockhash"      ), out->poh_hash           ) );
     147             : 
     148           0 :   if ( cJSON_GetObjectItem( head, "signature_count" ) != NULL )
     149           0 :     out->signature_cnt = cJSON_GetObjectItem( head, "signature_count" )->valueulong;
     150           0 :   else
     151           0 :     out->signature_cnt = 0;
     152             : 
     153           0 :   cJSON * accs = cJSON_GetObjectItem( head, "accounts" );
     154           0 :   FD_TEST( accs );
     155           0 :   out->account_cnt = (ulong)cJSON_GetArraySize( accs );
     156           0 : }
     157             : 
     158             : /* unmarshal_account reads account meta/data from given JSON object.
     159             :    Object should be a dictionary and is found as an element of the
     160             :    "accounts" array.  On success, returns pointer to account data
     161             :    (allocated in current scratch frame), and copies metadata to given
     162             :    out structs.  On failure, aborts application via error log. */
     163             : 
     164             : static void *
     165             : unmarshal_account( cJSON const *             json,
     166             :                    fd_solcap_account_tbl_t * rec,
     167           0 :                    fd_solcap_AccountMeta *   meta ) {
     168             : 
     169             :   /* TODO !!! THIS IS OBVIOUSLY UNSAFE
     170             :      Representing lamports as double causes precision-loss for values
     171             :      exceeding 2^53-1.  This appears to be a limitation of the cJSON
     172             :      library. */
     173           0 :   meta->lamports = cJSON_GetObjectItem( json, "lamports" )->valueulong;
     174             : 
     175           0 :   meta->rent_epoch = cJSON_GetObjectItem( json, "rent_epoch" )->valueulong;
     176             : 
     177           0 :   cJSON * executable_o = cJSON_GetObjectItem( json, "executable" );
     178           0 :   FD_TEST( executable_o );
     179           0 :   meta->executable = cJSON_IsBool( executable_o ) & cJSON_IsTrue( executable_o );
     180             : 
     181           0 :   FD_TEST( unmarshal_hash( cJSON_GetObjectItem( json, "pubkey" ), rec->key    ) );
     182           0 :   FD_TEST( unmarshal_hash( cJSON_GetObjectItem( json, "hash"   ), rec->hash   ) );
     183           0 :   FD_TEST( unmarshal_hash( cJSON_GetObjectItem( json, "owner"  ), meta->owner ) );
     184             : 
     185             :   /* Data handling ... Base64 decode */
     186             : 
     187           0 :   char const * data_b64 = cJSON_GetStringValue( cJSON_GetObjectItem( json, "data" ) );
     188           0 :   FD_TEST( data_b64 );
     189             : 
     190             :   /* sigh ... cJSON doesn't remember string length, although it
     191             :      obviously had this information while parsing. */
     192           0 :   ulong data_b64_len = strlen( data_b64 );
     193             : 
     194             :   /* Very rough upper bound for decoded data sz.
     195             :      Could do better here, but better to be on the safe side. */
     196           0 :   ulong approx_data_sz = 3UL + data_b64_len/2UL;
     197             : 
     198             :   /* Grab scratch memory suitable for storing account data */
     199           0 :   void * data = fd_scratch_alloc( /* align */ 1UL, /* sz */ approx_data_sz );
     200           0 :   FD_TEST( data );
     201             : 
     202             :   /* Base64 decode */
     203           0 :   long data_sz = fd_base64_decode( data, data_b64, data_b64_len );
     204           0 :   FD_TEST( data_sz>=0L ); /* check for corruption */
     205           0 :   meta->data_sz = (ulong)data_sz;
     206             : 
     207           0 :   return data;
     208           0 : }
     209             : 
     210             : void write_slots( const char * in_path,
     211             :                   fd_solcap_writer_t * writer,
     212             :                   fd_wksp_t * wksp,
     213           0 :                   fd_alloc_t * alloc ) {
     214             :   /* Iterate through the directory to get all of the bank hash details file */
     215           0 :   struct dirent * ent;
     216           0 :   DIR * dir = opendir( in_path );
     217             : 
     218           0 :   if ( dir == NULL ) {
     219           0 :     FD_LOG_ERR(( "unable to open the directory=%s", in_path ));
     220           0 :   }
     221             :   /* TODO: sort the files that are read in. The API makes no guarantee that the
     222             :      files are alphabetically sorted, but in practice they are. */
     223           0 :   for ( ent = readdir( dir ); ent != NULL; ent = readdir( dir ) ) {
     224           0 :     if ( ent->d_type != DT_REG ) {
     225           0 :       continue;
     226           0 :     }
     227             : 
     228           0 :     char path_buf[ 256UL ];
     229           0 :     char * path_buf_ptr = path_buf;
     230           0 :     fd_memset( path_buf_ptr, '\0', sizeof( path_buf ) );
     231           0 :     fd_memcpy( path_buf_ptr, in_path, strlen( in_path ) );
     232           0 :     fd_memcpy( path_buf_ptr + strlen( in_path ), ent->d_name, strlen( ent->d_name ) );
     233           0 :     FD_LOG_NOTICE(( "Reading input file=%s", path_buf_ptr ));
     234             : 
     235           0 :     cJSON * json = read_json_file( wksp, alloc, path_buf_ptr );
     236           0 :     if( FD_UNLIKELY( !json ) ) {
     237           0 :       FD_LOG_ERR(( "Failed to read input file=%s", path_buf_ptr ));
     238           0 :     }
     239             : 
     240             :     // The structure of 1.18 is different to 1.17, and includes bank_hash_details
     241           0 :     cJSON * bank_hash_details = cJSON_GetObjectItem( json, "bank_hash_details" );
     242           0 :     if ( bank_hash_details != NULL ) {
     243           0 :       json = cJSON_GetArrayItem( bank_hash_details, 0 );
     244           0 :     }
     245             : 
     246           0 :     fd_solcap_BankPreimage preimg[1] = {{0}};
     247           0 :     unmarshal_bank_preimage( json, preimg );
     248             : 
     249           0 :     fd_solcap_writer_set_slot( writer, preimg->slot );
     250             : 
     251           0 :     cJSON * json_acc = cJSON_GetObjectItem( json, "accounts" );
     252           0 :     cJSON * acc = cJSON_GetArrayItem( json_acc, 0 );
     253           0 :     int n = cJSON_GetArraySize( json_acc );
     254             : 
     255           0 :     for( int i=0; i<n; i++ ) {
     256           0 :       fd_scratch_push();
     257             : 
     258           0 :       fd_solcap_account_tbl_t  rec[1]; memset( rec,  0, sizeof(fd_solcap_account_tbl_t) );
     259           0 :       fd_solcap_AccountMeta   meta[1]; memset( meta, 0, sizeof(fd_solcap_AccountMeta  ) );
     260           0 :       void * data = unmarshal_account( acc, rec, meta );
     261             : 
     262           0 :       FD_TEST( 0==fd_solcap_write_account2( writer, rec, meta, data, meta->data_sz ) );
     263             : 
     264           0 :       fd_scratch_pop();
     265             : 
     266           0 :       acc = acc->next;
     267           0 :     }
     268             : 
     269           0 :     FD_TEST( 0==fd_solcap_write_bank_preimage2( writer, preimg ) );
     270           0 :     cJSON_free( json );
     271           0 :   }
     272           0 :   closedir( dir );
     273           0 : }
     274             : 
     275             : int
     276             : main( int     argc,
     277             :       char ** argv ) {
     278             :   fd_boot( &argc, &argv );
     279             :   fd_flamenco_boot( &argc, &argv );
     280             : 
     281             :   /* Command line handling */
     282             : 
     283             :   for( int i=1; i<argc; i++ )
     284             :     if( 0==strcmp( argv[i], "--help" ) ) return usage();
     285             : 
     286             :   char const * _page_sz   = fd_env_strip_cmdline_cstr ( &argc, &argv, "--page-sz",    NULL, "gigantic" );
     287             :   ulong        page_cnt   = fd_env_strip_cmdline_ulong( &argc, &argv, "--page-cnt",   NULL, 2UL        );
     288             :   ulong        scratch_mb = fd_env_strip_cmdline_ulong( &argc, &argv, "--scratch-mb", NULL, 1024UL     );
     289             : 
     290             :   ulong page_sz = fd_cstr_to_shmem_page_sz( _page_sz );
     291             :   if( FD_UNLIKELY( !page_sz ) ) FD_LOG_ERR(( "unsupported --page-sz" ));
     292             : 
     293             :   if( argc!=3 ) {
     294             :     fprintf( stderr, "ERROR: expected 2 arguments, got %d\n", argc-1 );
     295             :     usage();
     296             :     return 1;
     297             :   }
     298             : 
     299             :   /* Acquire workspace */
     300             : 
     301             :   fd_wksp_t * wksp = fd_wksp_new_anonymous( page_sz, page_cnt, fd_log_cpu_id(), "wksp", 0UL );
     302             :   if( FD_UNLIKELY( !wksp ) ) FD_LOG_ERR(( "fd_wksp_new_anonymous() failed" ));
     303             : 
     304             :   /* Create scratch allocator */
     305             : 
     306             :   ulong  smax = scratch_mb << 20;
     307             :   void * smem = fd_wksp_alloc_laddr( wksp, fd_scratch_smem_align(), smax, 1UL );
     308             :   if( FD_UNLIKELY( !smem ) ) FD_LOG_ERR(( "Failed to alloc scratch mem" ));
     309             : 
     310             : # define SCRATCH_DEPTH (4UL)
     311             :   ulong fmem[ SCRATCH_DEPTH ] __attribute((aligned(FD_SCRATCH_FMEM_ALIGN)));
     312             : 
     313             :   fd_scratch_attach( smem, fmem, smax, SCRATCH_DEPTH );
     314             : 
     315             :   /* Create heap allocator */
     316             : 
     317             :   void * alloc_buf = fd_wksp_alloc_laddr( wksp, fd_alloc_align(), fd_alloc_footprint(), 2UL );
     318             :   fd_alloc_t * alloc = fd_alloc_join( fd_alloc_new( alloc_buf, 2UL ), 0UL );
     319             :   if( FD_UNLIKELY( !alloc ) ) FD_LOG_ERR(( "Failed to create heap" ));
     320             : 
     321             :   /* Create output file */
     322             : 
     323             :   FILE * out_file = fopen( argv[2], "wb" );
     324             :   if( FD_UNLIKELY( !out_file ) )
     325             :     FD_LOG_ERR(( "fopen(%s) failed (%d-%s)", argv[2], errno, strerror( errno ) ));
     326             :   if( FD_UNLIKELY( 0!=ftruncate( fileno( out_file ), 0L ) ) )
     327             :     FD_LOG_ERR(( "ftruncate failed (%d-%s)", errno, strerror( errno ) ));
     328             : 
     329             :   /* Create solcap writer */
     330             : 
     331             :   void * writer_mem = fd_wksp_alloc_laddr( wksp, fd_solcap_writer_align(), fd_solcap_writer_footprint(), 1UL );
     332             :   fd_solcap_writer_t * writer = fd_solcap_writer_init( fd_solcap_writer_new( writer_mem ), out_file );
     333             :   if( FD_UNLIKELY( !writer ))
     334             :     FD_LOG_ERR(( "Failed to create solcap writer" ));
     335             : 
     336             :   write_slots( argv[1], writer, wksp, alloc );
     337             : 
     338             :   /* Cleanup */
     339             : 
     340             :   fd_wksp_free_laddr( fd_solcap_writer_delete( fd_solcap_writer_flush( writer ) ) );
     341             :   fclose( out_file );
     342             :   fd_wksp_free_laddr( fd_alloc_delete( fd_alloc_leave( alloc ) ) );
     343             :   FD_TEST( fd_scratch_frame_used()==0UL );
     344             :   fd_wksp_free_laddr( fd_scratch_detach( NULL ) );
     345             :   fd_flamenco_halt();
     346             :   fd_halt();
     347             :   return 0;
     348             : }