Line data    Source code

       1             : #include "fd_chacha_rng.h"
       2             : #include "../../util/simd/fd_avx512.h"
       3             : 
       4   375163424 : #define wwu_rol16(a) wwb_exch_adj_pair( (a) )
       5   375163424 : #define wwu_rol12(a) wwu_rol( (a), 12 )
       6   375163424 : #define wwu_rol7(a)  wwu_rol( (a),  7 )
       7             : 
       8             : static inline __attribute__((always_inline)) wwu_t
       9   375163424 : wwu_rol8( wwu_t x ) {
      10   375163424 :   wwb_t const mask =
      11   375163424 :     wwb_bcast_hex( 3,0,1,2, 7,4,5,6, 11,8,9,10, 15,12,13,14 );
      12   375163424 :   return _mm512_shuffle_epi8( x, mask );
      13   375163424 : }
      14             : 
      15             : static void
      16             : fd_chacha_rng_refill_avx512( fd_chacha_rng_t * rng,
      17     6217621 :                              ulong             rnd2_cnt ) {
      18             : 
      19             :   /* This function should only be called if the buffer is empty. */
      20     6217621 :   if( FD_UNLIKELY( rng->buf_off != rng->buf_fill ) ) {
      21           0 :     FD_LOG_CRIT(( "refill out of sync: buf_off=%lu buf_fill=%lu", rng->buf_off, rng->buf_fill ));
      22           0 :   }
      23             : 
      24     6217621 :   wwu_t iv0  = wwu_bcast( 0x61707865U );
      25     6217621 :   wwu_t iv1  = wwu_bcast( 0x3320646eU );
      26     6217621 :   wwu_t iv2  = wwu_bcast( 0x79622d32U );
      27     6217621 :   wwu_t iv3  = wwu_bcast( 0x6b206574U );
      28     6217621 :   wwu_t zero = wwu_zero();
      29             : 
      30             :   /* Unpack key equivalent to:
      31             : 
      32             :        c4 = wwu_bcast( (uint const *)(rng->key)[0] );
      33             :        c5 = wwu_bcast( (uint const *)(rng->key)[1] );
      34             :        ...
      35             :        cB = wwu_bcast( (uint const *)(rng->key)[7] ); */
      36             : 
      37     6217621 :   __m128i key_lo_v = _mm_load_si128( (__m128i const *)rng->key   ); /* [0,1,2,3] */
      38     6217621 :   __m128i key_hi_v = _mm_load_si128( (__m128i const *)rng->key+1 ); /* [4,5,6,7] */
      39     6217621 :   wwu_t key_lo = _mm512_broadcast_i32x4( key_lo_v );  /* [0,1,2,3,0,1,2,3] */
      40     6217621 :   wwu_t key_hi = _mm512_broadcast_i32x4( key_hi_v );  /* [4,5,6,7,4,5,6,7] */
      41     6217621 :   wwu_t k0 = _mm512_shuffle_epi32( key_lo, 0x00 );
      42     6217621 :   wwu_t k1 = _mm512_shuffle_epi32( key_lo, 0x55 );
      43     6217621 :   wwu_t k2 = _mm512_shuffle_epi32( key_lo, 0xaa );
      44     6217621 :   wwu_t k3 = _mm512_shuffle_epi32( key_lo, 0xff );
      45     6217621 :   wwu_t k4 = _mm512_shuffle_epi32( key_hi, 0x00 );
      46     6217621 :   wwu_t k5 = _mm512_shuffle_epi32( key_hi, 0x55 );
      47     6217621 :   wwu_t k6 = _mm512_shuffle_epi32( key_hi, 0xaa );
      48     6217621 :   wwu_t k7 = _mm512_shuffle_epi32( key_hi, 0xff );
      49             : 
      50             :   /* Derive block index (64-bit counter split across words 12-13) */
      51             : 
      52     6217621 :   ulong idx = rng->buf_fill / FD_CHACHA_BLOCK_SZ;  /* really a right shift */
      53     6217621 :   wwu_t offsets  = wwu( 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 );
      54     6217621 :   wwu_t idxs_lo  = wwu_add( wwu_bcast( (uint)idx ), offsets );
      55     6217621 :   int   carry    = wwu_lt( idxs_lo, offsets );
      56     6217621 :   wwu_t idxs_hi  = wwu_add_if( carry, wwu_bcast( (uint)(idx>>32) ), wwu_bcast( 1U ), wwu_bcast( (uint)(idx>>32) ) );
      57             : 
      58             :   /* Run through the round function */
      59             : 
      60     6217621 :   wwu_t c0 = iv0;      wwu_t c1 = iv1;      wwu_t c2 = iv2;   wwu_t c3 = iv3;
      61     6217621 :   wwu_t c4 = k0;       wwu_t c5 = k1;       wwu_t c6 = k2;    wwu_t c7 = k3;
      62     6217621 :   wwu_t c8 = k4;       wwu_t c9 = k5;       wwu_t cA = k6;    wwu_t cB = k7;
      63     6217621 :   wwu_t cC = idxs_lo;  wwu_t cD = idxs_hi;  wwu_t cE = zero;  wwu_t cF = zero;
      64             : 
      65     6217621 : # define QUARTER_ROUND(a,b,c,d)                                   \
      66   375163424 :   do {                                                            \
      67   375163424 :     a = wwu_add( a, b ); d = wwu_xor( d, a ); d = wwu_rol16( d ); \
      68   375163424 :     c = wwu_add( c, d ); b = wwu_xor( b, c ); b = wwu_rol12( b ); \
      69   375163424 :     a = wwu_add( a, b ); d = wwu_xor( d, a ); d = wwu_rol8( d );  \
      70   375163424 :     c = wwu_add( c, d ); b = wwu_xor( b, c ); b = wwu_rol7( b );  \
      71   375163424 :   } while(0)
      72             : 
      73    53113049 :   for( ulong i=0UL; i<rnd2_cnt; i++ ) {
      74    46895428 :     QUARTER_ROUND( c0, c4, c8, cC );
      75    46895428 :     QUARTER_ROUND( c1, c5, c9, cD );
      76    46895428 :     QUARTER_ROUND( c2, c6, cA, cE );
      77    46895428 :     QUARTER_ROUND( c3, c7, cB, cF );
      78    46895428 :     QUARTER_ROUND( c0, c5, cA, cF );
      79    46895428 :     QUARTER_ROUND( c1, c6, cB, cC );
      80    46895428 :     QUARTER_ROUND( c2, c7, c8, cD );
      81    46895428 :     QUARTER_ROUND( c3, c4, c9, cE );
      82    46895428 :   }
      83     6217621 : # undef QUARTER_ROUND
      84             : 
      85             :   /* Finalize */
      86             : 
      87     6217621 :   c0 = wwu_add( c0, iv0  );
      88     6217621 :   c1 = wwu_add( c1, iv1  );
      89     6217621 :   c2 = wwu_add( c2, iv2  );
      90     6217621 :   c3 = wwu_add( c3, iv3  );
      91     6217621 :   c4 = wwu_add( c4, k0   );
      92     6217621 :   c5 = wwu_add( c5, k1   );
      93     6217621 :   c6 = wwu_add( c6, k2   );
      94     6217621 :   c7 = wwu_add( c7, k3   );
      95     6217621 :   c8 = wwu_add( c8, k4   );
      96     6217621 :   c9 = wwu_add( c9, k5   );
      97     6217621 :   cA = wwu_add( cA, k6   );
      98     6217621 :   cB = wwu_add( cB, k7   );
      99     6217621 :   cC = wwu_add( cC, idxs_lo );
     100     6217621 :   cD = wwu_add( cD, idxs_hi );
     101             :   //cE = wwu_add( cE, zero );
     102             :   //cF = wwu_add( cF, zero );
     103             : 
     104             :   /* Transpose matrix to get output vector */
     105             : 
     106     6217621 :   wwu_transpose_16x16( c0, c1, c2, c3, c4, c5, c6, c7,
     107     6217621 :                        c8, c9, cA, cB, cC, cD, cE, cF,
     108     6217621 :                        c0, c1, c2, c3, c4, c5, c6, c7,
     109     6217621 :                        c8, c9, cA, cB, cC, cD, cE, cF );
     110             : 
     111             :   /* Update ring buffer */
     112             : 
     113     6217621 :   uint * out = (uint *)rng->buf;
     114     6217621 :   wwu_st( out+0x00, c0 ); wwu_st( out+0x10, c1 );
     115     6217621 :   wwu_st( out+0x20, c2 ); wwu_st( out+0x30, c3 );
     116     6217621 :   wwu_st( out+0x40, c4 ); wwu_st( out+0x50, c5 );
     117     6217621 :   wwu_st( out+0x60, c6 ); wwu_st( out+0x70, c7 );
     118     6217621 :   wwu_st( out+0x80, c8 ); wwu_st( out+0x90, c9 );
     119     6217621 :   wwu_st( out+0xa0, cA ); wwu_st( out+0xb0, cB );
     120     6217621 :   wwu_st( out+0xc0, cC ); wwu_st( out+0xd0, cD );
     121     6217621 :   wwu_st( out+0xe0, cE ); wwu_st( out+0xf0, cF );
     122             : 
     123             :   /* Update ring descriptor */
     124             : 
     125     6217621 :   rng->buf_fill += 16*FD_CHACHA_BLOCK_SZ;
     126     6217621 : }
     127             : 
     128             : void
     129     2546797 : fd_chacha8_rng_refill_avx512( fd_chacha_rng_t * rng ) {
     130     2546797 :   fd_chacha_rng_refill_avx512( rng, 4UL );
     131     2546797 : }
     132             : 
     133             : void
     134     3670824 : fd_chacha20_rng_refill_avx512( fd_chacha_rng_t * rng ) {
     135     3670824 :   fd_chacha_rng_refill_avx512( rng, 10UL );
     136     3670824 : }