Line data    Source code

       1             : #include "fd_chacha_rng.h"
       2             : #include "../../util/simd/fd_avx.h"
       3             : 
       4  1344899344 : #define wu_rol16(a) wb_exch_adj_pair( (a) )
       5  1344899344 : #define wu_rol12(a) wu_rol( (a), 12 )
       6  1344899344 : #define wu_rol7(a)  wu_rol( (a),  7 )
       7             : 
       8             : static inline __attribute__((always_inline)) wu_t
       9  1344899344 : wu_rol8( wu_t x ) {
      10  1344899344 :   wb_t const mask =
      11  1344899344 :     wb_bcast_hex( 3,0,1,2, 7,4,5,6, 11,8,9,10, 15,12,13,14 );
      12  1344899344 :   return _mm256_shuffle_epi8( x, mask );
      13  1344899344 : }
      14             : 
      15             : __attribute__((always_inline)) static inline void
      16             : fd_chacha_rng_refill_avx( fd_chacha_rng_t * rng,
      17    21664469 :                           ulong             rnd2_cnt ) {
      18             : 
      19    21664469 :   wu_t iv0  = wu_bcast( 0x61707865U );
      20    21664469 :   wu_t iv1  = wu_bcast( 0x3320646eU );
      21    21664469 :   wu_t iv2  = wu_bcast( 0x79622d32U );
      22    21664469 :   wu_t iv3  = wu_bcast( 0x6b206574U );
      23    21664469 :   wb_t key  = wb_ld( rng->key );
      24    21664469 :   wu_t zero = wu_zero();
      25             : 
      26             :   /* Unpack key equivalent to:
      27             : 
      28             :        c4 = wu_bcast( (uint const *)(rng->key)[0] );
      29             :        c5 = wu_bcast( (uint const *)(rng->key)[1] );
      30             :        ...
      31             :        cB = wu_bcast( (uint const *)(rng->key)[7] ); */
      32             : 
      33    21664469 :   wu_t key_lo = _mm256_permute2x128_si256( key, key, 0x00 );  /* [0,1,2,3,0,1,2,3] */
      34    21664469 :   wu_t key_hi = _mm256_permute2x128_si256( key, key, 0x11 );  /* [4,5,6,7,4,5,6,7] */
      35    21664469 :   wu_t k0 = _mm256_shuffle_epi32( key_lo, 0x00 );
      36    21664469 :   wu_t k1 = _mm256_shuffle_epi32( key_lo, 0x55 );
      37    21664469 :   wu_t k2 = _mm256_shuffle_epi32( key_lo, 0xaa );
      38    21664469 :   wu_t k3 = _mm256_shuffle_epi32( key_lo, 0xff );
      39    21664469 :   wu_t k4 = _mm256_shuffle_epi32( key_hi, 0x00 );
      40    21664469 :   wu_t k5 = _mm256_shuffle_epi32( key_hi, 0x55 );
      41    21664469 :   wu_t k6 = _mm256_shuffle_epi32( key_hi, 0xaa );
      42    21664469 :   wu_t k7 = _mm256_shuffle_epi32( key_hi, 0xff );
      43             : 
      44             :   /* Derive block index (64-bit counter split across words 12-13) */
      45             : 
      46    21664469 :   ulong idx = rng->buf_fill / FD_CHACHA_BLOCK_SZ;  /* really a right shift */
      47    21664469 :   wu_t offsets = wu( 0, 1, 2, 3, 4, 5, 6, 7 );
      48    21664469 :   wu_t idxs_lo = wu_add( wu_bcast( (uint)idx ), offsets );
      49    21664469 :   wu_t carry   = wu_and( wu_lt( idxs_lo, offsets ), wu_bcast( 1U ) );
      50    21664469 :   wu_t idxs_hi = wu_add( wu_bcast( (uint)(idx>>32) ), carry );
      51             : 
      52             :   /* Run through the round function */
      53             : 
      54    21664469 :   wu_t c0 = iv0;      wu_t c1 = iv1;      wu_t c2 = iv2;   wu_t c3 = iv3;
      55    21664469 :   wu_t c4 = k0;       wu_t c5 = k1;       wu_t c6 = k2;    wu_t c7 = k3;
      56    21664469 :   wu_t c8 = k4;       wu_t c9 = k5;       wu_t cA = k6;    wu_t cB = k7;
      57    21664469 :   wu_t cC = idxs_lo;  wu_t cD = idxs_hi;  wu_t cE = zero;  wu_t cF = zero;
      58             : 
      59    21664469 : # define QUARTER_ROUND(a,b,c,d)                                        \
      60  1344899344 :   do {                                                                 \
      61  1344899344 :     a = wu_add( a, b ); d = wu_xor( d, a ); d = wu_rol16( d );         \
      62  1344899344 :     c = wu_add( c, d ); b = wu_xor( b, c ); b = wu_rol12( b );         \
      63  1344899344 :     a = wu_add( a, b ); d = wu_xor( d, a ); d = wu_rol8( d );          \
      64  1344899344 :     c = wu_add( c, d ); b = wu_xor( b, c ); b = wu_rol7( b );          \
      65  1344899344 :   } while(0)
      66             : 
      67   189776887 :   for( ulong i=0UL; i<rnd2_cnt; i++ ) {
      68   168112418 :     QUARTER_ROUND( c0, c4, c8, cC );
      69   168112418 :     QUARTER_ROUND( c1, c5, c9, cD );
      70   168112418 :     QUARTER_ROUND( c2, c6, cA, cE );
      71   168112418 :     QUARTER_ROUND( c3, c7, cB, cF );
      72   168112418 :     QUARTER_ROUND( c0, c5, cA, cF );
      73   168112418 :     QUARTER_ROUND( c1, c6, cB, cC );
      74   168112418 :     QUARTER_ROUND( c2, c7, c8, cD );
      75   168112418 :     QUARTER_ROUND( c3, c4, c9, cE );
      76   168112418 :   }
      77    21664469 : # undef QUARTER_ROUND
      78             : 
      79             :   /* Finalize */
      80             : 
      81    21664469 :   c0 = wu_add( c0, iv0  );
      82    21664469 :   c1 = wu_add( c1, iv1  );
      83    21664469 :   c2 = wu_add( c2, iv2  );
      84    21664469 :   c3 = wu_add( c3, iv3  );
      85    21664469 :   c4 = wu_add( c4, k0   );
      86    21664469 :   c5 = wu_add( c5, k1   );
      87    21664469 :   c6 = wu_add( c6, k2   );
      88    21664469 :   c7 = wu_add( c7, k3   );
      89    21664469 :   c8 = wu_add( c8, k4   );
      90    21664469 :   c9 = wu_add( c9, k5   );
      91    21664469 :   cA = wu_add( cA, k6   );
      92    21664469 :   cB = wu_add( cB, k7   );
      93    21664469 :   cC = wu_add( cC, idxs_lo );
      94    21664469 :   cD = wu_add( cD, idxs_hi );
      95             :   //cE = wu_add( cE, zero );
      96             :   //cF = wu_add( cF, zero );
      97             : 
      98             :   /* Transpose matrix to get output vector */
      99             : 
     100    21664469 :   wu_transpose_8x8( c0, c1, c2, c3, c4, c5, c6, c7,
     101    21664469 :                     c0, c1, c2, c3, c4, c5, c6, c7 );
     102    21664469 :   wu_transpose_8x8( c8, c9, cA, cB, cC, cD, cE, cF,
     103    21664469 :                     c8, c9, cA, cB, cC, cD, cE, cF );
     104             : 
     105             :   /* Update ring buffer */
     106             : 
     107    21664469 :   uint * out = (uint *)rng->buf;
     108    21664469 :   wu_st( out+0x00, c0 ); wu_st( out+0x08, c8 );
     109    21664469 :   wu_st( out+0x10, c1 ); wu_st( out+0x18, c9 );
     110    21664469 :   wu_st( out+0x20, c2 ); wu_st( out+0x28, cA );
     111    21664469 :   wu_st( out+0x30, c3 ); wu_st( out+0x38, cB );
     112    21664469 :   wu_st( out+0x40, c4 ); wu_st( out+0x48, cC );
     113    21664469 :   wu_st( out+0x50, c5 ); wu_st( out+0x58, cD );
     114    21664469 :   wu_st( out+0x60, c6 ); wu_st( out+0x68, cE );
     115    21664469 :   wu_st( out+0x70, c7 ); wu_st( out+0x78, cF );
     116             : 
     117             :   /* Update ring descriptor */
     118             : 
     119    21664469 :   rng->buf_fill += 8*FD_CHACHA_BLOCK_SZ;
     120    21664469 : }
     121             : 
     122             : void
     123     8088712 : fd_chacha8_rng_refill_avx( fd_chacha_rng_t * rng ) {
     124     8088712 :   fd_chacha_rng_refill_avx( rng, 4UL );
     125     8088712 : }
     126             : 
     127             : void
     128    13575757 : fd_chacha20_rng_refill_avx( fd_chacha_rng_t * rng ) {
     129    13575757 :   fd_chacha_rng_refill_avx( rng, 10UL );
     130    13575757 : }