Line data Source code
1 : #include "fd_chacha20rng.h"
2 : #include "../../util/simd/fd_avx.h"
3 : #include <assert.h>
4 :
5 3058481520 : #define wu_rol16(a) wb_exch_adj_pair( (a) )
6 3058481520 : #define wu_rol12(a) wu_rol( (a), 12 )
7 3058481520 : #define wu_rol7(a) wu_rol( (a), 7 )
8 :
9 : static inline __attribute__((always_inline)) wu_t
10 3058481520 : wu_rol8( wu_t x ) {
11 3058481520 : wb_t const mask =
12 3058481520 : wb( 3,0,1,2, 7,4,5,6, 11,8,9,10, 15,12,13,14,
13 3058481520 : 3,0,1,2, 7,4,5,6, 11,8,9,10, 15,12,13,14 );
14 3058481520 : return _mm256_shuffle_epi8( x, mask );
15 3058481520 : }
16 :
17 : void
18 38231019 : fd_chacha20rng_refill_avx( fd_chacha20rng_t * rng ) {
19 :
20 : /* This function should only be called if the buffer is empty. */
21 38231019 : assert( rng->buf_off == rng->buf_fill );
22 :
23 38231019 : wu_t iv0 = wu_bcast( 0x61707865U );
24 38231019 : wu_t iv1 = wu_bcast( 0x3320646eU );
25 38231019 : wu_t iv2 = wu_bcast( 0x79622d32U );
26 38231019 : wu_t iv3 = wu_bcast( 0x6b206574U );
27 38231019 : wb_t key = wb_ld( rng->key );
28 38231019 : wu_t zero = wu_zero();
29 :
30 : /* Unpack key equivalent to:
31 :
32 : c4 = wu_bcast( (uint const *)(rng->key)[0] );
33 : c5 = wu_bcast( (uint const *)(rng->key)[1] );
34 : ...
35 : cB = wu_bcast( (uint const *)(rng->key)[7] ); */
36 :
37 38231019 : wu_t key_lo = _mm256_permute2x128_si256( key, key, 0x00 ); /* [0,1,2,3,0,1,2,3] */
38 38231019 : wu_t key_hi = _mm256_permute2x128_si256( key, key, 0x11 ); /* [4,5,6,7,4,5,6,7] */
39 38231019 : wu_t k0 = _mm256_shuffle_epi32( key_lo, 0x00 );
40 38231019 : wu_t k1 = _mm256_shuffle_epi32( key_lo, 0x55 );
41 38231019 : wu_t k2 = _mm256_shuffle_epi32( key_lo, 0xaa );
42 38231019 : wu_t k3 = _mm256_shuffle_epi32( key_lo, 0xff );
43 38231019 : wu_t k4 = _mm256_shuffle_epi32( key_hi, 0x00 );
44 38231019 : wu_t k5 = _mm256_shuffle_epi32( key_hi, 0x55 );
45 38231019 : wu_t k6 = _mm256_shuffle_epi32( key_hi, 0xaa );
46 38231019 : wu_t k7 = _mm256_shuffle_epi32( key_hi, 0xff );
47 :
48 : /* Derive block index */
49 :
50 38231019 : ulong idx = rng->buf_fill / FD_CHACHA20_BLOCK_SZ; /* really a right shift */
51 38231019 : wu_t idxs = wu_add( wu_bcast( idx ), wu( 0, 1, 2, 3, 4, 5, 6, 7 ) );
52 :
53 : /* Run through the round function */
54 :
55 38231019 : wu_t c0 = iv0; wu_t c1 = iv1; wu_t c2 = iv2; wu_t c3 = iv3;
56 38231019 : wu_t c4 = k0; wu_t c5 = k1; wu_t c6 = k2; wu_t c7 = k3;
57 38231019 : wu_t c8 = k4; wu_t c9 = k5; wu_t cA = k6; wu_t cB = k7;
58 38231019 : wu_t cC = idxs; wu_t cD = zero; wu_t cE = zero; wu_t cF = zero;
59 :
60 38231019 : # define QUARTER_ROUND(a,b,c,d) \
61 3058481520 : do { \
62 3058481520 : a = wu_add( a, b ); d = wu_xor( d, a ); d = wu_rol16( d ); \
63 3058481520 : c = wu_add( c, d ); b = wu_xor( b, c ); b = wu_rol12( b ); \
64 3058481520 : a = wu_add( a, b ); d = wu_xor( d, a ); d = wu_rol8( d ); \
65 3058481520 : c = wu_add( c, d ); b = wu_xor( b, c ); b = wu_rol7( b ); \
66 3058481520 : } while(0)
67 :
68 420541209 : for( ulong i=0UL; i<10UL; i++ ) {
69 382310190 : QUARTER_ROUND( c0, c4, c8, cC );
70 382310190 : QUARTER_ROUND( c1, c5, c9, cD );
71 382310190 : QUARTER_ROUND( c2, c6, cA, cE );
72 382310190 : QUARTER_ROUND( c3, c7, cB, cF );
73 382310190 : QUARTER_ROUND( c0, c5, cA, cF );
74 382310190 : QUARTER_ROUND( c1, c6, cB, cC );
75 382310190 : QUARTER_ROUND( c2, c7, c8, cD );
76 382310190 : QUARTER_ROUND( c3, c4, c9, cE );
77 382310190 : }
78 38231019 : # undef QUARTER_ROUND
79 :
80 : /* Finalize */
81 :
82 38231019 : c0 = wu_add( c0, iv0 );
83 38231019 : c1 = wu_add( c1, iv1 );
84 38231019 : c2 = wu_add( c2, iv2 );
85 38231019 : c3 = wu_add( c3, iv3 );
86 38231019 : c4 = wu_add( c4, k0 );
87 38231019 : c5 = wu_add( c5, k1 );
88 38231019 : c6 = wu_add( c6, k2 );
89 38231019 : c7 = wu_add( c7, k3 );
90 38231019 : c8 = wu_add( c8, k4 );
91 38231019 : c9 = wu_add( c9, k5 );
92 38231019 : cA = wu_add( cA, k6 );
93 38231019 : cB = wu_add( cB, k7 );
94 38231019 : cC = wu_add( cC, idxs );
95 : //cD = wu_add( cD, zero );
96 : //cE = wu_add( cE, zero );
97 : //cF = wu_add( cF, zero );
98 :
99 : /* Transpose matrix to get output vector */
100 :
101 38231019 : wu_transpose_8x8( c0, c1, c2, c3, c4, c5, c6, c7,
102 38231019 : c0, c1, c2, c3, c4, c5, c6, c7 );
103 38231019 : wu_transpose_8x8( c8, c9, cA, cB, cC, cD, cE, cF,
104 38231019 : c8, c9, cA, cB, cC, cD, cE, cF );
105 :
106 : /* Update ring buffer */
107 :
108 38231019 : uint * out = (uint *)rng->buf;
109 38231019 : wu_st( out+0x00, c0 ); wu_st( out+0x08, c8 );
110 38231019 : wu_st( out+0x10, c1 ); wu_st( out+0x18, c9 );
111 38231019 : wu_st( out+0x20, c2 ); wu_st( out+0x28, cA );
112 38231019 : wu_st( out+0x30, c3 ); wu_st( out+0x38, cB );
113 38231019 : wu_st( out+0x40, c4 ); wu_st( out+0x48, cC );
114 38231019 : wu_st( out+0x50, c5 ); wu_st( out+0x58, cD );
115 38231019 : wu_st( out+0x60, c6 ); wu_st( out+0x68, cE );
116 38231019 : wu_st( out+0x70, c7 ); wu_st( out+0x78, cF );
117 :
118 : /* Update ring descriptor */
119 :
120 38231019 : rng->buf_fill += 8*FD_CHACHA20_BLOCK_SZ;
121 38231019 : }
|
