Line data Source code
1 : #define FD_SHA256_BATCH_IMPL 2
2 :
3 : #include "fd_sha256.h"
4 : #include "fd_sha256_constants.h"
5 : #include "../../util/simd/fd_avx512.h"
6 : #include "../../util/simd/fd_avx.h"
7 :
8 : FD_STATIC_ASSERT( FD_SHA256_BATCH_MAX==16UL, compat );
9 :
10 : void
11 : fd_sha256_private_batch_avx( ulong batch_cnt,
12 : void const * batch_data,
13 : ulong const * batch_sz,
14 : void * const * batch_hash );
15 :
16 : void
17 : fd_sha256_private_batch_avx512( ulong batch_cnt,
18 : void const * _batch_data,
19 : ulong const * batch_sz,
20 1080626 : void * const * _batch_hash ) {
21 :
22 : /* If the batch is small enough, it is more efficient to use the
23 : narrow batched implementations. The threshold for fallback depends
24 : on whether that itself narrower batched implementation is using
25 : SHA-NI acceleration for really small batches. */
26 :
27 1080626 : # if FD_HAS_SHANI
28 1080626 : # define MIN_BATCH_CNT (5UL)
29 : # else
30 : # define MIN_BATCH_CNT (2UL)
31 : # endif
32 :
33 1080626 : if( FD_UNLIKELY( batch_cnt<MIN_BATCH_CNT ) ) {
34 502982 : fd_sha256_private_batch_avx( batch_cnt, _batch_data, batch_sz, _batch_hash );
35 502982 : return;
36 502982 : }
37 :
38 577644 : # undef MIN_BATCH_CNT
39 :
40 : /* SHA appends to the end of each message 9 bytes of additional data
41 : (a messaging terminator byte and the big endian ulong with the
42 : message size in bits) and enough zero padding to make the message
43 : an integer number of blocks long. We compute the 1 or 2 tail
44 : blocks of each message here. We then process complete blocks of
45 : the original messages in place, switching to processing these tail
46 : blocks in the same pass toward the end. TODO: This code could
47 : probably be SIMD optimized slightly more (this is where all the
48 : really performance suboptimally designed parts of SHA live so it is
49 : just inherently gross). The main optimization would probably be to
50 : allow tail reading to use a faster memcpy and then maybe some
51 : vectorization of the bswap. */
52 :
53 577644 : ulong const * batch_data = (ulong const *)_batch_data;
54 :
55 577644 : ulong batch_tail_data[ FD_SHA256_BATCH_MAX ] __attribute__((aligned(64)));
56 577644 : ulong batch_tail_rem [ FD_SHA256_BATCH_MAX ] __attribute__((aligned(64)));
57 :
58 577644 : uchar scratch[ FD_SHA256_BATCH_MAX*2UL*FD_SHA256_PRIVATE_BUF_MAX ] __attribute__((aligned(128)));
59 577644 : do {
60 577644 : ulong scratch_free = (ulong)scratch;
61 :
62 577644 : wwv_t zero = wwv_zero();
63 :
64 8911423 : for( ulong batch_idx=0UL; batch_idx<batch_cnt; batch_idx++ ) {
65 :
66 : /* Allocate the tail blocks for this message */
67 :
68 8333779 : ulong data = batch_data[ batch_idx ];
69 8333779 : ulong sz = batch_sz [ batch_idx ];
70 :
71 8333779 : ulong tail_data = scratch_free;
72 8333779 : ulong tail_data_sz = sz & (FD_SHA256_PRIVATE_BUF_MAX-1UL);
73 8333779 : ulong tail_data_off = fd_ulong_align_dn( sz, FD_SHA256_PRIVATE_BUF_MAX );
74 8333779 : ulong tail_sz = fd_ulong_align_up( tail_data_sz+9UL, FD_SHA256_PRIVATE_BUF_MAX );
75 :
76 8333779 : batch_tail_data[ batch_idx ] = tail_data;
77 8333779 : batch_tail_rem [ batch_idx ] = tail_sz >> FD_SHA256_PRIVATE_LG_BUF_MAX;
78 :
79 8333779 : scratch_free += tail_sz;
80 :
81 : /* Populate the tail blocks. We first clear the blocks (note that
82 : it is okay to clobber bytes 64:127 if tail_sz only 64, saving a
83 : nasty branch). Then we copy any straggler data bytes into the
84 : tail, terminate the message, and finally record the size of the
85 : message in bits at the end as a big endian ulong. */
86 :
87 8333779 : wwv_st( (ulong *) tail_data, zero );
88 8333779 : wwv_st( (ulong *)(tail_data+64), zero );
89 :
90 8333779 : # if 1
91 : /* Quick experiments found that, once again, straight memcpy is
92 : much slower than a fd_memcpy is slightly slower than a
93 : site-optimized handrolled memcpy (fd_memcpy would be less L1I
94 : cache footprint though). They also found that doing the below
95 : in a branchless way is slightly worse and an ILP optimized
96 : version of the conditional calculation is about the same. They
97 : also found that vectorizing the overall loop and/or Duffing the
98 : vectorized loop did not provide noticeable performance
99 : improvements under various styles of memcpy. */
100 8333779 : ulong src = data + tail_data_off;
101 8333779 : ulong dst = tail_data;
102 8333779 : ulong rem = tail_data_sz;
103 10863962 : while( rem>=32UL ) { wv_st( (ulong *)dst, wv_ldu( (ulong const *)src ) ); dst += 32UL; src += 32UL; rem -= 32UL; }
104 15403323 : while( rem>= 8UL ) { *(ulong *)dst = FD_LOAD( ulong, src ); dst += 8UL; src += 8UL; rem -= 8UL; }
105 8333779 : if ( rem>= 4UL ) { *(uint *)dst = FD_LOAD( uint, src ); dst += 4UL; src += 4UL; rem -= 4UL; }
106 8333779 : if ( rem>= 2UL ) { *(ushort *)dst = FD_LOAD( ushort, src ); dst += 2UL; src += 2UL; rem -= 2UL; }
107 8333779 : if ( rem ) { *(uchar *)dst = FD_LOAD( uchar, src ); dst++; }
108 8333779 : *(uchar *)dst = (uchar)0x80;
109 : # else
110 : fd_memcpy( (void *)tail_data, (void const *)(data + tail_data_off), tail_data_sz );
111 : *((uchar *)(tail_data+tail_data_sz)) = (uchar)0x80;
112 : # endif
113 :
114 8333779 : *((ulong *)(tail_data+tail_sz-8UL )) = fd_ulong_bswap( sz<<3 );
115 8333779 : }
116 577644 : } while(0);
117 :
118 577644 : wwu_t s0 = wwu_bcast( FD_SHA256_INITIAL_A );
119 577644 : wwu_t s1 = wwu_bcast( FD_SHA256_INITIAL_B );
120 577644 : wwu_t s2 = wwu_bcast( FD_SHA256_INITIAL_C );
121 577644 : wwu_t s3 = wwu_bcast( FD_SHA256_INITIAL_D );
122 577644 : wwu_t s4 = wwu_bcast( FD_SHA256_INITIAL_E );
123 577644 : wwu_t s5 = wwu_bcast( FD_SHA256_INITIAL_F );
124 577644 : wwu_t s6 = wwu_bcast( FD_SHA256_INITIAL_G );
125 577644 : wwu_t s7 = wwu_bcast( FD_SHA256_INITIAL_H );
126 :
127 577644 : wwv_t zero = wwv_zero();
128 577644 : wwv_t one = wwv_one();
129 577644 : wwv_t wwv_64 = wwv_bcast( FD_SHA256_PRIVATE_BUF_MAX );
130 577644 : wwv_t W_sentinel = wwv_bcast( (ulong)scratch );
131 :
132 577644 : wwv_t tail_lo = wwv_ld( batch_tail_data ); wwv_t tail_hi = wwv_ld( batch_tail_data+8 );
133 577644 : wwv_t tail_rem_lo = wwv_ld( batch_tail_rem ); wwv_t tail_rem_hi = wwv_ld( batch_tail_rem +8 );
134 577644 : wwv_t W_lo = wwv_ld( batch_data ); wwv_t W_hi = wwv_ld( batch_data +8 );
135 :
136 577644 : wwv_t block_rem_lo = wwv_if( ((1<<batch_cnt)-1) & 0xff,
137 577644 : wwv_add( wwv_shr( wwv_ld( batch_sz ), FD_SHA256_PRIVATE_LG_BUF_MAX ), tail_rem_lo ), zero );
138 577644 : wwv_t block_rem_hi = wwv_if( ((1<<batch_cnt)-1) >> 8,
139 577644 : wwv_add( wwv_shr( wwv_ld( batch_sz+8 ), FD_SHA256_PRIVATE_LG_BUF_MAX ), tail_rem_hi ), zero );
140 :
141 5699416 : for(;;) {
142 5699416 : int active_lane_lo = wwv_ne( block_rem_lo, zero );
143 5699416 : int active_lane_hi = wwv_ne( block_rem_hi, zero );
144 5699416 : if( FD_UNLIKELY( !(active_lane_lo | active_lane_hi) ) ) break;
145 :
146 : /* Switch lanes that have hit the end of their in-place bulk
147 : processing to their out-of-place scratch tail regions as
148 : necessary. */
149 :
150 5121772 : W_lo = wwv_if( wwv_eq( block_rem_lo, tail_rem_lo ), tail_lo, W_lo );
151 5121772 : W_hi = wwv_if( wwv_eq( block_rem_hi, tail_rem_hi ), tail_hi, W_hi );
152 :
153 : /* At this point, we have at least 1 block in this message segment
154 : pass that has not been processed. Load the next 64 bytes of
155 : each unprocessed block. Inactive lanes (e.g. message segments
156 : in this pass for which we've already processed all the blocks)
157 : will load garbage from a sentinel location (and the result of
158 : the state computations for the inactive lane will be ignored). */
159 :
160 5121772 : ulong _W0; ulong _W1; ulong _W2; ulong _W3; ulong _W4; ulong _W5; ulong _W6; ulong _W7;
161 5121772 : ulong _W8; ulong _W9; ulong _Wa; ulong _Wb; ulong _Wc; ulong _Wd; ulong _We; ulong _Wf;
162 5121772 : wwv_unpack( wwv_if( active_lane_lo, W_lo, W_sentinel ), _W0, _W1, _W2, _W3, _W4, _W5, _W6, _W7 );
163 5121772 : wwv_unpack( wwv_if( active_lane_hi, W_hi, W_sentinel ), _W8, _W9, _Wa, _Wb, _Wc, _Wd, _We, _Wf );
164 5121772 : uchar const * W0 = (uchar const *)_W0; uchar const * W1 = (uchar const *)_W1;
165 5121772 : uchar const * W2 = (uchar const *)_W2; uchar const * W3 = (uchar const *)_W3;
166 5121772 : uchar const * W4 = (uchar const *)_W4; uchar const * W5 = (uchar const *)_W5;
167 5121772 : uchar const * W6 = (uchar const *)_W6; uchar const * W7 = (uchar const *)_W7;
168 5121772 : uchar const * W8 = (uchar const *)_W8; uchar const * W9 = (uchar const *)_W9;
169 5121772 : uchar const * Wa = (uchar const *)_Wa; uchar const * Wb = (uchar const *)_Wb;
170 5121772 : uchar const * Wc = (uchar const *)_Wc; uchar const * Wd = (uchar const *)_Wd;
171 5121772 : uchar const * We = (uchar const *)_We; uchar const * Wf = (uchar const *)_Wf;
172 :
173 5121772 : wwu_t x0; wwu_t x1; wwu_t x2; wwu_t x3; wwu_t x4; wwu_t x5; wwu_t x6; wwu_t x7;
174 5121772 : wwu_t x8; wwu_t x9; wwu_t xa; wwu_t xb; wwu_t xc; wwu_t xd; wwu_t xe; wwu_t xf;
175 5121772 : wwu_transpose_16x16( wwu_bswap( wwu_ldu( W0 ) ), wwu_bswap( wwu_ldu( W1 ) ),
176 5121772 : wwu_bswap( wwu_ldu( W2 ) ), wwu_bswap( wwu_ldu( W3 ) ),
177 5121772 : wwu_bswap( wwu_ldu( W4 ) ), wwu_bswap( wwu_ldu( W5 ) ),
178 5121772 : wwu_bswap( wwu_ldu( W6 ) ), wwu_bswap( wwu_ldu( W7 ) ),
179 5121772 : wwu_bswap( wwu_ldu( W8 ) ), wwu_bswap( wwu_ldu( W9 ) ),
180 5121772 : wwu_bswap( wwu_ldu( Wa ) ), wwu_bswap( wwu_ldu( Wb ) ),
181 5121772 : wwu_bswap( wwu_ldu( Wc ) ), wwu_bswap( wwu_ldu( Wd ) ),
182 5121772 : wwu_bswap( wwu_ldu( We ) ), wwu_bswap( wwu_ldu( Wf ) ),
183 5121772 : x0, x1, x2, x3, x4, x5, x6, x7, x8, x9, xa, xb, xc, xd, xe, xf );
184 :
185 : /* Compute the SHA-256 state updates */
186 :
187 5121772 : wwu_t a = s0; wwu_t b = s1; wwu_t c = s2; wwu_t d = s3; wwu_t e = s4; wwu_t f = s5; wwu_t g = s6; wwu_t h = s7;
188 :
189 : /* One vpternlogd per 3-input boolean; GCC does not fuse these. */
190 5121772 : # define Sigma0(x) _mm512_ternarylogic_epi32( wwu_rol(x,30), wwu_rol(x,19), wwu_rol(x,10), 0x96 )
191 5121772 : # define Sigma1(x) _mm512_ternarylogic_epi32( wwu_rol(x,26), wwu_rol(x,21), wwu_rol(x, 7), 0x96 )
192 5121772 : # define sigma0(x) _mm512_ternarylogic_epi32( wwu_rol(x,25), wwu_rol(x,14), wwu_shr(x, 3), 0x96 )
193 5121772 : # define sigma1(x) _mm512_ternarylogic_epi32( wwu_rol(x,15), wwu_rol(x,13), wwu_shr(x,10), 0x96 )
194 5121772 : # define Ch(x,y,z) _mm512_ternarylogic_epi32( (x), (y), (z), 0xCA )
195 5121772 : # define Maj(x,y,z) _mm512_ternarylogic_epi32( (x), (y), (z), 0xE8 )
196 5121772 : # define SHA_CORE(xi,ki) \
197 327793408 : T1 = wwu_add( wwu_add(xi,ki), wwu_add( wwu_add( h, Sigma1(e) ), Ch(e, f, g) ) ); \
198 327793408 : T2 = wwu_add( Sigma0(a), Maj(a, b, c) ); \
199 327793408 : h = g; \
200 327793408 : g = f; \
201 327793408 : f = e; \
202 327793408 : e = wwu_add( d, T1 ); \
203 327793408 : d = c; \
204 327793408 : c = b; \
205 327793408 : b = a; \
206 327793408 : a = wwu_add( T1, T2 )
207 :
208 5121772 : wwu_t T1;
209 5121772 : wwu_t T2;
210 :
211 5121772 : SHA_CORE( x0, wwu_bcast( fd_sha256_K[ 0] ) );
212 5121772 : SHA_CORE( x1, wwu_bcast( fd_sha256_K[ 1] ) );
213 5121772 : SHA_CORE( x2, wwu_bcast( fd_sha256_K[ 2] ) );
214 5121772 : SHA_CORE( x3, wwu_bcast( fd_sha256_K[ 3] ) );
215 5121772 : SHA_CORE( x4, wwu_bcast( fd_sha256_K[ 4] ) );
216 5121772 : SHA_CORE( x5, wwu_bcast( fd_sha256_K[ 5] ) );
217 5121772 : SHA_CORE( x6, wwu_bcast( fd_sha256_K[ 6] ) );
218 5121772 : SHA_CORE( x7, wwu_bcast( fd_sha256_K[ 7] ) );
219 5121772 : SHA_CORE( x8, wwu_bcast( fd_sha256_K[ 8] ) );
220 5121772 : SHA_CORE( x9, wwu_bcast( fd_sha256_K[ 9] ) );
221 5121772 : SHA_CORE( xa, wwu_bcast( fd_sha256_K[10] ) );
222 5121772 : SHA_CORE( xb, wwu_bcast( fd_sha256_K[11] ) );
223 5121772 : SHA_CORE( xc, wwu_bcast( fd_sha256_K[12] ) );
224 5121772 : SHA_CORE( xd, wwu_bcast( fd_sha256_K[13] ) );
225 5121772 : SHA_CORE( xe, wwu_bcast( fd_sha256_K[14] ) );
226 5121772 : SHA_CORE( xf, wwu_bcast( fd_sha256_K[15] ) );
227 20487088 : for( ulong i=16UL; i<64UL; i+=16UL ) {
228 15365316 : x0 = wwu_add( wwu_add( x0, sigma0(x1) ), wwu_add( sigma1(xe), x9 ) ); SHA_CORE( x0, wwu_bcast( fd_sha256_K[i ] ) );
229 15365316 : x1 = wwu_add( wwu_add( x1, sigma0(x2) ), wwu_add( sigma1(xf), xa ) ); SHA_CORE( x1, wwu_bcast( fd_sha256_K[i+ 1UL] ) );
230 15365316 : x2 = wwu_add( wwu_add( x2, sigma0(x3) ), wwu_add( sigma1(x0), xb ) ); SHA_CORE( x2, wwu_bcast( fd_sha256_K[i+ 2UL] ) );
231 15365316 : x3 = wwu_add( wwu_add( x3, sigma0(x4) ), wwu_add( sigma1(x1), xc ) ); SHA_CORE( x3, wwu_bcast( fd_sha256_K[i+ 3UL] ) );
232 15365316 : x4 = wwu_add( wwu_add( x4, sigma0(x5) ), wwu_add( sigma1(x2), xd ) ); SHA_CORE( x4, wwu_bcast( fd_sha256_K[i+ 4UL] ) );
233 15365316 : x5 = wwu_add( wwu_add( x5, sigma0(x6) ), wwu_add( sigma1(x3), xe ) ); SHA_CORE( x5, wwu_bcast( fd_sha256_K[i+ 5UL] ) );
234 15365316 : x6 = wwu_add( wwu_add( x6, sigma0(x7) ), wwu_add( sigma1(x4), xf ) ); SHA_CORE( x6, wwu_bcast( fd_sha256_K[i+ 6UL] ) );
235 15365316 : x7 = wwu_add( wwu_add( x7, sigma0(x8) ), wwu_add( sigma1(x5), x0 ) ); SHA_CORE( x7, wwu_bcast( fd_sha256_K[i+ 7UL] ) );
236 15365316 : x8 = wwu_add( wwu_add( x8, sigma0(x9) ), wwu_add( sigma1(x6), x1 ) ); SHA_CORE( x8, wwu_bcast( fd_sha256_K[i+ 8UL] ) );
237 15365316 : x9 = wwu_add( wwu_add( x9, sigma0(xa) ), wwu_add( sigma1(x7), x2 ) ); SHA_CORE( x9, wwu_bcast( fd_sha256_K[i+ 9UL] ) );
238 15365316 : xa = wwu_add( wwu_add( xa, sigma0(xb) ), wwu_add( sigma1(x8), x3 ) ); SHA_CORE( xa, wwu_bcast( fd_sha256_K[i+10UL] ) );
239 15365316 : xb = wwu_add( wwu_add( xb, sigma0(xc) ), wwu_add( sigma1(x9), x4 ) ); SHA_CORE( xb, wwu_bcast( fd_sha256_K[i+11UL] ) );
240 15365316 : xc = wwu_add( wwu_add( xc, sigma0(xd) ), wwu_add( sigma1(xa), x5 ) ); SHA_CORE( xc, wwu_bcast( fd_sha256_K[i+12UL] ) );
241 15365316 : xd = wwu_add( wwu_add( xd, sigma0(xe) ), wwu_add( sigma1(xb), x6 ) ); SHA_CORE( xd, wwu_bcast( fd_sha256_K[i+13UL] ) );
242 15365316 : xe = wwu_add( wwu_add( xe, sigma0(xf) ), wwu_add( sigma1(xc), x7 ) ); SHA_CORE( xe, wwu_bcast( fd_sha256_K[i+14UL] ) );
243 15365316 : xf = wwu_add( wwu_add( xf, sigma0(x0) ), wwu_add( sigma1(xd), x8 ) ); SHA_CORE( xf, wwu_bcast( fd_sha256_K[i+15UL] ) );
244 15365316 : }
245 :
246 5121772 : # undef SHA_CORE
247 5121772 : # undef Sigma0
248 5121772 : # undef Sigma1
249 5121772 : # undef sigma0
250 5121772 : # undef sigma1
251 5121772 : # undef Ch
252 5121772 : # undef Maj
253 :
254 : /* Apply the state updates to the active lanes */
255 :
256 5121772 : int active_lane = active_lane_lo | (active_lane_hi<<8);
257 :
258 5121772 : s0 = wwu_add_if( active_lane, s0, a, s0 );
259 5121772 : s1 = wwu_add_if( active_lane, s1, b, s1 );
260 5121772 : s2 = wwu_add_if( active_lane, s2, c, s2 );
261 5121772 : s3 = wwu_add_if( active_lane, s3, d, s3 );
262 5121772 : s4 = wwu_add_if( active_lane, s4, e, s4 );
263 5121772 : s5 = wwu_add_if( active_lane, s5, f, s5 );
264 5121772 : s6 = wwu_add_if( active_lane, s6, g, s6 );
265 5121772 : s7 = wwu_add_if( active_lane, s7, h, s7 );
266 :
267 : /* Advance to the next message segment blocks. In pseudo code,
268 : the below is:
269 :
270 : W += 64; if( block_rem ) block_rem--;
271 :
272 : Since we do not load anything at W(lane) above unless
273 : block_rem(lane) is non-zero, we can omit vector conditional
274 : operations for W(lane) below. */
275 :
276 5121772 : W_lo = wwv_add( W_lo, wwv_64 );
277 5121772 : W_hi = wwv_add( W_hi, wwv_64 );
278 :
279 5121772 : block_rem_lo = wwv_sub_if( active_lane_lo, block_rem_lo, one, block_rem_lo );
280 5121772 : block_rem_hi = wwv_sub_if( active_lane_hi, block_rem_hi, one, block_rem_hi );
281 5121772 : }
282 :
283 : /* Store the results. FIXME: Probably could optimize the transpose
284 : further by taking into account needed stores (and then maybe go
285 : direct into memory ... would need a family of such transposed
286 : stores). */
287 :
288 577644 : wwu_transpose_2x8x8( wwu_bswap(s0), wwu_bswap(s1), wwu_bswap(s2), wwu_bswap(s3),
289 577644 : wwu_bswap(s4), wwu_bswap(s5), wwu_bswap(s6), wwu_bswap(s7), s0,s1,s2,s3,s4,s5,s6,s7 );
290 :
291 577644 : uint * const * batch_hash = (uint * const *)_batch_hash;
292 577644 : switch( batch_cnt ) { /* application dependent prob */
293 467976 : case 16UL: wu_stu( batch_hash[15], _mm512_extracti32x8_epi32( s7, 1 ) ); __attribute__((fallthrough));
294 469956 : case 15UL: wu_stu( batch_hash[14], _mm512_extracti32x8_epi32( s6, 1 ) ); __attribute__((fallthrough));
295 471985 : case 14UL: wu_stu( batch_hash[13], _mm512_extracti32x8_epi32( s5, 1 ) ); __attribute__((fallthrough));
296 474003 : case 13UL: wu_stu( batch_hash[12], _mm512_extracti32x8_epi32( s4, 1 ) ); __attribute__((fallthrough));
297 476044 : case 12UL: wu_stu( batch_hash[11], _mm512_extracti32x8_epi32( s3, 1 ) ); __attribute__((fallthrough));
298 478002 : case 11UL: wu_stu( batch_hash[10], _mm512_extracti32x8_epi32( s2, 1 ) ); __attribute__((fallthrough));
299 479966 : case 10UL: wu_stu( batch_hash[ 9], _mm512_extracti32x8_epi32( s1, 1 ) ); __attribute__((fallthrough));
300 481985 : case 9UL: wu_stu( batch_hash[ 8], _mm512_extracti32x8_epi32( s0, 1 ) ); __attribute__((fallthrough));
301 546505 : case 8UL: wu_stu( batch_hash[ 7], _mm512_extracti32x8_epi32( s7, 0 ) ); __attribute__((fallthrough));
302 548550 : case 7UL: wu_stu( batch_hash[ 6], _mm512_extracti32x8_epi32( s6, 0 ) ); __attribute__((fallthrough));
303 550587 : case 6UL: wu_stu( batch_hash[ 5], _mm512_extracti32x8_epi32( s5, 0 ) ); __attribute__((fallthrough));
304 577644 : case 5UL: wu_stu( batch_hash[ 4], _mm512_extracti32x8_epi32( s4, 0 ) ); __attribute__((fallthrough));
305 577644 : case 4UL: wu_stu( batch_hash[ 3], _mm512_extracti32x8_epi32( s3, 0 ) ); __attribute__((fallthrough));
306 577644 : case 3UL: wu_stu( batch_hash[ 2], _mm512_extracti32x8_epi32( s2, 0 ) ); __attribute__((fallthrough));
307 577644 : case 2UL: wu_stu( batch_hash[ 1], _mm512_extracti32x8_epi32( s1, 0 ) ); __attribute__((fallthrough));
308 577644 : case 1UL: wu_stu( batch_hash[ 0], _mm512_extracti32x8_epi32( s0, 0 ) ); __attribute__((fallthrough));
309 577644 : default: break;
310 577644 : }
311 577644 : }
312 :
313 : #if defined(__znver5__)
314 : #define MIN_ACTIVE (6) /* Zen 5 has high AVX-512 throughput */
315 : #else
316 35 : #define MIN_ACTIVE (8) /* Baseline 1 IPC AVX-512 needs more batching to win against SHA-NI */
317 : #endif
318 :
319 35 : ulong fd_sha256_simd_lane_min( void ) { return MIN_ACTIVE; }
320 1036 : ulong fd_sha256_simd_lane_max( void ) { return 16UL; }
321 30 : ulong fd_sha256_simd_iter_cost_q8( void ) { return 1357UL; } /* 5.3x on Zen 5: 16 lanes at 96.5 M hashes/s vs 31.7 M hashes/s single lane with SHA-NI */
322 :
323 : void
324 : fd_sha256_hash_32_repeated_batch_avx512( uchar const * hash_in,
325 : uchar * hash_out,
326 : ulong cnt,
327 1000 : ulong batch_cnt ) {
328 :
329 : /* Below the SIMD floor, SHA-NI (or the scalar core) wins. */
330 :
331 1000 : if( FD_UNLIKELY( batch_cnt<MIN_ACTIVE ) ) {
332 2061 : for( ulong i=0UL; i<batch_cnt; i++ ) fd_sha256_hash_32_repeated( hash_in+32UL*i, hash_out+32UL*i, cnt );
333 461 : return;
334 461 : }
335 :
336 : /* Gather the batch into a 16 lane scratch buffer. Lanes at and
337 : beyond batch_cnt hash zeros; their results are never stored. */
338 :
339 539 : uchar scratch[ 16UL*32UL ] __attribute__((aligned(64)));
340 539 : memset( scratch, 0, sizeof(scratch) );
341 539 : memcpy( scratch, hash_in, 32UL*batch_cnt );
342 :
343 539 : wwu_t const iv0 = wwu_bcast( FD_SHA256_INITIAL_A );
344 539 : wwu_t const iv1 = wwu_bcast( FD_SHA256_INITIAL_B );
345 539 : wwu_t const iv2 = wwu_bcast( FD_SHA256_INITIAL_C );
346 539 : wwu_t const iv3 = wwu_bcast( FD_SHA256_INITIAL_D );
347 539 : wwu_t const iv4 = wwu_bcast( FD_SHA256_INITIAL_E );
348 539 : wwu_t const iv5 = wwu_bcast( FD_SHA256_INITIAL_F );
349 539 : wwu_t const iv6 = wwu_bcast( FD_SHA256_INITIAL_G );
350 539 : wwu_t const iv7 = wwu_bcast( FD_SHA256_INITIAL_H );
351 :
352 539 : # define SCALAR_ROTR(x,n) ( ((x)>>(n)) | ((x)<<(32-(n))) )
353 539 : # define SCALAR_sigma0(x) ( SCALAR_ROTR((x), 7) ^ SCALAR_ROTR((x),18) ^ ((x)>> 3) )
354 539 : # define SCALAR_sigma1(x) ( SCALAR_ROTR((x),17) ^ SCALAR_ROTR((x),19) ^ ((x)>>10) )
355 539 : uint const PAD8 = 0x80000000U;
356 539 : uint const PADF = 256U;
357 :
358 539 : # define LOAD_PAIR( i ) _mm512_inserti64x4( _mm512_castsi256_si512( wu_ld( (uint const *)(scratch+32UL*(i)) ) ), \
359 539 : wu_ld( (uint const *)(scratch+32UL*((i)+8UL)) ), 1 )
360 4312 : # define STORE_PAIR( i, s ) do { \
361 4312 : wu_st( (uint *)(scratch+32UL*(i)), _mm512_extracti32x8_epi32( (s), 0 ) ); \
362 4312 : wu_st( (uint *)(scratch+32UL*((i)+8UL)), _mm512_extracti32x8_epi32( (s), 1 ) ); \
363 4312 : } while(0)
364 :
365 539 : # define Sigma0(x) _mm512_ternarylogic_epi32( wwu_rol(x,30), wwu_rol(x,19), wwu_rol(x,10), 0x96 )
366 539 : # define Sigma1(x) _mm512_ternarylogic_epi32( wwu_rol(x,26), wwu_rol(x,21), wwu_rol(x, 7), 0x96 )
367 539 : # define sigma0(x) _mm512_ternarylogic_epi32( wwu_rol(x,25), wwu_rol(x,14), wwu_shr(x, 3), 0x96 )
368 539 : # define sigma1(x) _mm512_ternarylogic_epi32( wwu_rol(x,15), wwu_rol(x,13), wwu_shr(x,10), 0x96 )
369 539 : # define Ch(x,y,z) _mm512_ternarylogic_epi32( (x), (y), (z), 0xCA )
370 539 : # define Maj(x,y,z) _mm512_ternarylogic_epi32( (x), (y), (z), 0xE8 )
371 :
372 539 : # define SHA_CORE(wk) \
373 3398144 : T1 = wwu_add( (wk), wwu_add( wwu_add( h, Sigma1(e) ), Ch(e, f, g) ) ); \
374 3398144 : T2 = wwu_add( Sigma0(a), Maj(a, b, c) ); \
375 3398144 : h = g; \
376 3398144 : g = f; \
377 3398144 : f = e; \
378 3398144 : e = wwu_add( d, T1 ); \
379 3398144 : d = c; \
380 3398144 : c = b; \
381 3398144 : b = a; \
382 3398144 : a = wwu_add( T1, T2 )
383 :
384 2973376 : # define ROUND(xi,ki) SHA_CORE( wwu_add( xi, wwu_bcast( fd_sha256_K[ki] ) ) )
385 :
386 539 : # define EXPAND_ROUNDS(i) \
387 106192 : x0 = wwu_add( wwu_add( x0, sigma0(x1) ), wwu_add( sigma1(xe), x9 ) ); ROUND( x0, (i) ); \
388 106192 : x1 = wwu_add( wwu_add( x1, sigma0(x2) ), wwu_add( sigma1(xf), xa ) ); ROUND( x1, (i)+ 1UL ); \
389 106192 : x2 = wwu_add( wwu_add( x2, sigma0(x3) ), wwu_add( sigma1(x0), xb ) ); ROUND( x2, (i)+ 2UL ); \
390 106192 : x3 = wwu_add( wwu_add( x3, sigma0(x4) ), wwu_add( sigma1(x1), xc ) ); ROUND( x3, (i)+ 3UL ); \
391 106192 : x4 = wwu_add( wwu_add( x4, sigma0(x5) ), wwu_add( sigma1(x2), xd ) ); ROUND( x4, (i)+ 4UL ); \
392 106192 : x5 = wwu_add( wwu_add( x5, sigma0(x6) ), wwu_add( sigma1(x3), xe ) ); ROUND( x5, (i)+ 5UL ); \
393 106192 : x6 = wwu_add( wwu_add( x6, sigma0(x7) ), wwu_add( sigma1(x4), xf ) ); ROUND( x6, (i)+ 6UL ); \
394 106192 : x7 = wwu_add( wwu_add( x7, sigma0(x8) ), wwu_add( sigma1(x5), x0 ) ); ROUND( x7, (i)+ 7UL ); \
395 106192 : x8 = wwu_add( wwu_add( x8, sigma0(x9) ), wwu_add( sigma1(x6), x1 ) ); ROUND( x8, (i)+ 8UL ); \
396 106192 : x9 = wwu_add( wwu_add( x9, sigma0(xa) ), wwu_add( sigma1(x7), x2 ) ); ROUND( x9, (i)+ 9UL ); \
397 106192 : xa = wwu_add( wwu_add( xa, sigma0(xb) ), wwu_add( sigma1(x8), x3 ) ); ROUND( xa, (i)+10UL ); \
398 106192 : xb = wwu_add( wwu_add( xb, sigma0(xc) ), wwu_add( sigma1(x9), x4 ) ); ROUND( xb, (i)+11UL ); \
399 106192 : xc = wwu_add( wwu_add( xc, sigma0(xd) ), wwu_add( sigma1(xa), x5 ) ); ROUND( xc, (i)+12UL ); \
400 106192 : xd = wwu_add( wwu_add( xd, sigma0(xe) ), wwu_add( sigma1(xb), x6 ) ); ROUND( xd, (i)+13UL ); \
401 106192 : xe = wwu_add( wwu_add( xe, sigma0(xf) ), wwu_add( sigma1(xc), x7 ) ); ROUND( xe, (i)+14UL ); \
402 106192 : xf = wwu_add( wwu_add( xf, sigma0(x0) ), wwu_add( sigma1(xd), x8 ) ); ROUND( xf, (i)+15UL )
403 :
404 : /* Transpose 16 lanes x 8 words into 8 vectors of 16 lanes each. */
405 :
406 539 : wwu_t s0; wwu_t s1; wwu_t s2; wwu_t s3; wwu_t s4; wwu_t s5; wwu_t s6; wwu_t s7;
407 539 : wwu_transpose_2x8x8( wwu_bswap( LOAD_PAIR( 0 ) ), wwu_bswap( LOAD_PAIR( 1 ) ),
408 539 : wwu_bswap( LOAD_PAIR( 2 ) ), wwu_bswap( LOAD_PAIR( 3 ) ),
409 539 : wwu_bswap( LOAD_PAIR( 4 ) ), wwu_bswap( LOAD_PAIR( 5 ) ),
410 539 : wwu_bswap( LOAD_PAIR( 6 ) ), wwu_bswap( LOAD_PAIR( 7 ) ),
411 539 : s0, s1, s2, s3, s4, s5, s6, s7 );
412 :
413 53635 : for( ulong iter=0UL; iter<cnt; iter++ ) {
414 53096 : wwu_t a = iv0; wwu_t b = iv1; wwu_t c = iv2; wwu_t d = iv3; wwu_t e = iv4; wwu_t f = iv5; wwu_t g = iv6; wwu_t h = iv7;
415 53096 : wwu_t T1;
416 53096 : wwu_t T2;
417 :
418 53096 : wwu_t x0 = s0; wwu_t x1 = s1; wwu_t x2 = s2; wwu_t x3 = s3; wwu_t x4 = s4; wwu_t x5 = s5; wwu_t x6 = s6; wwu_t x7 = s7;
419 53096 : wwu_t x8; wwu_t x9; wwu_t xa; wwu_t xb; wwu_t xc; wwu_t xd; wwu_t xe; wwu_t xf;
420 :
421 53096 : ROUND( x0, 0 ); ROUND( x1, 1 ); ROUND( x2, 2 ); ROUND( x3, 3 );
422 53096 : ROUND( x4, 4 ); ROUND( x5, 5 ); ROUND( x6, 6 ); ROUND( x7, 7 );
423 53096 : SHA_CORE( wwu_bcast( fd_sha256_K[ 8] + PAD8 ) );
424 53096 : SHA_CORE( wwu_bcast( fd_sha256_K[ 9] ) );
425 53096 : SHA_CORE( wwu_bcast( fd_sha256_K[10] ) );
426 53096 : SHA_CORE( wwu_bcast( fd_sha256_K[11] ) );
427 53096 : SHA_CORE( wwu_bcast( fd_sha256_K[12] ) );
428 53096 : SHA_CORE( wwu_bcast( fd_sha256_K[13] ) );
429 53096 : SHA_CORE( wwu_bcast( fd_sha256_K[14] ) );
430 53096 : SHA_CORE( wwu_bcast( fd_sha256_K[15] + PADF ) );
431 :
432 : /* x8=PAD8, x9..xe=0, xf=PADF folded in as constants */
433 :
434 53096 : x0 = wwu_add( x0, sigma0(x1) ); ROUND( x0, 16 );
435 53096 : x1 = wwu_add( wwu_add( x1, sigma0(x2) ), wwu_bcast( SCALAR_sigma1(PADF) ) ); ROUND( x1, 17 );
436 53096 : x2 = wwu_add( wwu_add( x2, sigma0(x3) ), sigma1(x0) ); ROUND( x2, 18 );
437 53096 : x3 = wwu_add( wwu_add( x3, sigma0(x4) ), sigma1(x1) ); ROUND( x3, 19 );
438 53096 : x4 = wwu_add( wwu_add( x4, sigma0(x5) ), sigma1(x2) ); ROUND( x4, 20 );
439 53096 : x5 = wwu_add( wwu_add( x5, sigma0(x6) ), sigma1(x3) ); ROUND( x5, 21 );
440 53096 : x6 = wwu_add( wwu_add( x6, sigma0(x7) ), wwu_add( sigma1(x4), wwu_bcast( PADF ) ) ); ROUND( x6, 22 );
441 53096 : x7 = wwu_add( wwu_add( x7, wwu_bcast( SCALAR_sigma0(PAD8) ) ), wwu_add( sigma1(x5), x0 ) ); ROUND( x7, 23 );
442 53096 : x8 = wwu_add( wwu_bcast( PAD8 ), wwu_add( sigma1(x6), x1 ) ); ROUND( x8, 24 );
443 53096 : x9 = wwu_add( sigma1(x7), x2 ); ROUND( x9, 25 );
444 53096 : xa = wwu_add( sigma1(x8), x3 ); ROUND( xa, 26 );
445 53096 : xb = wwu_add( sigma1(x9), x4 ); ROUND( xb, 27 );
446 53096 : xc = wwu_add( sigma1(xa), x5 ); ROUND( xc, 28 );
447 53096 : xd = wwu_add( sigma1(xb), x6 ); ROUND( xd, 29 );
448 53096 : xe = wwu_add( wwu_bcast( SCALAR_sigma0(PADF) ), wwu_add( sigma1(xc), x7 ) ); ROUND( xe, 30 );
449 53096 : xf = wwu_add( wwu_add( wwu_bcast( PADF ), sigma0(x0) ), wwu_add( sigma1(xd), x8 ) ); ROUND( xf, 31 );
450 :
451 53096 : EXPAND_ROUNDS( 32 );
452 53096 : EXPAND_ROUNDS( 48 );
453 :
454 53096 : s0 = wwu_add( a, iv0 ); s1 = wwu_add( b, iv1 ); s2 = wwu_add( c, iv2 ); s3 = wwu_add( d, iv3 );
455 53096 : s4 = wwu_add( e, iv4 ); s5 = wwu_add( f, iv5 ); s6 = wwu_add( g, iv6 ); s7 = wwu_add( h, iv7 );
456 53096 : }
457 :
458 539 : wwu_t t0; wwu_t t1; wwu_t t2; wwu_t t3; wwu_t t4; wwu_t t5; wwu_t t6; wwu_t t7;
459 539 : wwu_transpose_2x8x8( wwu_bswap(s0), wwu_bswap(s1), wwu_bswap(s2), wwu_bswap(s3),
460 539 : wwu_bswap(s4), wwu_bswap(s5), wwu_bswap(s6), wwu_bswap(s7), t0,t1,t2,t3,t4,t5,t6,t7 );
461 539 : STORE_PAIR( 0, t0 ); STORE_PAIR( 1, t1 ); STORE_PAIR( 2, t2 ); STORE_PAIR( 3, t3 );
462 539 : STORE_PAIR( 4, t4 ); STORE_PAIR( 5, t5 ); STORE_PAIR( 6, t6 ); STORE_PAIR( 7, t7 );
463 :
464 539 : memcpy( hash_out, scratch, 32UL*batch_cnt );
465 :
466 539 : # undef EXPAND_ROUNDS
467 539 : # undef ROUND
468 539 : # undef SHA_CORE
469 539 : # undef Maj
470 539 : # undef Ch
471 539 : # undef sigma1
472 539 : # undef sigma0
473 539 : # undef Sigma1
474 539 : # undef Sigma0
475 539 : # undef SCALAR_sigma1
476 539 : # undef SCALAR_sigma0
477 539 : # undef SCALAR_ROTR
478 539 : # undef STORE_PAIR
479 539 : # undef LOAD_PAIR
480 539 : }
481 :
482 : #undef MIN_ACTIVE
|
