Line data Source code
1 : #include "fd_sha256.h"
2 : #include "fd_sha256_constants.h"
3 :
4 : #if FD_HAS_SHANI
5 : /* For the optimized repeated hash */
6 : #include "../../util/simd/fd_sse.h"
7 : #endif
8 :
9 : ulong
10 1121784 : fd_sha256_align( void ) {
11 1121784 : return FD_SHA256_ALIGN;
12 1121784 : }
13 :
14 : ulong
15 560880 : fd_sha256_footprint( void ) {
16 560880 : return FD_SHA256_FOOTPRINT;
17 560880 : }
18 :
19 : void *
20 560883 : fd_sha256_new( void * shmem ) {
21 560883 : fd_sha256_t * sha = (fd_sha256_t *)shmem;
22 :
23 560883 : if( FD_UNLIKELY( !shmem ) ) {
24 3 : FD_LOG_WARNING(( "NULL shmem" ));
25 3 : return NULL;
26 3 : }
27 :
28 560880 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)shmem, fd_sha256_align() ) ) ) {
29 3 : FD_LOG_WARNING(( "misaligned shmem" ));
30 3 : return NULL;
31 3 : }
32 :
33 560877 : ulong footprint = fd_sha256_footprint();
34 :
35 560877 : fd_memset( sha, 0, footprint );
36 :
37 560877 : FD_COMPILER_MFENCE();
38 560877 : FD_VOLATILE( sha->magic ) = FD_SHA256_MAGIC;
39 560877 : FD_COMPILER_MFENCE();
40 :
41 560877 : return (void *)sha;
42 560880 : }
43 :
44 : fd_sha256_t *
45 560883 : fd_sha256_join( void * shsha ) {
46 :
47 560883 : if( FD_UNLIKELY( !shsha ) ) {
48 3 : FD_LOG_WARNING(( "NULL shsha" ));
49 3 : return NULL;
50 3 : }
51 :
52 560880 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)shsha, fd_sha256_align() ) ) ) {
53 3 : FD_LOG_WARNING(( "misaligned shsha" ));
54 3 : return NULL;
55 3 : }
56 :
57 560877 : fd_sha256_t * sha = (fd_sha256_t *)shsha;
58 :
59 560877 : if( FD_UNLIKELY( sha->magic!=FD_SHA256_MAGIC ) ) {
60 0 : FD_LOG_WARNING(( "bad magic" ));
61 0 : return NULL;
62 0 : }
63 :
64 560877 : return sha;
65 560877 : }
66 :
67 : void *
68 21 : fd_sha256_leave( fd_sha256_t * sha ) {
69 :
70 21 : if( FD_UNLIKELY( !sha ) ) {
71 3 : FD_LOG_WARNING(( "NULL sha" ));
72 3 : return NULL;
73 3 : }
74 :
75 18 : return (void *)sha;
76 21 : }
77 :
78 : void *
79 24 : fd_sha256_delete( void * shsha ) {
80 :
81 24 : if( FD_UNLIKELY( !shsha ) ) {
82 3 : FD_LOG_WARNING(( "NULL shsha" ));
83 3 : return NULL;
84 3 : }
85 :
86 21 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)shsha, fd_sha256_align() ) ) ) {
87 3 : FD_LOG_WARNING(( "misaligned shsha" ));
88 3 : return NULL;
89 3 : }
90 :
91 18 : fd_sha256_t * sha = (fd_sha256_t *)shsha;
92 :
93 18 : if( FD_UNLIKELY( sha->magic!=FD_SHA256_MAGIC ) ) {
94 0 : FD_LOG_WARNING(( "bad magic" ));
95 0 : return NULL;
96 0 : }
97 :
98 18 : FD_COMPILER_MFENCE();
99 18 : FD_VOLATILE( sha->magic ) = 0UL;
100 18 : FD_COMPILER_MFENCE();
101 :
102 18 : return (void *)sha;
103 18 : }
104 :
105 : #ifndef FD_SHA256_CORE_IMPL
106 : #if FD_HAS_ARM_SHA256
107 : #define FD_SHA256_CORE_IMPL 2
108 : #elif FD_HAS_SHANI
109 : #define FD_SHA256_CORE_IMPL 1
110 : #elif FD_HAS_RISCV_SHA256
111 : #define FD_SHA256_CORE_IMPL 3
112 : #else
113 : #define FD_SHA256_CORE_IMPL 0
114 : #endif
115 : #endif
116 :
117 : #if FD_SHA256_CORE_IMPL==0
118 :
119 : /* The implementation below was derived from OpenSSL's SHA-256
120 : implementation (Apache-2.0 licensed). See in particular:
121 :
122 : https://github.com/openssl/openssl/blob/master/crypto/sha/sha256.c
123 :
124 : (link valid circa 2022-Dec). It has been made more strict with more
125 : extensive implementation documentation, has been simplified and has
126 : been streamlined specifically for use inside Firedancer base machine
127 : model (no machine specific capabilities required).
128 :
129 : In particular, fd_sha256_core_ref is based on OpenSSL's
130 : OPENSSL_SMALL_FOOTPRINT SHA-256 implementation (Apache licensed).
131 : This should work anywhere but it is not the highest performance
132 : implementation possible.
133 :
134 : It is also straightforward to replace these implementations with HPC
135 : implementations that target specific machine capabilities without
136 : requiring any changes to caller code. */
137 :
138 : static void
139 : fd_sha256_core_ref( uint * state,
140 : uchar const * block,
141 23291929 : ulong block_cnt ) {
142 :
143 :
144 14882770368 : # define ROTATE fd_uint_rotate_left
145 1653641152 : # define Sigma0(x) (ROTATE((x),30) ^ ROTATE((x),19) ^ ROTATE((x),10))
146 1653641152 : # define Sigma1(x) (ROTATE((x),26) ^ ROTATE((x),21) ^ ROTATE((x),7))
147 1240230864 : # define sigma0(x) (ROTATE((x),25) ^ ROTATE((x),14) ^ ((x)>>3))
148 1240230864 : # define sigma1(x) (ROTATE((x),15) ^ ROTATE((x),13) ^ ((x)>>10))
149 1653641152 : # define Ch(x,y,z) (((x) & (y)) ^ ((~(x)) & (z)))
150 1653641152 : # define Maj(x,y,z) (((x) & (y)) ^ ((x) & (z)) ^ ((y) & (z)))
151 :
152 23291929 : uchar const * W = block;
153 25838143 : do {
154 25838143 : uint a = state[0];
155 25838143 : uint b = state[1];
156 25838143 : uint c = state[2];
157 25838143 : uint d = state[3];
158 25838143 : uint e = state[4];
159 25838143 : uint f = state[5];
160 25838143 : uint g = state[6];
161 25838143 : uint h = state[7];
162 :
163 25838143 : uint X[16];
164 :
165 25838143 : ulong i;
166 439248431 : for( i=0UL; i<16UL; i++ ) {
167 413410288 : X[i] = fd_uint_bswap( FD_LOAD( uint, W + i*sizeof(uint) ) );
168 413410288 : uint T1 = X[i] + h + Sigma1(e) + Ch(e, f, g) + fd_sha256_K[i];
169 413410288 : uint T2 = Sigma0(a) + Maj(a, b, c);
170 413410288 : h = g;
171 413410288 : g = f;
172 413410288 : f = e;
173 413410288 : e = d + T1;
174 413410288 : d = c;
175 413410288 : c = b;
176 413410288 : b = a;
177 413410288 : a = T1 + T2;
178 413410288 : }
179 1266069007 : for( ; i<64UL; i++ ) {
180 1240230864 : uint s0 = X[(i + 1UL) & 0x0fUL];
181 1240230864 : uint s1 = X[(i + 14UL) & 0x0fUL];
182 1240230864 : s0 = sigma0(s0);
183 1240230864 : s1 = sigma1(s1);
184 1240230864 : X[i & 0xfUL] += s0 + s1 + X[(i + 9UL) & 0xfUL];
185 1240230864 : uint T1 = X[i & 0xfUL ] + h + Sigma1(e) + Ch(e, f, g) + fd_sha256_K[i];
186 1240230864 : uint T2 = Sigma0(a) + Maj(a, b, c);
187 1240230864 : h = g;
188 1240230864 : g = f;
189 1240230864 : f = e;
190 1240230864 : e = d + T1;
191 1240230864 : d = c;
192 1240230864 : c = b;
193 1240230864 : b = a;
194 1240230864 : a = T1 + T2;
195 1240230864 : }
196 :
197 25838143 : state[0] += a;
198 25838143 : state[1] += b;
199 25838143 : state[2] += c;
200 25838143 : state[3] += d;
201 25838143 : state[4] += e;
202 25838143 : state[5] += f;
203 25838143 : state[6] += g;
204 25838143 : state[7] += h;
205 :
206 25838143 : W += 16UL*sizeof(uint);
207 25838143 : } while( --block_cnt );
208 :
209 23291929 : # undef ROTATE
210 23291929 : # undef Sigma0
211 23291929 : # undef Sigma1
212 23291929 : # undef sigma0
213 23291929 : # undef sigma1
214 23291929 : # undef Ch
215 23291929 : # undef Maj
216 :
217 23291929 : }
218 :
219 23291929 : #define fd_sha256_core fd_sha256_core_ref
220 :
221 : #elif FD_SHA256_CORE_IMPL==1
222 :
223 : /* _mm_sha256rnds2_epu32 does two rounds, one from the first uint in
224 : wk and one from the second. Since wk stores four rounds worth of
225 : message schedule values, it makes sense for the macro to do four
226 : rounds at a time. We need to permute wk in between so that the
227 : second call to the intrinsic will use the other values. */
228 250407568 : #define FOUR_ROUNDS_STATE( stateFEBA, stateHGDC, wk ) do { \
229 250407568 : vu_t __wk = (wk); \
230 250407568 : vu_t temp_state = stateFEBA; \
231 250407568 : stateFEBA = _mm_sha256rnds2_epu32( stateHGDC, stateFEBA, __wk ); \
232 250407568 : stateHGDC = temp_state; \
233 250407568 : \
234 250407568 : temp_state = stateFEBA; \
235 250407568 : stateFEBA = _mm_sha256rnds2_epu32( stateHGDC, stateFEBA, vu_permute( __wk, 2,3,0,1 ) );\
236 250407568 : stateHGDC = temp_state; \
237 250407568 : } while( 0 )
238 :
239 247881296 : #define FOUR_ROUNDS( wk ) FOUR_ROUNDS_STATE( stateFEBA, stateHGDC, wk )
240 :
241 :
242 : /* For completeness, here's the documentation for _mm_sha256msg1_epu32
243 : and _mm_sha256msg2_epu32 in a slightly reformatted way, where all
244 : values are uints, and "-" indicates a don't-care value:
245 :
246 : _mm_sha256msg1_epu32( (w[j ], w[j+1], w[j+1], w[j+3]),
247 : (w[j+4], -, -, - ) )
248 : = ( w[j ]+s0( w[j+1] ), w[j+1]+s0( w[j+2] ),
249 : w[j+2]+s0( w[j+3] ), w[j+3]+s0( w[j+4] ) ).
250 :
251 :
252 : _mm_sha256msg2_epu32( (v[j ], v[j+1], v[j+1], v[j+3]),
253 : (-, -, w[j-2], w[j-1]) )
254 : sets w[j ] = v[j ] + s1( w[j-2] ) and
255 : w[j+1] = v[j+1] + s1( w[j-1] ), and then returns
256 :
257 : ( v[j ]+s1( w[j-2] ), v[j+1]+s1( w[j-1] ),
258 : v[j+2]+s1( w[j ] ), v[j+3]+s1( w[j+1] ) ) */
259 :
260 :
261 : /* w[i] for i>= 16 is w[i-16] + s0(w[i-15]) + w[i-7] + s1(w[i-2])
262 : Since our vector size is 4 uints, it's only s1 that is a little
263 : problematic, because it references items in the same vector.
264 : Thankfully, the msg2 intrinsic takes care of the complexity, but we
265 : need to execute it last.
266 :
267 : We get w[i-16] and s0(s[i-15]) using the msg1 intrinsic, setting j =
268 : i-16. For example, to compute w1013, we pass in w0003 and w0407.
269 : Then we can get w[i-7] by using the alignr instruction on
270 : (w[i-8], w[i-7], w[i-6], w[i-5]) and (w[i-4], w[i-3], w[i-2], w[i-1])
271 : to concatenate them and shift by one uint. Continuing with the
272 : example of w1013, we need w080b and w0c0f. We then put
273 : v[i] = w[i-16] + s0(w[i-15]) + w[i-7],
274 : and invoke the msg2 intrinsic with j=i, which gives w[i], as desired.
275 : Each invocation of NEXT_W computes 4 values of w. */
276 :
277 187805676 : #define NEXT_W( w_minus_16, w_minus_12, w_minus_8, w_minus_4 ) (__extension__({ \
278 187805676 : vu_t __w_i_16_s0_i_15 = _mm_sha256msg1_epu32( w_minus_16, w_minus_12 ); \
279 187805676 : vu_t __w_i_7 = _mm_alignr_epi8( w_minus_4, w_minus_8, 4 ); \
280 187805676 : _mm_sha256msg2_epu32( vu_add( __w_i_7, __w_i_16_s0_i_15 ), w_minus_4 ); \
281 187805676 : }))
282 :
283 : /* Zen 5's sha256rnds2 has an RTP of 2, while Zen 4's has an RTP of 1. We can
284 : win some performance by moving the schedule updates earlier in the loop,
285 : which improves the speed by around 1M hashes/s for the repeated hashing. */
286 : #ifdef __znver5__
287 : #define FULL_ROUNDS() do { \
288 : vu_t w1013 = NEXT_W( w0003, w0407, w080b, w0c0f ); FOUR_ROUNDS( vu_add( w0003, vu_ld( fd_sha256_K+ 0UL ) ) ); \
289 : vu_t w1417 = NEXT_W( w0407, w080b, w0c0f, w1013 ); FOUR_ROUNDS( vu_add( w0407, vu_ld( fd_sha256_K+ 4UL ) ) ); \
290 : vu_t w181b = NEXT_W( w080b, w0c0f, w1013, w1417 ); FOUR_ROUNDS( vu_add( w080b, vu_ld( fd_sha256_K+ 8UL ) ) ); \
291 : vu_t w1c1f = NEXT_W( w0c0f, w1013, w1417, w181b ); FOUR_ROUNDS( vu_add( w0c0f, vu_ld( fd_sha256_K+12UL ) ) ); \
292 : vu_t w2023 = NEXT_W( w1013, w1417, w181b, w1c1f ); FOUR_ROUNDS( vu_add( w1013, vu_ld( fd_sha256_K+16UL ) ) ); \
293 : vu_t w2427 = NEXT_W( w1417, w181b, w1c1f, w2023 ); FOUR_ROUNDS( vu_add( w1417, vu_ld( fd_sha256_K+20UL ) ) ); \
294 : vu_t w282b = NEXT_W( w181b, w1c1f, w2023, w2427 ); FOUR_ROUNDS( vu_add( w181b, vu_ld( fd_sha256_K+24UL ) ) ); \
295 : vu_t w2c2f = NEXT_W( w1c1f, w2023, w2427, w282b ); FOUR_ROUNDS( vu_add( w1c1f, vu_ld( fd_sha256_K+28UL ) ) ); \
296 : vu_t w3033 = NEXT_W( w2023, w2427, w282b, w2c2f ); FOUR_ROUNDS( vu_add( w2023, vu_ld( fd_sha256_K+32UL ) ) ); \
297 : vu_t w3437 = NEXT_W( w2427, w282b, w2c2f, w3033 ); FOUR_ROUNDS( vu_add( w2427, vu_ld( fd_sha256_K+36UL ) ) ); \
298 : vu_t w383b = NEXT_W( w282b, w2c2f, w3033, w3437 ); FOUR_ROUNDS( vu_add( w282b, vu_ld( fd_sha256_K+40UL ) ) ); \
299 : vu_t w3c3f = NEXT_W( w2c2f, w3033, w3437, w383b ); FOUR_ROUNDS( vu_add( w2c2f, vu_ld( fd_sha256_K+44UL ) ) ); \
300 : /* */ FOUR_ROUNDS( vu_add( w3033, vu_ld( fd_sha256_K+48UL ) ) ); \
301 : /* */ FOUR_ROUNDS( vu_add( w3437, vu_ld( fd_sha256_K+52UL ) ) ); \
302 : /* */ FOUR_ROUNDS( vu_add( w383b, vu_ld( fd_sha256_K+56UL ) ) ); \
303 : /* */ FOUR_ROUNDS( vu_add( w3c3f, vu_ld( fd_sha256_K+60UL ) ) ); \
304 : } while ( 0 )
305 : #else
306 15492581 : #define FULL_ROUNDS() do { \
307 15492581 : /* */ FOUR_ROUNDS( vu_add( w0003, vu_ld( fd_sha256_K+ 0UL ) ) ); \
308 15492581 : /* */ FOUR_ROUNDS( vu_add( w0407, vu_ld( fd_sha256_K+ 4UL ) ) ); \
309 15492581 : /* */ FOUR_ROUNDS( vu_add( w080b, vu_ld( fd_sha256_K+ 8UL ) ) ); \
310 15492581 : /* */ FOUR_ROUNDS( vu_add( w0c0f, vu_ld( fd_sha256_K+12UL ) ) ); \
311 15492581 : vu_t w1013 = NEXT_W( w0003, w0407, w080b, w0c0f ); FOUR_ROUNDS( vu_add( w1013, vu_ld( fd_sha256_K+16UL ) ) ); \
312 15492581 : vu_t w1417 = NEXT_W( w0407, w080b, w0c0f, w1013 ); FOUR_ROUNDS( vu_add( w1417, vu_ld( fd_sha256_K+20UL ) ) ); \
313 15492581 : vu_t w181b = NEXT_W( w080b, w0c0f, w1013, w1417 ); FOUR_ROUNDS( vu_add( w181b, vu_ld( fd_sha256_K+24UL ) ) ); \
314 15492581 : vu_t w1c1f = NEXT_W( w0c0f, w1013, w1417, w181b ); FOUR_ROUNDS( vu_add( w1c1f, vu_ld( fd_sha256_K+28UL ) ) ); \
315 15492581 : vu_t w2023 = NEXT_W( w1013, w1417, w181b, w1c1f ); FOUR_ROUNDS( vu_add( w2023, vu_ld( fd_sha256_K+32UL ) ) ); \
316 15492581 : vu_t w2427 = NEXT_W( w1417, w181b, w1c1f, w2023 ); FOUR_ROUNDS( vu_add( w2427, vu_ld( fd_sha256_K+36UL ) ) ); \
317 15492581 : vu_t w282b = NEXT_W( w181b, w1c1f, w2023, w2427 ); FOUR_ROUNDS( vu_add( w282b, vu_ld( fd_sha256_K+40UL ) ) ); \
318 15492581 : vu_t w2c2f = NEXT_W( w1c1f, w2023, w2427, w282b ); FOUR_ROUNDS( vu_add( w2c2f, vu_ld( fd_sha256_K+44UL ) ) ); \
319 15492581 : vu_t w3033 = NEXT_W( w2023, w2427, w282b, w2c2f ); FOUR_ROUNDS( vu_add( w3033, vu_ld( fd_sha256_K+48UL ) ) ); \
320 15492581 : vu_t w3437 = NEXT_W( w2427, w282b, w2c2f, w3033 ); FOUR_ROUNDS( vu_add( w3437, vu_ld( fd_sha256_K+52UL ) ) ); \
321 15492581 : vu_t w383b = NEXT_W( w282b, w2c2f, w3033, w3437 ); FOUR_ROUNDS( vu_add( w383b, vu_ld( fd_sha256_K+56UL ) ) ); \
322 15492581 : vu_t w3c3f = NEXT_W( w2c2f, w3033, w3437, w383b ); FOUR_ROUNDS( vu_add( w3c3f, vu_ld( fd_sha256_K+60UL ) ) ); \
323 15492581 : } while ( 0 )
324 : #endif
325 :
326 :
327 : void
328 : fd_sha256_core_shaext( uint * state, /* 64-byte aligned, 8 entries */
329 : uchar const * block, /* ideally 128-byte aligned (but not required), 64*block_cnt in size */
330 9755588 : ulong block_cnt ) {/* positive */
331 9755588 : vu_t stateABCD = vu_ld( state );
332 9755588 : vu_t stateEFGH = vu_ld( state+4UL );
333 :
334 9755588 : vu_t baseFEBA = vu_permute2( stateEFGH, stateABCD, 1, 0, 1, 0 );
335 9755588 : vu_t baseHGDC = vu_permute2( stateEFGH, stateABCD, 3, 2, 3, 2 );
336 :
337 21789666 : for( ulong b=0UL; b<block_cnt; b++ ) {
338 12034078 : vu_t stateFEBA = baseFEBA;
339 12034078 : vu_t stateHGDC = baseHGDC;
340 :
341 12034078 : vu_t w0003 = vu_bswap( vu_ldu( block+64UL*b ) );
342 12034078 : vu_t w0407 = vu_bswap( vu_ldu( block+64UL*b+16UL ) );
343 12034078 : vu_t w080b = vu_bswap( vu_ldu( block+64UL*b+32UL ) );
344 12034078 : vu_t w0c0f = vu_bswap( vu_ldu( block+64UL*b+48UL ) );
345 :
346 12034078 : FULL_ROUNDS();
347 :
348 12034078 : baseFEBA = vu_add( baseFEBA, stateFEBA );
349 12034078 : baseHGDC = vu_add( baseHGDC, stateHGDC );
350 :
351 12034078 : }
352 :
353 9755588 : stateABCD = vu_permute2( baseFEBA, baseHGDC, 3, 2, 3, 2 );
354 9755588 : stateEFGH = vu_permute2( baseFEBA, baseHGDC, 1, 0, 1, 0 );
355 9755588 : vu_st( state, stateABCD );
356 9755588 : vu_st( state+4UL, stateEFGH );
357 9755588 : }
358 :
359 9755588 : #define fd_sha256_core fd_sha256_core_shaext
360 :
361 : #elif FD_SHA256_CORE_IMPL==2
362 :
363 : void
364 : fd_sha256_core_arm( uint * state,
365 : uchar const * block,
366 : ulong block_cnt );
367 :
368 : #define fd_sha256_core fd_sha256_core_arm
369 :
370 : #elif FD_SHA256_CORE_IMPL==3
371 :
372 : void
373 : fd_sha256_core_riscv( uint * state,
374 : uchar const * block,
375 : ulong block_cnt );
376 :
377 : #define fd_sha256_core fd_sha256_core_riscv
378 :
379 : #else
380 : #error "Unsupported FD_SHA256_CORE_IMPL"
381 : #endif
382 :
383 : fd_sha256_t *
384 1173378 : fd_sha256_init( fd_sha256_t * sha ) {
385 1173378 : sha->state[0] = FD_SHA256_INITIAL_A;
386 1173378 : sha->state[1] = FD_SHA256_INITIAL_B;
387 1173378 : sha->state[2] = FD_SHA256_INITIAL_C;
388 1173378 : sha->state[3] = FD_SHA256_INITIAL_D;
389 1173378 : sha->state[4] = FD_SHA256_INITIAL_E;
390 1173378 : sha->state[5] = FD_SHA256_INITIAL_F;
391 1173378 : sha->state[6] = FD_SHA256_INITIAL_G;
392 1173378 : sha->state[7] = FD_SHA256_INITIAL_H;
393 1173378 : sha->buf_used = 0UL;
394 1173378 : sha->bit_cnt = 0UL;
395 1173378 : return sha;
396 1173378 : }
397 :
398 : fd_sha256_t *
399 : fd_sha256_append( fd_sha256_t * sha,
400 : void const * _data,
401 2455107 : ulong sz ) {
402 :
403 : /* If no data to append, we are done */
404 :
405 2455107 : if( FD_UNLIKELY( !sz ) ) return sha; /* optimize for non-trivial append */
406 :
407 : /* Unpack inputs */
408 :
409 2454861 : uint * state = sha->state;
410 2454861 : uchar * buf = sha->buf;
411 2454861 : ulong buf_used = sha->buf_used;
412 2454861 : ulong bit_cnt = sha->bit_cnt;
413 :
414 2454861 : uchar const * data = (uchar const *)_data;
415 :
416 : /* Update bit_cnt */
417 : /* FIXME: could accumulate bytes here and do bit conversion in append */
418 : /* FIXME: Overflow handling if more than 2^64 bits (unlikely) */
419 :
420 2454861 : sha->bit_cnt = bit_cnt + (sz<<3);
421 :
422 : /* Handle buffered bytes from previous appends */
423 :
424 2454861 : if( FD_UNLIKELY( buf_used ) ) { /* optimized for well aligned use of append */
425 :
426 : /* If the append isn't large enough to complete the current block,
427 : buffer these bytes too and return */
428 :
429 165920 : ulong buf_rem = FD_SHA256_PRIVATE_BUF_MAX - buf_used; /* In (0,FD_SHA256_PRIVATE_BUF_MAX) */
430 165920 : if( FD_UNLIKELY( sz < buf_rem ) ) { /* optimize for large append */
431 49454 : fd_memcpy( buf + buf_used, data, sz );
432 49454 : sha->buf_used = buf_used + sz;
433 49454 : return sha;
434 49454 : }
435 :
436 : /* Otherwise, buffer enough leading bytes of data to complete the
437 : block, update the hash and then continue processing any remaining
438 : bytes of data. */
439 :
440 116466 : fd_memcpy( buf + buf_used, data, buf_rem );
441 116466 : data += buf_rem;
442 116466 : sz -= buf_rem;
443 :
444 116466 : fd_sha256_core( state, buf, 1UL );
445 116466 : sha->buf_used = 0UL;
446 116466 : }
447 :
448 : /* Append the bulk of the data */
449 :
450 2405407 : ulong block_cnt = sz >> FD_SHA256_PRIVATE_LG_BUF_MAX;
451 2405407 : if( FD_LIKELY( block_cnt ) ) fd_sha256_core( state, data, block_cnt ); /* optimized for large append */
452 :
453 : /* Buffer any leftover bytes */
454 :
455 2405407 : buf_used = sz & (FD_SHA256_PRIVATE_BUF_MAX-1UL); /* In [0,FD_SHA256_PRIVATE_BUF_MAX) */
456 2405407 : if( FD_UNLIKELY( buf_used ) ) { /* optimized for well aligned use of append */
457 1289552 : fd_memcpy( buf, data + (block_cnt << FD_SHA256_PRIVATE_LG_BUF_MAX), buf_used );
458 1289552 : sha->buf_used = buf_used; /* In (0,FD_SHA256_PRIVATE_BUF_MAX) */
459 1289552 : }
460 :
461 2405407 : return sha;
462 2454861 : }
463 :
464 : void *
465 : fd_sha256_fini( fd_sha256_t * sha,
466 1239192 : void * _hash ) {
467 :
468 : /* Unpack inputs */
469 :
470 1239192 : uint * state = sha->state;
471 1239192 : uchar * buf = sha->buf;
472 1239192 : ulong buf_used = sha->buf_used; /* In [0,FD_SHA256_PRIVATE_BUF_MAX) */
473 1239192 : ulong bit_cnt = sha->bit_cnt;
474 :
475 : /* Append the terminating message byte */
476 :
477 1239192 : buf[ buf_used ] = (uchar)0x80;
478 1239192 : buf_used++;
479 :
480 : /* If there isn't enough room to save the message length in bits at
481 : the end of the in progress block, clear the rest of the in progress
482 : block, update the hash and start a new block. */
483 :
484 1239192 : if( FD_UNLIKELY( buf_used > (FD_SHA256_PRIVATE_BUF_MAX-8UL) ) ) { /* optimize for well aligned use of append */
485 16897 : fd_memset( buf + buf_used, 0, FD_SHA256_PRIVATE_BUF_MAX-buf_used );
486 16897 : fd_sha256_core( state, buf, 1UL );
487 16897 : buf_used = 0UL;
488 16897 : }
489 :
490 : /* Clear in progress block up to last 64-bits, append the message
491 : size in bytes in the last 64-bits of the in progress block and
492 : update the hash to finalize it. */
493 :
494 1239192 : fd_memset( buf + buf_used, 0, FD_SHA256_PRIVATE_BUF_MAX-8UL-buf_used );
495 1239192 : FD_STORE( ulong, buf+FD_SHA256_PRIVATE_BUF_MAX-8UL, fd_ulong_bswap( bit_cnt ) );
496 1239192 : fd_sha256_core( state, buf, 1UL );
497 :
498 : /* Unpack the result into md (annoying bswaps here) */
499 :
500 1239192 : state[0] = fd_uint_bswap( state[0] );
501 1239192 : state[1] = fd_uint_bswap( state[1] );
502 1239192 : state[2] = fd_uint_bswap( state[2] );
503 1239192 : state[3] = fd_uint_bswap( state[3] );
504 1239192 : state[4] = fd_uint_bswap( state[4] );
505 1239192 : state[5] = fd_uint_bswap( state[5] );
506 1239192 : state[6] = fd_uint_bswap( state[6] );
507 1239192 : state[7] = fd_uint_bswap( state[7] );
508 1239192 : return memcpy( _hash, state, 32 );
509 1239192 : }
510 :
511 : void *
512 : fd_sha256_hash( void const * _data,
513 : ulong sz,
514 19912784 : void * _hash ) {
515 19912784 : uchar const * data = (uchar const *)_data;
516 :
517 : /* This is just the above streamlined to eliminate all the overheads
518 : to support incremental hashing. */
519 :
520 19912784 : uchar buf[ FD_SHA256_PRIVATE_BUF_MAX ] __attribute__((aligned(128)));
521 19912784 : uint state[8] __attribute__((aligned(32)));
522 :
523 19912784 : state[0] = FD_SHA256_INITIAL_A;
524 19912784 : state[1] = FD_SHA256_INITIAL_B;
525 19912784 : state[2] = FD_SHA256_INITIAL_C;
526 19912784 : state[3] = FD_SHA256_INITIAL_D;
527 19912784 : state[4] = FD_SHA256_INITIAL_E;
528 19912784 : state[5] = FD_SHA256_INITIAL_F;
529 19912784 : state[6] = FD_SHA256_INITIAL_G;
530 19912784 : state[7] = FD_SHA256_INITIAL_H;
531 :
532 19912784 : ulong block_cnt = sz >> FD_SHA256_PRIVATE_LG_BUF_MAX;
533 19912784 : if( FD_LIKELY( block_cnt ) ) fd_sha256_core( state, data, block_cnt );
534 :
535 19912784 : ulong buf_used = sz & (FD_SHA256_PRIVATE_BUF_MAX-1UL);
536 19912784 : if( FD_UNLIKELY( buf_used ) ) fd_memcpy( buf, data + (block_cnt << FD_SHA256_PRIVATE_LG_BUF_MAX), buf_used );
537 19912784 : buf[ buf_used ] = (uchar)0x80;
538 19912784 : buf_used++;
539 :
540 19912784 : if( FD_UNLIKELY( buf_used > (FD_SHA256_PRIVATE_BUF_MAX-8UL) ) ) {
541 164258 : fd_memset( buf + buf_used, 0, FD_SHA256_PRIVATE_BUF_MAX-buf_used );
542 164258 : fd_sha256_core( state, buf, 1UL );
543 164258 : buf_used = 0UL;
544 164258 : }
545 :
546 19912784 : ulong bit_cnt = sz << 3;
547 19912784 : fd_memset( buf + buf_used, 0, FD_SHA256_PRIVATE_BUF_MAX-8UL-buf_used );
548 19912784 : FD_STORE( ulong, buf+FD_SHA256_PRIVATE_BUF_MAX-8UL, fd_ulong_bswap( bit_cnt ) );
549 19912784 : fd_sha256_core( state, buf, 1UL );
550 :
551 19912784 : state[0] = fd_uint_bswap( state[0] );
552 19912784 : state[1] = fd_uint_bswap( state[1] );
553 19912784 : state[2] = fd_uint_bswap( state[2] );
554 19912784 : state[3] = fd_uint_bswap( state[3] );
555 19912784 : state[4] = fd_uint_bswap( state[4] );
556 19912784 : state[5] = fd_uint_bswap( state[5] );
557 19912784 : state[6] = fd_uint_bswap( state[6] );
558 19912784 : state[7] = fd_uint_bswap( state[7] );
559 19912784 : return memcpy( _hash, state, 32 );
560 19912784 : }
561 :
562 :
563 :
564 : #if FD_SHA256_CORE_IMPL==2
565 : void
566 : fd_sha256_hash_32_repeated_arm( uchar const * data,
567 : uchar * hash,
568 : ulong cnt );
569 : #elif FD_SHA256_CORE_IMPL==3
570 : void
571 : fd_sha256_hash_32_repeated_riscv( uchar const * data,
572 : uchar * hash,
573 : ulong cnt );
574 : #endif
575 :
576 : void *
577 : fd_sha256_hash_32_repeated( void const * _data,
578 : void * _hash,
579 11111 : ulong cnt ) {
580 : #if FD_SHA256_CORE_IMPL==2
581 : fd_sha256_hash_32_repeated_arm( _data, _hash, cnt );
582 : return _hash;
583 : #elif FD_SHA256_CORE_IMPL==3
584 : fd_sha256_hash_32_repeated_riscv( _data, _hash, cnt );
585 : return _hash;
586 : #elif FD_SHA256_CORE_IMPL==1
587 :
588 9025 : uchar const * data = (uchar const *)_data;
589 9025 : uchar * hash = (uchar *)_hash;
590 :
591 9025 : vu_t w0003 = vu_bswap( vu_ldu( data ) );
592 9025 : vu_t w0407 = vu_bswap( vu_ldu( data+16UL ) );
593 9025 : vb_t const w080b = vb( 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00,
594 9025 : 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 );
595 9025 : vb_t const w0c0f = vb( 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
596 9025 : 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00 ); /* 32 bytes */
597 :
598 9025 : vu_t const initialFEBA = vu( FD_SHA256_INITIAL_F, FD_SHA256_INITIAL_E, FD_SHA256_INITIAL_B, FD_SHA256_INITIAL_A );
599 9025 : vu_t const initialHGDC = vu( FD_SHA256_INITIAL_H, FD_SHA256_INITIAL_G, FD_SHA256_INITIAL_D, FD_SHA256_INITIAL_C );
600 :
601 3467528 : for( ulong iter=0UL; iter<cnt; iter++ ) {
602 3458503 : vu_t stateFEBA = initialFEBA;
603 3458503 : vu_t stateHGDC = initialHGDC;
604 :
605 3458503 : FULL_ROUNDS();
606 :
607 3458503 : stateFEBA = vu_add( stateFEBA, initialFEBA );
608 3458503 : stateHGDC = vu_add( stateHGDC, initialHGDC );
609 :
610 3458503 : vu_t stateABCD = vu_permute2( stateFEBA, stateHGDC, 3, 2, 3, 2 );
611 3458503 : vu_t stateEFGH = vu_permute2( stateFEBA, stateHGDC, 1, 0, 1, 0 );
612 :
613 3458503 : w0003 = stateABCD;
614 3458503 : w0407 = stateEFGH;
615 3458503 : }
616 9025 : vu_stu( hash, vu_bswap( w0003 ) );
617 9025 : vu_stu( hash+16UL, vu_bswap( w0407 ) );
618 :
619 : #else
620 :
621 2086 : uchar const * data = (uchar const *)_data;
622 2086 : uchar * hash = (uchar *)_hash;
623 :
624 2086 : uchar buf[ FD_SHA256_PRIVATE_BUF_MAX ] __attribute__((aligned(128)));
625 :
626 : /* Prepare padding once */
627 2086 : ulong buf_used = 32UL;
628 2086 : memcpy( buf, data, 32UL );
629 2086 : buf[ buf_used ] = (uchar)0x80;
630 2086 : buf_used++;
631 :
632 2086 : ulong bit_cnt = 32UL << 3;
633 2086 : memset( buf + buf_used, 0, FD_SHA256_PRIVATE_BUF_MAX-8UL-buf_used );
634 2086 : FD_STORE( ulong, buf+FD_SHA256_PRIVATE_BUF_MAX-8UL, fd_ulong_bswap( bit_cnt ) );
635 :
636 : /* This is just the above streamlined to eliminate all the overheads
637 : to support incremental hashing. */
638 5498092 : for( ulong iter=0UL; iter<cnt; iter++ ) {
639 :
640 5496006 : uint state[8] __attribute__((aligned(32)));
641 :
642 5496006 : state[0] = FD_SHA256_INITIAL_A;
643 5496006 : state[1] = FD_SHA256_INITIAL_B;
644 5496006 : state[2] = FD_SHA256_INITIAL_C;
645 5496006 : state[3] = FD_SHA256_INITIAL_D;
646 5496006 : state[4] = FD_SHA256_INITIAL_E;
647 5496006 : state[5] = FD_SHA256_INITIAL_F;
648 5496006 : state[6] = FD_SHA256_INITIAL_G;
649 5496006 : state[7] = FD_SHA256_INITIAL_H;
650 :
651 5496006 : fd_sha256_core( state, buf, 1UL );
652 :
653 5496006 : state[0] = fd_uint_bswap( state[0] );
654 5496006 : state[1] = fd_uint_bswap( state[1] );
655 5496006 : state[2] = fd_uint_bswap( state[2] );
656 5496006 : state[3] = fd_uint_bswap( state[3] );
657 5496006 : state[4] = fd_uint_bswap( state[4] );
658 5496006 : state[5] = fd_uint_bswap( state[5] );
659 5496006 : state[6] = fd_uint_bswap( state[6] );
660 5496006 : state[7] = fd_uint_bswap( state[7] );
661 5496006 : memcpy( buf, state, 32UL );
662 5496006 : }
663 2086 : memcpy( hash, buf, 32UL );
664 2086 : #endif
665 11111 : return _hash;
666 11111 : }
667 :
668 : #undef fd_sha256_core
669 :
670 : #if FD_SHA256_CORE_IMPL==1
671 :
672 : /* A single repeated hash chain is bound by the sha256rnds2 dependency
673 : chain. The kernels below run 2-5 independent chains, issuing each
674 : step for every chain in turn so their rounds overlap. Every chain
675 : has its own named registers (suffix _<lane>). Zen 4 M hashes/s by
676 : chain count: 1: 31.1, 2: 58.9, 3: 74.8, 4: 80.1, 5: 79.6 (6+ spill
677 : heavily and get slower). */
678 :
679 1393184 : #define EACH2( M, ... ) M( 0, __VA_ARGS__ ) M( 1, __VA_ARGS__ )
680 1209746 : #define EACH3( M, ... ) EACH2( M, __VA_ARGS__ ) M( 2, __VA_ARGS__ )
681 531304 : #define EACH4( M, ... ) EACH3( M, __VA_ARGS__ ) M( 3, __VA_ARGS__ )
682 213076 : #define EACH5( M, ... ) EACH4( M, __VA_ARGS__ ) M( 4, __VA_ARGS__ )
683 :
684 : #define X_LOAD( l, in ) \
685 1867 : vu_t w0003_##l = vu_bswap( vu_ldu( (in)+32UL*(l) ) ); \
686 1867 : vu_t w0407_##l = vu_bswap( vu_ldu( (in)+32UL*(l)+16UL ) );
687 : #define X_INIT( l, _ ) \
688 157892 : vu_t feba_##l = initialFEBA; vu_t hgdc_##l = initialHGDC; \
689 157892 : vu_t w080b_##l = w080b; vu_t w0c0f_##l = w0c0f;
690 2526272 : #define X_R( l, w, k ) FOUR_ROUNDS_STATE( feba_##l, hgdc_##l, vu_add( w##_##l, vu_ld( fd_sha256_K+(k) ) ) );
691 1894704 : #define X_W( l, d, a, b, c, e ) vu_t d##_##l = NEXT_W( a##_##l, b##_##l, c##_##l, e##_##l );
692 : #define X_FINI( l, _ ) \
693 157892 : feba_##l = vu_add( feba_##l, initialFEBA ); hgdc_##l = vu_add( hgdc_##l, initialHGDC ); \
694 157892 : w0003_##l = vu_permute2( feba_##l, hgdc_##l, 3, 2, 3, 2 ); \
695 157892 : w0407_##l = vu_permute2( feba_##l, hgdc_##l, 1, 0, 1, 0 );
696 : #define X_STORE( l, out ) \
697 1867 : vu_stu( (out)+32UL*(l), vu_bswap( w0003_##l ) ); \
698 1867 : vu_stu( (out)+32UL*(l)+16UL, vu_bswap( w0407_##l ) );
699 :
700 547 : #define REPEATED_MULTI( EACH ) do { \
701 547 : vu_t const w080b = vu( 0x80000000U, 0U, 0U, 0U ); \
702 547 : vu_t const w0c0f = vu( 0U, 0U, 0U, 256U ); /* 32 bytes */ \
703 547 : vu_t const initialFEBA = vu( FD_SHA256_INITIAL_F, FD_SHA256_INITIAL_E, FD_SHA256_INITIAL_B, FD_SHA256_INITIAL_A ); \
704 547 : vu_t const initialHGDC = vu( FD_SHA256_INITIAL_H, FD_SHA256_INITIAL_G, FD_SHA256_INITIAL_D, FD_SHA256_INITIAL_C ); \
705 1867 : EACH( X_LOAD, in ) \
706 46950 : for( ulong iter=0UL; iter<cnt; iter++ ) { \
707 157892 : EACH( X_INIT, _ ) \
708 157892 : /* */ EACH( X_R, w0003, 0 ); \
709 157892 : /* */ EACH( X_R, w0407, 4 ); \
710 157892 : /* */ EACH( X_R, w080b, 8 ); \
711 157892 : /* */ EACH( X_R, w0c0f, 12 ); \
712 157892 : EACH( X_W, w1013, w0003, w0407, w080b, w0c0f ); EACH( X_R, w1013, 16 ); \
713 157892 : EACH( X_W, w1417, w0407, w080b, w0c0f, w1013 ); EACH( X_R, w1417, 20 ); \
714 157892 : EACH( X_W, w181b, w080b, w0c0f, w1013, w1417 ); EACH( X_R, w181b, 24 ); \
715 157892 : EACH( X_W, w1c1f, w0c0f, w1013, w1417, w181b ); EACH( X_R, w1c1f, 28 ); \
716 157892 : EACH( X_W, w2023, w1013, w1417, w181b, w1c1f ); EACH( X_R, w2023, 32 ); \
717 157892 : EACH( X_W, w2427, w1417, w181b, w1c1f, w2023 ); EACH( X_R, w2427, 36 ); \
718 157892 : EACH( X_W, w282b, w181b, w1c1f, w2023, w2427 ); EACH( X_R, w282b, 40 ); \
719 157892 : EACH( X_W, w2c2f, w1c1f, w2023, w2427, w282b ); EACH( X_R, w2c2f, 44 ); \
720 157892 : EACH( X_W, w3033, w2023, w2427, w282b, w2c2f ); EACH( X_R, w3033, 48 ); \
721 157892 : EACH( X_W, w3437, w2427, w282b, w2c2f, w3033 ); EACH( X_R, w3437, 52 ); \
722 157892 : EACH( X_W, w383b, w282b, w2c2f, w3033, w3437 ); EACH( X_R, w383b, 56 ); \
723 157892 : EACH( X_W, w3c3f, w2c2f, w3033, w3437, w383b ); EACH( X_R, w3c3f, 60 ); \
724 157892 : EACH( X_FINI, _ ) \
725 46403 : } \
726 1867 : EACH( X_STORE, out ) \
727 547 : } while( 0 )
728 :
729 183438 : static void fd_sha256_hash_32_repeated_x2( uchar const * in, uchar * out, ulong cnt ) { REPEATED_MULTI( EACH2 ); }
730 678442 : static void fd_sha256_hash_32_repeated_x3( uchar const * in, uchar * out, ulong cnt ) { REPEATED_MULTI( EACH3 ); }
731 318228 : static void fd_sha256_hash_32_repeated_x4( uchar const * in, uchar * out, ulong cnt ) { REPEATED_MULTI( EACH4 ); }
732 213076 : static void fd_sha256_hash_32_repeated_x5( uchar const * in, uchar * out, ulong cnt ) { REPEATED_MULTI( EACH5 ); }
733 :
734 : #undef REPEATED_MULTI
735 : #undef X_STORE
736 : #undef X_FINI
737 : #undef X_W
738 : #undef X_R
739 : #undef X_INIT
740 : #undef X_LOAD
741 : #undef EACH5
742 : #undef EACH4
743 : #undef EACH3
744 : #undef EACH2
745 :
746 : static void
747 : fd_sha256_hash_32_repeated_shani( uchar const * in,
748 : uchar * out,
749 : ulong cnt,
750 557 : ulong lane_cnt ) {
751 : /* Split into the fewest groups of at most 5 chains, of near equal
752 : size (e.g. 4+3 beats 5+2). */
753 557 : ulong grp_cnt = (lane_cnt+4UL)/5UL;
754 1173 : while( lane_cnt ) {
755 616 : ulong k = (lane_cnt+grp_cnt-1UL)/grp_cnt;
756 616 : switch( k ) {
757 83 : case 5UL: fd_sha256_hash_32_repeated_x5( in, out, cnt ); break;
758 129 : case 4UL: fd_sha256_hash_32_repeated_x4( in, out, cnt ); break;
759 266 : case 3UL: fd_sha256_hash_32_repeated_x3( in, out, cnt ); break;
760 69 : case 2UL: fd_sha256_hash_32_repeated_x2( in, out, cnt ); break;
761 69 : default: fd_sha256_hash_32_repeated ( in, out, cnt ); break;
762 616 : }
763 616 : in += 32UL*k; out += 32UL*k; lane_cnt -= k; grp_cnt--;
764 616 : }
765 557 : }
766 :
767 : #undef NEXT_W
768 : #undef FOUR_ROUNDS
769 : #undef FOUR_ROUNDS_STATE
770 : #undef FULL_ROUNDS
771 :
772 : #endif
773 :
774 : #if FD_SHA256_CORE_IMPL==2
775 : void
776 : fd_sha256_hash_32_repeated_batch_arm( uchar const * hash_in,
777 : uchar * hash_out,
778 : ulong cnt,
779 : ulong batch_cnt );
780 : #endif
781 :
782 : #if FD_HAS_AVX512
783 : void
784 : fd_sha256_hash_32_repeated_batch_avx512( uchar const * hash_in,
785 : uchar * hash_out,
786 : ulong cnt,
787 : ulong batch_cnt );
788 : #endif
789 :
790 : /* x86 lane policy. AVX512_LANE_MIN is the batch width at which the
791 : 16 lane AVX-512 kernel beats running the lanes as interleaved SHA-NI
792 : chains (or serially without SHA-NI). On Zen 4 SHA-NI wins at every
793 : width (16 lanes: 80 vs 66 M hashes/s). */
794 :
795 : #if FD_SHA256_CORE_IMPL==1 || (FD_SHA256_CORE_IMPL==0 && FD_HAS_AVX512)
796 : #define X86_BATCH 1
797 : #if !FD_HAS_AVX512
798 : #define AVX512_LANE_MIN (ULONG_MAX)
799 : #elif FD_SHA256_CORE_IMPL==1 && defined(__znver4__)
800 : #define AVX512_LANE_MIN (ULONG_MAX)
801 : #elif defined(__znver5__)
802 : #define AVX512_LANE_MIN (6UL) /* Zen 5 has high AVX-512 throughput */
803 : #else
804 1359 : #define AVX512_LANE_MIN (8UL) /* Baseline 1 IPC AVX-512 needs more batching to win against SHA-NI */
805 : #endif
806 : #endif
807 :
808 : void
809 : fd_sha256_hash_32_repeated_batch( void const * _hash_in,
810 : void * _hash_out,
811 : ulong cnt,
812 3255 : ulong batch_cnt ) {
813 3255 : uchar const * hash_in = (uchar const *)_hash_in;
814 3255 : uchar * hash_out = (uchar *)_hash_out;
815 3255 : if( FD_UNLIKELY( batch_cnt>fd_sha256_simd_lane_max() ) ) FD_LOG_CRIT(( "batch_cnt %lu exceeds fd_sha256_simd_lane_max %lu", batch_cnt, fd_sha256_simd_lane_max() ));
816 : #if FD_SHA256_CORE_IMPL==2
817 : fd_sha256_hash_32_repeated_batch_arm( hash_in, hash_out, cnt, batch_cnt );
818 : return;
819 : #elif defined(X86_BATCH)
820 : # if FD_HAS_AVX512
821 1205 : if( batch_cnt>=AVX512_LANE_MIN ) {
822 648 : fd_sha256_hash_32_repeated_batch_avx512( hash_in, hash_out, cnt, batch_cnt );
823 648 : return;
824 648 : }
825 557 : # endif
826 557 : # if FD_SHA256_CORE_IMPL==1
827 557 : fd_sha256_hash_32_repeated_shani( hash_in, hash_out, cnt, batch_cnt );
828 557 : return;
829 0 : # endif
830 0 : #endif
831 3078 : for( ulong i=0UL; i<batch_cnt; i++ ) {
832 1028 : fd_sha256_hash_32_repeated( hash_in+32*i, hash_out+32*i, cnt );
833 1028 : }
834 2050 : }
835 :
836 : #ifdef X86_BATCH
837 :
838 : ulong
839 105 : fd_sha256_simd_lane_min( void ) {
840 105 : # if FD_SHA256_CORE_IMPL==1
841 105 : return fd_ulong_min( 2UL, AVX512_LANE_MIN );
842 : # else
843 : return AVX512_LANE_MIN;
844 : # endif
845 105 : }
846 :
847 1310 : ulong fd_sha256_simd_lane_max( void ) { return 16UL; }
848 :
849 : ulong
850 49 : fd_sha256_simd_iter_cost_q8( ulong lane_cnt ) {
851 49 : if( lane_cnt>=AVX512_LANE_MIN ) return 1357UL; /* 5.3x on Zen 5: 16 lanes at 96.5 M hashes/s vs 31.7 M hashes/s single lane with SHA-NI */
852 32 : # if FD_SHA256_CORE_IMPL==1
853 : /* Zen 4, per group of k interleaved chains: 256*k*rate(1)/rate(k) */
854 32 : static ushort const group_cost_q8[6] = { 0, 256, 273, 322, 398, 495 };
855 32 : ulong cost = 0UL;
856 32 : ulong grp_cnt = (lane_cnt+4UL)/5UL;
857 68 : while( lane_cnt ) {
858 36 : ulong k = (lane_cnt+grp_cnt-1UL)/grp_cnt;
859 36 : cost += group_cost_q8[ k ]; lane_cnt -= k; grp_cnt--;
860 36 : }
861 32 : return fd_ulong_max( cost, 256UL );
862 : # else
863 : return fd_ulong_max( lane_cnt, 1UL )*256UL;
864 : # endif
865 49 : }
866 :
867 : #undef AVX512_LANE_MIN
868 : #undef X86_BATCH
869 :
870 : #else
871 :
872 208 : __attribute__((weak)) ulong fd_sha256_simd_lane_min( void ) { return ULONG_MAX; }
873 2260 : __attribute__((weak)) ulong fd_sha256_simd_lane_max( void ) { return 1UL; }
874 2 : __attribute__((weak)) ulong fd_sha256_simd_iter_cost_q8( ulong lane_cnt ) { return fd_ulong_max( lane_cnt, 1UL )*256UL; }
875 :
876 : #endif
|
