Line data Source code
1 : #include "fd_sha256.h"
2 : #include "fd_sha256_constants.h"
3 :
4 : #if FD_HAS_SHANI
5 : /* For the optimized repeated hash */
6 : #include "../../util/simd/fd_sse.h"
7 : #endif
8 :
9 : ulong
10 931047 : fd_sha256_align( void ) {
11 931047 : return FD_SHA256_ALIGN;
12 931047 : }
13 :
14 : ulong
15 465513 : fd_sha256_footprint( void ) {
16 465513 : return FD_SHA256_FOOTPRINT;
17 465513 : }
18 :
19 : void *
20 465516 : fd_sha256_new( void * shmem ) {
21 465516 : fd_sha256_t * sha = (fd_sha256_t *)shmem;
22 :
23 465516 : if( FD_UNLIKELY( !shmem ) ) {
24 3 : FD_LOG_WARNING(( "NULL shmem" ));
25 3 : return NULL;
26 3 : }
27 :
28 465513 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)shmem, fd_sha256_align() ) ) ) {
29 3 : FD_LOG_WARNING(( "misaligned shmem" ));
30 3 : return NULL;
31 3 : }
32 :
33 465510 : ulong footprint = fd_sha256_footprint();
34 :
35 465510 : fd_memset( sha, 0, footprint );
36 :
37 465510 : FD_COMPILER_MFENCE();
38 465510 : FD_VOLATILE( sha->magic ) = FD_SHA256_MAGIC;
39 465510 : FD_COMPILER_MFENCE();
40 :
41 465510 : return (void *)sha;
42 465513 : }
43 :
44 : fd_sha256_t *
45 465516 : fd_sha256_join( void * shsha ) {
46 :
47 465516 : if( FD_UNLIKELY( !shsha ) ) {
48 3 : FD_LOG_WARNING(( "NULL shsha" ));
49 3 : return NULL;
50 3 : }
51 :
52 465513 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)shsha, fd_sha256_align() ) ) ) {
53 3 : FD_LOG_WARNING(( "misaligned shsha" ));
54 3 : return NULL;
55 3 : }
56 :
57 465510 : fd_sha256_t * sha = (fd_sha256_t *)shsha;
58 :
59 465510 : if( FD_UNLIKELY( sha->magic!=FD_SHA256_MAGIC ) ) {
60 0 : FD_LOG_WARNING(( "bad magic" ));
61 0 : return NULL;
62 0 : }
63 :
64 465510 : return sha;
65 465510 : }
66 :
67 : void *
68 18 : fd_sha256_leave( fd_sha256_t * sha ) {
69 :
70 18 : if( FD_UNLIKELY( !sha ) ) {
71 3 : FD_LOG_WARNING(( "NULL sha" ));
72 3 : return NULL;
73 3 : }
74 :
75 15 : return (void *)sha;
76 18 : }
77 :
78 : void *
79 21 : fd_sha256_delete( void * shsha ) {
80 :
81 21 : if( FD_UNLIKELY( !shsha ) ) {
82 3 : FD_LOG_WARNING(( "NULL shsha" ));
83 3 : return NULL;
84 3 : }
85 :
86 18 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)shsha, fd_sha256_align() ) ) ) {
87 3 : FD_LOG_WARNING(( "misaligned shsha" ));
88 3 : return NULL;
89 3 : }
90 :
91 15 : fd_sha256_t * sha = (fd_sha256_t *)shsha;
92 :
93 15 : if( FD_UNLIKELY( sha->magic!=FD_SHA256_MAGIC ) ) {
94 0 : FD_LOG_WARNING(( "bad magic" ));
95 0 : return NULL;
96 0 : }
97 :
98 15 : FD_COMPILER_MFENCE();
99 15 : FD_VOLATILE( sha->magic ) = 0UL;
100 15 : FD_COMPILER_MFENCE();
101 :
102 15 : return (void *)sha;
103 15 : }
104 :
105 : #ifndef FD_SHA256_CORE_IMPL
106 : #if FD_HAS_ARM_SHA256
107 : #define FD_SHA256_CORE_IMPL 2
108 : #elif FD_HAS_SHANI
109 : #define FD_SHA256_CORE_IMPL 1
110 : #else
111 : #define FD_SHA256_CORE_IMPL 0
112 : #endif
113 : #endif
114 :
115 : #if FD_SHA256_CORE_IMPL==0
116 :
117 : /* The implementation below was derived from OpenSSL's SHA-256
118 : implementation (Apache-2.0 licensed). See in particular:
119 :
120 : https://github.com/openssl/openssl/blob/master/crypto/sha/sha256.c
121 :
122 : (link valid circa 2022-Dec). It has been made more strict with more
123 : extensive implementation documentation, has been simplified and has
124 : been streamlined specifically for use inside Firedancer base machine
125 : model (no machine specific capabilities required).
126 :
127 : In particular, fd_sha256_core_ref is based on OpenSSL's
128 : OPENSSL_SMALL_FOOTPRINT SHA-256 implementation (Apache licensed).
129 : This should work anywhere but it is not the highest performance
130 : implementation possible.
131 :
132 : It is also straightforward to replace these implementations with HPC
133 : implementations that target specific machine capabilities without
134 : requiring any changes to caller code. */
135 :
136 : static void
137 : fd_sha256_core_ref( uint * state,
138 : uchar const * block,
139 38197850 : ulong block_cnt ) {
140 :
141 :
142 22325984640 : # define ROTATE fd_uint_rotate_left
143 2480664960 : # define Sigma0(x) (ROTATE((x),30) ^ ROTATE((x),19) ^ ROTATE((x),10))
144 2480664960 : # define Sigma1(x) (ROTATE((x),26) ^ ROTATE((x),21) ^ ROTATE((x),7))
145 1860498720 : # define sigma0(x) (ROTATE((x),25) ^ ROTATE((x),14) ^ ((x)>>3))
146 1860498720 : # define sigma1(x) (ROTATE((x),15) ^ ROTATE((x),13) ^ ((x)>>10))
147 2480664960 : # define Ch(x,y,z) (((x) & (y)) ^ ((~(x)) & (z)))
148 2480664960 : # define Maj(x,y,z) (((x) & (y)) ^ ((x) & (z)) ^ ((y) & (z)))
149 :
150 38197850 : uchar const * W = block;
151 38760390 : do {
152 38760390 : uint a = state[0];
153 38760390 : uint b = state[1];
154 38760390 : uint c = state[2];
155 38760390 : uint d = state[3];
156 38760390 : uint e = state[4];
157 38760390 : uint f = state[5];
158 38760390 : uint g = state[6];
159 38760390 : uint h = state[7];
160 :
161 38760390 : uint X[16];
162 :
163 38760390 : ulong i;
164 658926630 : for( i=0UL; i<16UL; i++ ) {
165 620166240 : X[i] = fd_uint_bswap( FD_LOAD( uint, W + i*sizeof(uint) ) );
166 620166240 : uint T1 = X[i] + h + Sigma1(e) + Ch(e, f, g) + fd_sha256_K[i];
167 620166240 : uint T2 = Sigma0(a) + Maj(a, b, c);
168 620166240 : h = g;
169 620166240 : g = f;
170 620166240 : f = e;
171 620166240 : e = d + T1;
172 620166240 : d = c;
173 620166240 : c = b;
174 620166240 : b = a;
175 620166240 : a = T1 + T2;
176 620166240 : }
177 1899259110 : for( ; i<64UL; i++ ) {
178 1860498720 : uint s0 = X[(i + 1UL) & 0x0fUL];
179 1860498720 : uint s1 = X[(i + 14UL) & 0x0fUL];
180 1860498720 : s0 = sigma0(s0);
181 1860498720 : s1 = sigma1(s1);
182 1860498720 : X[i & 0xfUL] += s0 + s1 + X[(i + 9UL) & 0xfUL];
183 1860498720 : uint T1 = X[i & 0xfUL ] + h + Sigma1(e) + Ch(e, f, g) + fd_sha256_K[i];
184 1860498720 : uint T2 = Sigma0(a) + Maj(a, b, c);
185 1860498720 : h = g;
186 1860498720 : g = f;
187 1860498720 : f = e;
188 1860498720 : e = d + T1;
189 1860498720 : d = c;
190 1860498720 : c = b;
191 1860498720 : b = a;
192 1860498720 : a = T1 + T2;
193 1860498720 : }
194 :
195 38760390 : state[0] += a;
196 38760390 : state[1] += b;
197 38760390 : state[2] += c;
198 38760390 : state[3] += d;
199 38760390 : state[4] += e;
200 38760390 : state[5] += f;
201 38760390 : state[6] += g;
202 38760390 : state[7] += h;
203 :
204 38760390 : W += 16UL*sizeof(uint);
205 38760390 : } while( --block_cnt );
206 :
207 38197850 : # undef ROTATE
208 38197850 : # undef Sigma0
209 38197850 : # undef Sigma1
210 38197850 : # undef sigma0
211 38197850 : # undef sigma1
212 38197850 : # undef Ch
213 38197850 : # undef Maj
214 :
215 38197850 : }
216 :
217 38197850 : #define fd_sha256_core fd_sha256_core_ref
218 :
219 : #elif FD_SHA256_CORE_IMPL==1
220 :
221 : /* _mm_sha256rnds2_epu32 does two rounds, one from the first uint in
222 : wk and one from the second. Since wk stores four rounds worth of
223 : message schedule values, it makes sense for the macro to do four
224 : rounds at a time. We need to permute wk in between so that the
225 : second call to the intrinsic will use the other values. */
226 324376016 : #define FOUR_ROUNDS( wk ) do { \
227 324376016 : vu_t __wk = (wk); \
228 324376016 : vu_t temp_state = stateFEBA; \
229 324376016 : stateFEBA = _mm_sha256rnds2_epu32( stateHGDC, stateFEBA, __wk ); \
230 324376016 : stateHGDC = temp_state; \
231 324376016 : \
232 324376016 : temp_state = stateFEBA; \
233 324376016 : stateFEBA = _mm_sha256rnds2_epu32( stateHGDC, stateFEBA, vu_permute( __wk, 2,3,0,1 ) );\
234 324376016 : stateHGDC = temp_state; \
235 324376016 : } while( 0 )
236 :
237 :
238 : /* For completeness, here's the documentation for _mm_sha256msg1_epu32
239 : and _mm_sha256msg2_epu32 in a slightly reformatted way, where all
240 : values are uints, and "-" indicates a don't-care value:
241 :
242 : _mm_sha256msg1_epu32( (w[j ], w[j+1], w[j+1], w[j+3]),
243 : (w[j+4], -, -, - ) )
244 : = ( w[j ]+s0( w[j+1] ), w[j+1]+s0( w[j+2] ),
245 : w[j+2]+s0( w[j+3] ), w[j+3]+s0( w[j+4] ) ).
246 :
247 :
248 : _mm_sha256msg2_epu32( (v[j ], v[j+1], v[j+1], v[j+3]),
249 : (-, -, w[j-2], w[j-1]) )
250 : sets w[j ] = v[j ] + s1( w[j-2] ) and
251 : w[j+1] = v[j+1] + s1( w[j-1] ), and then returns
252 :
253 : ( v[j ]+s1( w[j-2] ), v[j+1]+s1( w[j-1] ),
254 : v[j+2]+s1( w[j ] ), v[j+3]+s1( w[j+1] ) ) */
255 :
256 :
257 : /* w[i] for i>= 16 is w[i-16] + s0(w[i-15]) + w[i-7] + s1(w[i-2])
258 : Since our vector size is 4 uints, it's only s1 that is a little
259 : problematic, because it references items in the same vector.
260 : Thankfully, the msg2 intrinsic takes care of the complexity, but we
261 : need to execute it last.
262 :
263 : We get w[i-16] and s0(s[i-15]) using the msg1 intrinsic, setting j =
264 : i-16. For example, to compute w1013, we pass in w0003 and w0407.
265 : Then we can get w[i-7] by using the alignr instruction on
266 : (w[i-8], w[i-7], w[i-6], w[i-5]) and (w[i-4], w[i-3], w[i-2], w[i-1])
267 : to concatenate them and shift by one uint. Continuing with the
268 : example of w1013, we need w080b and w0c0f. We then put
269 : v[i] = w[i-16] + s0(w[i-15]) + w[i-7],
270 : and invoke the msg2 intrinsic with j=i, which gives w[i], as desired.
271 : Each invocation of NEXT_W computes 4 values of w. */
272 :
273 243282012 : #define NEXT_W( w_minus_16, w_minus_12, w_minus_8, w_minus_4 ) (__extension__({ \
274 243282012 : vu_t __w_i_16_s0_i_15 = _mm_sha256msg1_epu32( w_minus_16, w_minus_12 ); \
275 243282012 : vu_t __w_i_7 = _mm_alignr_epi8( w_minus_4, w_minus_8, 4 ); \
276 243282012 : _mm_sha256msg2_epu32( vu_add( __w_i_7, __w_i_16_s0_i_15 ), w_minus_4 ); \
277 243282012 : }))
278 :
279 : /* Zen 5's sha256rnds2 has an RTP of 2, while Zen 4's has an RTP of 1. We can
280 : win some performance by moving the schedule updates earlier in the loop,
281 : which improves the speed by around 1M hashes/s for the repeated hashing. */
282 : #ifdef __znver5__
283 : #define FULL_ROUNDS() do { \
284 : vu_t w1013 = NEXT_W( w0003, w0407, w080b, w0c0f ); FOUR_ROUNDS( vu_add( w0003, vu_ld( fd_sha256_K+ 0UL ) ) ); \
285 : vu_t w1417 = NEXT_W( w0407, w080b, w0c0f, w1013 ); FOUR_ROUNDS( vu_add( w0407, vu_ld( fd_sha256_K+ 4UL ) ) ); \
286 : vu_t w181b = NEXT_W( w080b, w0c0f, w1013, w1417 ); FOUR_ROUNDS( vu_add( w080b, vu_ld( fd_sha256_K+ 8UL ) ) ); \
287 : vu_t w1c1f = NEXT_W( w0c0f, w1013, w1417, w181b ); FOUR_ROUNDS( vu_add( w0c0f, vu_ld( fd_sha256_K+12UL ) ) ); \
288 : vu_t w2023 = NEXT_W( w1013, w1417, w181b, w1c1f ); FOUR_ROUNDS( vu_add( w1013, vu_ld( fd_sha256_K+16UL ) ) ); \
289 : vu_t w2427 = NEXT_W( w1417, w181b, w1c1f, w2023 ); FOUR_ROUNDS( vu_add( w1417, vu_ld( fd_sha256_K+20UL ) ) ); \
290 : vu_t w282b = NEXT_W( w181b, w1c1f, w2023, w2427 ); FOUR_ROUNDS( vu_add( w181b, vu_ld( fd_sha256_K+24UL ) ) ); \
291 : vu_t w2c2f = NEXT_W( w1c1f, w2023, w2427, w282b ); FOUR_ROUNDS( vu_add( w1c1f, vu_ld( fd_sha256_K+28UL ) ) ); \
292 : vu_t w3033 = NEXT_W( w2023, w2427, w282b, w2c2f ); FOUR_ROUNDS( vu_add( w2023, vu_ld( fd_sha256_K+32UL ) ) ); \
293 : vu_t w3437 = NEXT_W( w2427, w282b, w2c2f, w3033 ); FOUR_ROUNDS( vu_add( w2427, vu_ld( fd_sha256_K+36UL ) ) ); \
294 : vu_t w383b = NEXT_W( w282b, w2c2f, w3033, w3437 ); FOUR_ROUNDS( vu_add( w282b, vu_ld( fd_sha256_K+40UL ) ) ); \
295 : vu_t w3c3f = NEXT_W( w2c2f, w3033, w3437, w383b ); FOUR_ROUNDS( vu_add( w2c2f, vu_ld( fd_sha256_K+44UL ) ) ); \
296 : /* */ FOUR_ROUNDS( vu_add( w3033, vu_ld( fd_sha256_K+48UL ) ) ); \
297 : /* */ FOUR_ROUNDS( vu_add( w3437, vu_ld( fd_sha256_K+52UL ) ) ); \
298 : /* */ FOUR_ROUNDS( vu_add( w383b, vu_ld( fd_sha256_K+56UL ) ) ); \
299 : /* */ FOUR_ROUNDS( vu_add( w3c3f, vu_ld( fd_sha256_K+60UL ) ) ); \
300 : } while ( 0 )
301 : #else
302 20273501 : #define FULL_ROUNDS() do { \
303 20273501 : /* */ FOUR_ROUNDS( vu_add( w0003, vu_ld( fd_sha256_K+ 0UL ) ) ); \
304 20273501 : /* */ FOUR_ROUNDS( vu_add( w0407, vu_ld( fd_sha256_K+ 4UL ) ) ); \
305 20273501 : /* */ FOUR_ROUNDS( vu_add( w080b, vu_ld( fd_sha256_K+ 8UL ) ) ); \
306 20273501 : /* */ FOUR_ROUNDS( vu_add( w0c0f, vu_ld( fd_sha256_K+12UL ) ) ); \
307 20273501 : vu_t w1013 = NEXT_W( w0003, w0407, w080b, w0c0f ); FOUR_ROUNDS( vu_add( w1013, vu_ld( fd_sha256_K+16UL ) ) ); \
308 20273501 : vu_t w1417 = NEXT_W( w0407, w080b, w0c0f, w1013 ); FOUR_ROUNDS( vu_add( w1417, vu_ld( fd_sha256_K+20UL ) ) ); \
309 20273501 : vu_t w181b = NEXT_W( w080b, w0c0f, w1013, w1417 ); FOUR_ROUNDS( vu_add( w181b, vu_ld( fd_sha256_K+24UL ) ) ); \
310 20273501 : vu_t w1c1f = NEXT_W( w0c0f, w1013, w1417, w181b ); FOUR_ROUNDS( vu_add( w1c1f, vu_ld( fd_sha256_K+28UL ) ) ); \
311 20273501 : vu_t w2023 = NEXT_W( w1013, w1417, w181b, w1c1f ); FOUR_ROUNDS( vu_add( w2023, vu_ld( fd_sha256_K+32UL ) ) ); \
312 20273501 : vu_t w2427 = NEXT_W( w1417, w181b, w1c1f, w2023 ); FOUR_ROUNDS( vu_add( w2427, vu_ld( fd_sha256_K+36UL ) ) ); \
313 20273501 : vu_t w282b = NEXT_W( w181b, w1c1f, w2023, w2427 ); FOUR_ROUNDS( vu_add( w282b, vu_ld( fd_sha256_K+40UL ) ) ); \
314 20273501 : vu_t w2c2f = NEXT_W( w1c1f, w2023, w2427, w282b ); FOUR_ROUNDS( vu_add( w2c2f, vu_ld( fd_sha256_K+44UL ) ) ); \
315 20273501 : vu_t w3033 = NEXT_W( w2023, w2427, w282b, w2c2f ); FOUR_ROUNDS( vu_add( w3033, vu_ld( fd_sha256_K+48UL ) ) ); \
316 20273501 : vu_t w3437 = NEXT_W( w2427, w282b, w2c2f, w3033 ); FOUR_ROUNDS( vu_add( w3437, vu_ld( fd_sha256_K+52UL ) ) ); \
317 20273501 : vu_t w383b = NEXT_W( w282b, w2c2f, w3033, w3437 ); FOUR_ROUNDS( vu_add( w383b, vu_ld( fd_sha256_K+56UL ) ) ); \
318 20273501 : vu_t w3c3f = NEXT_W( w2c2f, w3033, w3437, w383b ); FOUR_ROUNDS( vu_add( w3c3f, vu_ld( fd_sha256_K+60UL ) ) ); \
319 20273501 : } while ( 0 )
320 : #endif
321 :
322 :
323 : void
324 : fd_sha256_core_shaext( uint * state, /* 64-byte aligned, 8 entries */
325 : uchar const * block, /* ideally 128-byte aligned (but not required), 64*block_cnt in size */
326 16373152 : ulong block_cnt ) {/* positive */
327 16373152 : vu_t stateABCD = vu_ld( state );
328 16373152 : vu_t stateEFGH = vu_ld( state+4UL );
329 :
330 16373152 : vu_t baseFEBA = vu_permute2( stateEFGH, stateABCD, 1, 0, 1, 0 );
331 16373152 : vu_t baseHGDC = vu_permute2( stateEFGH, stateABCD, 3, 2, 3, 2 );
332 :
333 33032517 : for( ulong b=0UL; b<block_cnt; b++ ) {
334 16659365 : vu_t stateFEBA = baseFEBA;
335 16659365 : vu_t stateHGDC = baseHGDC;
336 :
337 16659365 : vu_t w0003 = vu_bswap( vu_ldu( block+64UL*b ) );
338 16659365 : vu_t w0407 = vu_bswap( vu_ldu( block+64UL*b+16UL ) );
339 16659365 : vu_t w080b = vu_bswap( vu_ldu( block+64UL*b+32UL ) );
340 16659365 : vu_t w0c0f = vu_bswap( vu_ldu( block+64UL*b+48UL ) );
341 :
342 16659365 : FULL_ROUNDS();
343 :
344 16659365 : baseFEBA = vu_add( baseFEBA, stateFEBA );
345 16659365 : baseHGDC = vu_add( baseHGDC, stateHGDC );
346 :
347 16659365 : }
348 :
349 16373152 : stateABCD = vu_permute2( baseFEBA, baseHGDC, 3, 2, 3, 2 );
350 16373152 : stateEFGH = vu_permute2( baseFEBA, baseHGDC, 1, 0, 1, 0 );
351 16373152 : vu_st( state, stateABCD );
352 16373152 : vu_st( state+4UL, stateEFGH );
353 16373152 : }
354 :
355 16373152 : #define fd_sha256_core fd_sha256_core_shaext
356 :
357 : #elif FD_SHA256_CORE_IMPL==2
358 :
359 : void
360 : fd_sha256_core_arm( uint * state,
361 : uchar const * block,
362 : ulong block_cnt );
363 :
364 : #define fd_sha256_core fd_sha256_core_arm
365 :
366 : #else
367 : #error "Unsupported FD_SHA256_CORE_IMPL"
368 : #endif
369 :
370 : fd_sha256_t *
371 963054 : fd_sha256_init( fd_sha256_t * sha ) {
372 963054 : sha->state[0] = FD_SHA256_INITIAL_A;
373 963054 : sha->state[1] = FD_SHA256_INITIAL_B;
374 963054 : sha->state[2] = FD_SHA256_INITIAL_C;
375 963054 : sha->state[3] = FD_SHA256_INITIAL_D;
376 963054 : sha->state[4] = FD_SHA256_INITIAL_E;
377 963054 : sha->state[5] = FD_SHA256_INITIAL_F;
378 963054 : sha->state[6] = FD_SHA256_INITIAL_G;
379 963054 : sha->state[7] = FD_SHA256_INITIAL_H;
380 963054 : sha->buf_used = 0UL;
381 963054 : sha->bit_cnt = 0UL;
382 963054 : return sha;
383 963054 : }
384 :
385 : fd_sha256_t *
386 : fd_sha256_append( fd_sha256_t * sha,
387 : void const * _data,
388 2010642 : ulong sz ) {
389 :
390 : /* If no data to append, we are done */
391 :
392 2010642 : if( FD_UNLIKELY( !sz ) ) return sha; /* optimize for non-trivial append */
393 :
394 : /* Unpack inputs */
395 :
396 2010396 : uint * state = sha->state;
397 2010396 : uchar * buf = sha->buf;
398 2010396 : ulong buf_used = sha->buf_used;
399 2010396 : ulong bit_cnt = sha->bit_cnt;
400 :
401 2010396 : uchar const * data = (uchar const *)_data;
402 :
403 : /* Update bit_cnt */
404 : /* FIXME: could accumulate bytes here and do bit conversion in append */
405 : /* FIXME: Overflow handling if more than 2^64 bits (unlikely) */
406 :
407 2010396 : sha->bit_cnt = bit_cnt + (sz<<3);
408 :
409 : /* Handle buffered bytes from previous appends */
410 :
411 2010396 : if( FD_UNLIKELY( buf_used ) ) { /* optimized for well aligned use of append */
412 :
413 : /* If the append isn't large enough to complete the current block,
414 : buffer these bytes too and return */
415 :
416 110349 : ulong buf_rem = FD_SHA256_PRIVATE_BUF_MAX - buf_used; /* In (0,FD_SHA256_PRIVATE_BUF_MAX) */
417 110349 : if( FD_UNLIKELY( sz < buf_rem ) ) { /* optimize for large append */
418 27285 : fd_memcpy( buf + buf_used, data, sz );
419 27285 : sha->buf_used = buf_used + sz;
420 27285 : return sha;
421 27285 : }
422 :
423 : /* Otherwise, buffer enough leading bytes of data to complete the
424 : block, update the hash and then continue processing any remaining
425 : bytes of data. */
426 :
427 83064 : fd_memcpy( buf + buf_used, data, buf_rem );
428 83064 : data += buf_rem;
429 83064 : sz -= buf_rem;
430 :
431 83064 : fd_sha256_core( state, buf, 1UL );
432 83064 : sha->buf_used = 0UL;
433 83064 : }
434 :
435 : /* Append the bulk of the data */
436 :
437 1983111 : ulong block_cnt = sz >> FD_SHA256_PRIVATE_LG_BUF_MAX;
438 1983111 : if( FD_LIKELY( block_cnt ) ) fd_sha256_core( state, data, block_cnt ); /* optimized for large append */
439 :
440 : /* Buffer any leftover bytes */
441 :
442 1983111 : buf_used = sz & (FD_SHA256_PRIVATE_BUF_MAX-1UL); /* In [0,FD_SHA256_PRIVATE_BUF_MAX) */
443 1983111 : if( FD_UNLIKELY( buf_used ) ) { /* optimized for well aligned use of append */
444 1046052 : fd_memcpy( buf, data + (block_cnt << FD_SHA256_PRIVATE_LG_BUF_MAX), buf_used );
445 1046052 : sha->buf_used = buf_used; /* In (0,FD_SHA256_PRIVATE_BUF_MAX) */
446 1046052 : }
447 :
448 1983111 : return sha;
449 2010396 : }
450 :
451 : void *
452 : fd_sha256_fini( fd_sha256_t * sha,
453 1023819 : void * _hash ) {
454 :
455 : /* Unpack inputs */
456 :
457 1023819 : uint * state = sha->state;
458 1023819 : uchar * buf = sha->buf;
459 1023819 : ulong buf_used = sha->buf_used; /* In [0,FD_SHA256_PRIVATE_BUF_MAX) */
460 1023819 : ulong bit_cnt = sha->bit_cnt;
461 :
462 : /* Append the terminating message byte */
463 :
464 1023819 : buf[ buf_used ] = (uchar)0x80;
465 1023819 : buf_used++;
466 :
467 : /* If there isn't enough room to save the message length in bits at
468 : the end of the in progress block, clear the rest of the in progress
469 : block, update the hash and start a new block. */
470 :
471 1023819 : if( FD_UNLIKELY( buf_used > (FD_SHA256_PRIVATE_BUF_MAX-8UL) ) ) { /* optimize for well aligned use of append */
472 12342 : fd_memset( buf + buf_used, 0, FD_SHA256_PRIVATE_BUF_MAX-buf_used );
473 12342 : fd_sha256_core( state, buf, 1UL );
474 12342 : buf_used = 0UL;
475 12342 : }
476 :
477 : /* Clear in progress block up to last 64-bits, append the message
478 : size in bytes in the last 64-bits of the in progress block and
479 : update the hash to finalize it. */
480 :
481 1023819 : fd_memset( buf + buf_used, 0, FD_SHA256_PRIVATE_BUF_MAX-8UL-buf_used );
482 1023819 : FD_STORE( ulong, buf+FD_SHA256_PRIVATE_BUF_MAX-8UL, fd_ulong_bswap( bit_cnt ) );
483 1023819 : fd_sha256_core( state, buf, 1UL );
484 :
485 : /* Unpack the result into md (annoying bswaps here) */
486 :
487 1023819 : state[0] = fd_uint_bswap( state[0] );
488 1023819 : state[1] = fd_uint_bswap( state[1] );
489 1023819 : state[2] = fd_uint_bswap( state[2] );
490 1023819 : state[3] = fd_uint_bswap( state[3] );
491 1023819 : state[4] = fd_uint_bswap( state[4] );
492 1023819 : state[5] = fd_uint_bswap( state[5] );
493 1023819 : state[6] = fd_uint_bswap( state[6] );
494 1023819 : state[7] = fd_uint_bswap( state[7] );
495 1023819 : return memcpy( _hash, state, 32 );
496 1023819 : }
497 :
498 : void *
499 : fd_sha256_hash( void const * _data,
500 : ulong sz,
501 30851186 : void * _hash ) {
502 30851186 : uchar const * data = (uchar const *)_data;
503 :
504 : /* This is just the above streamlined to eliminate all the overheads
505 : to support incremental hashing. */
506 :
507 30851186 : uchar buf[ FD_SHA256_PRIVATE_BUF_MAX ] __attribute__((aligned(128)));
508 30851186 : uint state[8] __attribute__((aligned(32)));
509 :
510 30851186 : state[0] = FD_SHA256_INITIAL_A;
511 30851186 : state[1] = FD_SHA256_INITIAL_B;
512 30851186 : state[2] = FD_SHA256_INITIAL_C;
513 30851186 : state[3] = FD_SHA256_INITIAL_D;
514 30851186 : state[4] = FD_SHA256_INITIAL_E;
515 30851186 : state[5] = FD_SHA256_INITIAL_F;
516 30851186 : state[6] = FD_SHA256_INITIAL_G;
517 30851186 : state[7] = FD_SHA256_INITIAL_H;
518 :
519 30851186 : ulong block_cnt = sz >> FD_SHA256_PRIVATE_LG_BUF_MAX;
520 30851186 : if( FD_LIKELY( block_cnt ) ) fd_sha256_core( state, data, block_cnt );
521 :
522 30851186 : ulong buf_used = sz & (FD_SHA256_PRIVATE_BUF_MAX-1UL);
523 30851186 : if( FD_UNLIKELY( buf_used ) ) fd_memcpy( buf, data + (block_cnt << FD_SHA256_PRIVATE_LG_BUF_MAX), buf_used );
524 30851186 : buf[ buf_used ] = (uchar)0x80;
525 30851186 : buf_used++;
526 :
527 30851186 : if( FD_UNLIKELY( buf_used > (FD_SHA256_PRIVATE_BUF_MAX-8UL) ) ) {
528 147874 : fd_memset( buf + buf_used, 0, FD_SHA256_PRIVATE_BUF_MAX-buf_used );
529 147874 : fd_sha256_core( state, buf, 1UL );
530 147874 : buf_used = 0UL;
531 147874 : }
532 :
533 30851186 : ulong bit_cnt = sz << 3;
534 30851186 : fd_memset( buf + buf_used, 0, FD_SHA256_PRIVATE_BUF_MAX-8UL-buf_used );
535 30851186 : FD_STORE( ulong, buf+FD_SHA256_PRIVATE_BUF_MAX-8UL, fd_ulong_bswap( bit_cnt ) );
536 30851186 : fd_sha256_core( state, buf, 1UL );
537 :
538 30851186 : state[0] = fd_uint_bswap( state[0] );
539 30851186 : state[1] = fd_uint_bswap( state[1] );
540 30851186 : state[2] = fd_uint_bswap( state[2] );
541 30851186 : state[3] = fd_uint_bswap( state[3] );
542 30851186 : state[4] = fd_uint_bswap( state[4] );
543 30851186 : state[5] = fd_uint_bswap( state[5] );
544 30851186 : state[6] = fd_uint_bswap( state[6] );
545 30851186 : state[7] = fd_uint_bswap( state[7] );
546 30851186 : return memcpy( _hash, state, 32 );
547 30851186 : }
548 :
549 :
550 :
551 : #if FD_SHA256_CORE_IMPL==2
552 : void
553 : fd_sha256_hash_32_repeated_arm( uchar const * data,
554 : uchar * hash,
555 : ulong cnt );
556 : #endif
557 :
558 : void *
559 : fd_sha256_hash_32_repeated( void const * _data,
560 : void * _hash,
561 11943 : ulong cnt ) {
562 : #if FD_SHA256_CORE_IMPL==2
563 : fd_sha256_hash_32_repeated_arm( _data, _hash, cnt );
564 : return _hash;
565 : #elif FD_SHA256_CORE_IMPL==1
566 :
567 9785 : uchar const * data = (uchar const *)_data;
568 9785 : uchar * hash = (uchar *)_hash;
569 :
570 9785 : vu_t w0003 = vu_bswap( vu_ldu( data ) );
571 9785 : vu_t w0407 = vu_bswap( vu_ldu( data+16UL ) );
572 9785 : vb_t const w080b = vb( 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00,
573 9785 : 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 );
574 9785 : vb_t const w0c0f = vb( 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
575 9785 : 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00 ); /* 32 bytes */
576 :
577 9785 : vu_t const initialFEBA = vu( FD_SHA256_INITIAL_F, FD_SHA256_INITIAL_E, FD_SHA256_INITIAL_B, FD_SHA256_INITIAL_A );
578 9785 : vu_t const initialHGDC = vu( FD_SHA256_INITIAL_H, FD_SHA256_INITIAL_G, FD_SHA256_INITIAL_D, FD_SHA256_INITIAL_C );
579 :
580 3623921 : for( ulong iter=0UL; iter<cnt; iter++ ) {
581 3614136 : vu_t stateFEBA = initialFEBA;
582 3614136 : vu_t stateHGDC = initialHGDC;
583 :
584 3614136 : FULL_ROUNDS();
585 :
586 3614136 : stateFEBA = vu_add( stateFEBA, initialFEBA );
587 3614136 : stateHGDC = vu_add( stateHGDC, initialHGDC );
588 :
589 3614136 : vu_t stateABCD = vu_permute2( stateFEBA, stateHGDC, 3, 2, 3, 2 );
590 3614136 : vu_t stateEFGH = vu_permute2( stateFEBA, stateHGDC, 1, 0, 1, 0 );
591 :
592 3614136 : w0003 = stateABCD;
593 3614136 : w0407 = stateEFGH;
594 3614136 : }
595 9785 : vu_stu( hash, vu_bswap( w0003 ) );
596 9785 : vu_stu( hash+16UL, vu_bswap( w0407 ) );
597 9785 : #undef NEXT_W
598 9785 : #undef FOUR_ROUNDS
599 9785 : #undef FULL_ROUNDS
600 :
601 : #else
602 :
603 2158 : uchar const * data = (uchar const *)_data;
604 2158 : uchar * hash = (uchar *)_hash;
605 :
606 2158 : uchar buf[ FD_SHA256_PRIVATE_BUF_MAX ] __attribute__((aligned(128)));
607 :
608 : /* Prepare padding once */
609 2158 : ulong buf_used = 32UL;
610 2158 : memcpy( buf, data, 32UL );
611 2158 : buf[ buf_used ] = (uchar)0x80;
612 2158 : buf_used++;
613 :
614 2158 : ulong bit_cnt = 32UL << 3;
615 2158 : memset( buf + buf_used, 0, FD_SHA256_PRIVATE_BUF_MAX-8UL-buf_used );
616 2158 : FD_STORE( ulong, buf+FD_SHA256_PRIVATE_BUF_MAX-8UL, fd_ulong_bswap( bit_cnt ) );
617 :
618 : /* This is just the above streamlined to eliminate all the overheads
619 : to support incremental hashing. */
620 5505604 : for( ulong iter=0UL; iter<cnt; iter++ ) {
621 :
622 5503446 : uint state[8] __attribute__((aligned(32)));
623 :
624 5503446 : state[0] = FD_SHA256_INITIAL_A;
625 5503446 : state[1] = FD_SHA256_INITIAL_B;
626 5503446 : state[2] = FD_SHA256_INITIAL_C;
627 5503446 : state[3] = FD_SHA256_INITIAL_D;
628 5503446 : state[4] = FD_SHA256_INITIAL_E;
629 5503446 : state[5] = FD_SHA256_INITIAL_F;
630 5503446 : state[6] = FD_SHA256_INITIAL_G;
631 5503446 : state[7] = FD_SHA256_INITIAL_H;
632 :
633 5503446 : fd_sha256_core( state, buf, 1UL );
634 :
635 5503446 : state[0] = fd_uint_bswap( state[0] );
636 5503446 : state[1] = fd_uint_bswap( state[1] );
637 5503446 : state[2] = fd_uint_bswap( state[2] );
638 5503446 : state[3] = fd_uint_bswap( state[3] );
639 5503446 : state[4] = fd_uint_bswap( state[4] );
640 5503446 : state[5] = fd_uint_bswap( state[5] );
641 5503446 : state[6] = fd_uint_bswap( state[6] );
642 5503446 : state[7] = fd_uint_bswap( state[7] );
643 5503446 : memcpy( buf, state, 32UL );
644 5503446 : }
645 2158 : memcpy( hash, buf, 32UL );
646 2158 : #endif
647 11943 : return _hash;
648 11943 : }
649 :
650 : #undef fd_sha256_core
651 :
652 : #if FD_SHA256_CORE_IMPL==2
653 : void
654 : fd_sha256_hash_32_repeated_batch_arm( uchar const * hash_in,
655 : uchar * hash_out,
656 : ulong cnt,
657 : ulong batch_cnt );
658 : #endif
659 :
660 : #if FD_HAS_AVX512
661 : void
662 : fd_sha256_hash_32_repeated_batch_avx512( uchar const * hash_in,
663 : uchar * hash_out,
664 : ulong cnt,
665 : ulong batch_cnt );
666 : #endif
667 :
668 : void
669 : fd_sha256_hash_32_repeated_batch( void const * _hash_in,
670 : void * _hash_out,
671 : ulong cnt,
672 3000 : ulong batch_cnt ) {
673 3000 : uchar const * hash_in = (uchar const *)_hash_in;
674 3000 : uchar * hash_out = (uchar *)_hash_out;
675 3000 : if( FD_UNLIKELY( batch_cnt>fd_sha256_simd_lane_max() ) ) FD_LOG_CRIT(( "batch_cnt %lu exceeds fd_sha256_simd_lane_max %lu", batch_cnt, fd_sha256_simd_lane_max() ));
676 : #if FD_SHA256_CORE_IMPL==2
677 : fd_sha256_hash_32_repeated_batch_arm( hash_in, hash_out, cnt, batch_cnt );
678 : return;
679 : #elif FD_HAS_AVX512
680 1000 : fd_sha256_hash_32_repeated_batch_avx512( hash_in, hash_out, cnt, batch_cnt );
681 1000 : return;
682 0 : #endif
683 3052 : for( ulong i=0UL; i<batch_cnt; i++ ) {
684 1052 : fd_sha256_hash_32_repeated( hash_in+32*i, hash_out+32*i, cnt );
685 1052 : }
686 2000 : }
687 :
688 66 : __attribute__((weak)) ulong fd_sha256_simd_lane_min( void ) { return ULONG_MAX; }
689 3069 : __attribute__((weak)) ulong fd_sha256_simd_lane_max( void ) { return 1UL; }
690 51 : __attribute__((weak)) ulong fd_sha256_simd_iter_cost_q8( void ) { return 256UL; }
|
