Line data Source code
1 : #include "fd_sha256.h"
2 : #include "fd_sha256_constants.h"
3 :
4 : #if FD_HAS_SHANI
5 : #include "../../util/simd/fd_sse.h"
6 : #endif
7 :
8 : ulong
9 1122054 : fd_sha256_align( void ) {
10 1122054 : return FD_SHA256_ALIGN;
11 1122054 : }
12 :
13 : ulong
14 561015 : fd_sha256_footprint( void ) {
15 561015 : return FD_SHA256_FOOTPRINT;
16 561015 : }
17 :
18 : void *
19 561018 : fd_sha256_new( void * shmem ) {
20 561018 : fd_sha256_t * sha = (fd_sha256_t *)shmem;
21 :
22 561018 : if( FD_UNLIKELY( !shmem ) ) {
23 3 : FD_LOG_WARNING(( "NULL shmem" ));
24 3 : return NULL;
25 3 : }
26 :
27 561015 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)shmem, fd_sha256_align() ) ) ) {
28 3 : FD_LOG_WARNING(( "misaligned shmem" ));
29 3 : return NULL;
30 3 : }
31 :
32 561012 : ulong footprint = fd_sha256_footprint();
33 :
34 561012 : fd_memset( sha, 0, footprint );
35 :
36 561012 : FD_COMPILER_MFENCE();
37 561012 : FD_VOLATILE( sha->magic ) = FD_SHA256_MAGIC;
38 561012 : FD_COMPILER_MFENCE();
39 :
40 561012 : return (void *)sha;
41 561015 : }
42 :
43 : fd_sha256_t *
44 561018 : fd_sha256_join( void * shsha ) {
45 :
46 561018 : if( FD_UNLIKELY( !shsha ) ) {
47 3 : FD_LOG_WARNING(( "NULL shsha" ));
48 3 : return NULL;
49 3 : }
50 :
51 561015 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)shsha, fd_sha256_align() ) ) ) {
52 3 : FD_LOG_WARNING(( "misaligned shsha" ));
53 3 : return NULL;
54 3 : }
55 :
56 561012 : fd_sha256_t * sha = (fd_sha256_t *)shsha;
57 :
58 561012 : if( FD_UNLIKELY( sha->magic!=FD_SHA256_MAGIC ) ) {
59 0 : FD_LOG_WARNING(( "bad magic" ));
60 0 : return NULL;
61 0 : }
62 :
63 561012 : return sha;
64 561012 : }
65 :
66 : void *
67 21 : fd_sha256_leave( fd_sha256_t * sha ) {
68 :
69 21 : if( FD_UNLIKELY( !sha ) ) {
70 3 : FD_LOG_WARNING(( "NULL sha" ));
71 3 : return NULL;
72 3 : }
73 :
74 18 : return (void *)sha;
75 21 : }
76 :
77 : void *
78 24 : fd_sha256_delete( void * shsha ) {
79 :
80 24 : if( FD_UNLIKELY( !shsha ) ) {
81 3 : FD_LOG_WARNING(( "NULL shsha" ));
82 3 : return NULL;
83 3 : }
84 :
85 21 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)shsha, fd_sha256_align() ) ) ) {
86 3 : FD_LOG_WARNING(( "misaligned shsha" ));
87 3 : return NULL;
88 3 : }
89 :
90 18 : fd_sha256_t * sha = (fd_sha256_t *)shsha;
91 :
92 18 : if( FD_UNLIKELY( sha->magic!=FD_SHA256_MAGIC ) ) {
93 0 : FD_LOG_WARNING(( "bad magic" ));
94 0 : return NULL;
95 0 : }
96 :
97 18 : FD_COMPILER_MFENCE();
98 18 : FD_VOLATILE( sha->magic ) = 0UL;
99 18 : FD_COMPILER_MFENCE();
100 :
101 18 : return (void *)sha;
102 18 : }
103 :
104 : #ifndef FD_SHA256_CORE_IMPL
105 : #if FD_HAS_ARM_SHA256
106 : #define FD_SHA256_CORE_IMPL 2
107 : #elif FD_HAS_SHANI
108 : #define FD_SHA256_CORE_IMPL 1
109 : #elif FD_HAS_RISCV_SHA256
110 : #define FD_SHA256_CORE_IMPL 3
111 : #else
112 : #define FD_SHA256_CORE_IMPL 0
113 : #endif
114 : #endif
115 :
116 : #if FD_SHA256_CORE_IMPL==0
117 :
118 : /* The implementation below was derived from OpenSSL's SHA-256
119 : implementation (Apache-2.0 licensed). See in particular:
120 :
121 : https://github.com/openssl/openssl/blob/master/crypto/sha/sha256.c
122 :
123 : (link valid circa 2022-Dec). It has been made more strict with more
124 : extensive implementation documentation, has been simplified and has
125 : been streamlined specifically for use inside Firedancer base machine
126 : model (no machine specific capabilities required).
127 :
128 : In particular, fd_sha256_core_ref is based on OpenSSL's
129 : OPENSSL_SMALL_FOOTPRINT SHA-256 implementation (Apache licensed).
130 : This should work anywhere but it is not the highest performance
131 : implementation possible.
132 :
133 : It is also straightforward to replace these implementations with HPC
134 : implementations that target specific machine capabilities without
135 : requiring any changes to caller code. */
136 :
137 : static void
138 : fd_sha256_core_ref( uint * state,
139 : uchar const * block,
140 14908636 : ulong block_cnt ) {
141 :
142 :
143 10062820224 : # define ROTATE fd_uint_rotate_left
144 1118091136 : # define Sigma0(x) (ROTATE((x),30) ^ ROTATE((x),19) ^ ROTATE((x),10))
145 1118091136 : # define Sigma1(x) (ROTATE((x),26) ^ ROTATE((x),21) ^ ROTATE((x),7))
146 838568352 : # define sigma0(x) (ROTATE((x),25) ^ ROTATE((x),14) ^ ((x)>>3))
147 838568352 : # define sigma1(x) (ROTATE((x),15) ^ ROTATE((x),13) ^ ((x)>>10))
148 1118091136 : # define Ch(x,y,z) (((x) & (y)) ^ ((~(x)) & (z)))
149 1118091136 : # define Maj(x,y,z) (((x) & (y)) ^ ((x) & (z)) ^ ((y) & (z)))
150 :
151 14908636 : uchar const * W = block;
152 17470174 : do {
153 17470174 : uint a = state[0];
154 17470174 : uint b = state[1];
155 17470174 : uint c = state[2];
156 17470174 : uint d = state[3];
157 17470174 : uint e = state[4];
158 17470174 : uint f = state[5];
159 17470174 : uint g = state[6];
160 17470174 : uint h = state[7];
161 :
162 17470174 : uint X[16];
163 :
164 17470174 : ulong i;
165 296992958 : for( i=0UL; i<16UL; i++ ) {
166 279522784 : X[i] = fd_uint_bswap( FD_LOAD( uint, W + i*sizeof(uint) ) );
167 279522784 : uint T1 = X[i] + h + Sigma1(e) + Ch(e, f, g) + fd_sha256_K[i];
168 279522784 : uint T2 = Sigma0(a) + Maj(a, b, c);
169 279522784 : h = g;
170 279522784 : g = f;
171 279522784 : f = e;
172 279522784 : e = d + T1;
173 279522784 : d = c;
174 279522784 : c = b;
175 279522784 : b = a;
176 279522784 : a = T1 + T2;
177 279522784 : }
178 856038526 : for( ; i<64UL; i++ ) {
179 838568352 : uint s0 = X[(i + 1UL) & 0x0fUL];
180 838568352 : uint s1 = X[(i + 14UL) & 0x0fUL];
181 838568352 : s0 = sigma0(s0);
182 838568352 : s1 = sigma1(s1);
183 838568352 : X[i & 0xfUL] += s0 + s1 + X[(i + 9UL) & 0xfUL];
184 838568352 : uint T1 = X[i & 0xfUL ] + h + Sigma1(e) + Ch(e, f, g) + fd_sha256_K[i];
185 838568352 : uint T2 = Sigma0(a) + Maj(a, b, c);
186 838568352 : h = g;
187 838568352 : g = f;
188 838568352 : f = e;
189 838568352 : e = d + T1;
190 838568352 : d = c;
191 838568352 : c = b;
192 838568352 : b = a;
193 838568352 : a = T1 + T2;
194 838568352 : }
195 :
196 17470174 : state[0] += a;
197 17470174 : state[1] += b;
198 17470174 : state[2] += c;
199 17470174 : state[3] += d;
200 17470174 : state[4] += e;
201 17470174 : state[5] += f;
202 17470174 : state[6] += g;
203 17470174 : state[7] += h;
204 :
205 17470174 : W += 16UL*sizeof(uint);
206 17470174 : } while( --block_cnt );
207 :
208 14908636 : # undef ROTATE
209 14908636 : # undef Sigma0
210 14908636 : # undef Sigma1
211 14908636 : # undef sigma0
212 14908636 : # undef sigma1
213 14908636 : # undef Ch
214 14908636 : # undef Maj
215 :
216 14908636 : }
217 :
218 14908636 : #define fd_sha256_core fd_sha256_core_ref
219 :
220 : #elif FD_SHA256_CORE_IMPL==1
221 :
222 : /* _mm_sha256rnds2_epu32 does two rounds, one from the first uint in
223 : wk and one from the second. Since wk stores four rounds worth of
224 : message schedule values, it makes sense for the macro to do four
225 : rounds at a time. We need to permute wk in between so that the
226 : second call to the intrinsic will use the other values. */
227 163313024 : #define FOUR_ROUNDS( wk ) do { \
228 163313024 : vu_t __wk = (wk); \
229 163313024 : vu_t temp_state = stateFEBA; \
230 163313024 : stateFEBA = _mm_sha256rnds2_epu32( stateHGDC, stateFEBA, __wk ); \
231 163313024 : stateHGDC = temp_state; \
232 163313024 : \
233 163313024 : temp_state = stateFEBA; \
234 163313024 : stateFEBA = _mm_sha256rnds2_epu32( stateHGDC, stateFEBA, vu_permute( __wk, 2,3,0,1 ) );\
235 163313024 : stateHGDC = temp_state; \
236 163313024 : } while( 0 )
237 :
238 :
239 : /* For completeness, here's the documentation for _mm_sha256msg1_epu32
240 : and _mm_sha256msg2_epu32 in a slightly reformatted way, where all
241 : values are uints, and "-" indicates a don't-care value:
242 :
243 : _mm_sha256msg1_epu32( (w[j ], w[j+1], w[j+1], w[j+3]),
244 : (w[j+4], -, -, - ) )
245 : = ( w[j ]+s0( w[j+1] ), w[j+1]+s0( w[j+2] ),
246 : w[j+2]+s0( w[j+3] ), w[j+3]+s0( w[j+4] ) ).
247 :
248 :
249 : _mm_sha256msg2_epu32( (v[j ], v[j+1], v[j+1], v[j+3]),
250 : (-, -, w[j-2], w[j-1]) )
251 : sets w[j ] = v[j ] + s1( w[j-2] ) and
252 : w[j+1] = v[j+1] + s1( w[j-1] ), and then returns
253 :
254 : ( v[j ]+s1( w[j-2] ), v[j+1]+s1( w[j-1] ),
255 : v[j+2]+s1( w[j ] ), v[j+3]+s1( w[j+1] ) ) */
256 :
257 :
258 : /* w[i] for i>= 16 is w[i-16] + s0(w[i-15]) + w[i-7] + s1(w[i-2])
259 : Since our vector size is 4 uints, it's only s1 that is a little
260 : problematic, because it references items in the same vector.
261 : Thankfully, the msg2 intrinsic takes care of the complexity, but we
262 : need to execute it last.
263 :
264 : We get w[i-16] and s0(s[i-15]) using the msg1 intrinsic, setting j =
265 : i-16. For example, to compute w1013, we pass in w0003 and w0407.
266 : Then we can get w[i-7] by using the alignr instruction on
267 : (w[i-8], w[i-7], w[i-6], w[i-5]) and (w[i-4], w[i-3], w[i-2], w[i-1])
268 : to concatenate them and shift by one uint. Continuing with the
269 : example of w1013, we need w080b and w0c0f. We then put
270 : v[i] = w[i-16] + s0(w[i-15]) + w[i-7],
271 : and invoke the msg2 intrinsic with j=i, which gives w[i], as desired.
272 : Each invocation of NEXT_W computes 4 values of w. */
273 :
274 122484768 : #define NEXT_W( w_minus_16, w_minus_12, w_minus_8, w_minus_4 ) (__extension__({ \
275 122484768 : vu_t __w_i_16_s0_i_15 = _mm_sha256msg1_epu32( w_minus_16, w_minus_12 ); \
276 122484768 : vu_t __w_i_7 = _mm_alignr_epi8( w_minus_4, w_minus_8, 4 ); \
277 122484768 : _mm_sha256msg2_epu32( vu_add( __w_i_7, __w_i_16_s0_i_15 ), w_minus_4 ); \
278 122484768 : }))
279 :
280 : /* Zen 5's sha256rnds2 has an RTP of 2, while Zen 4's has an RTP of 1. We can
281 : win some performance by moving the schedule updates earlier in the loop. */
282 : #ifdef __znver5__
283 : #define FULL_ROUNDS() do { \
284 : vu_t w1013 = NEXT_W( w0003, w0407, w080b, w0c0f ); FOUR_ROUNDS( vu_add( w0003, vu_ld( fd_sha256_K+ 0UL ) ) ); \
285 : vu_t w1417 = NEXT_W( w0407, w080b, w0c0f, w1013 ); FOUR_ROUNDS( vu_add( w0407, vu_ld( fd_sha256_K+ 4UL ) ) ); \
286 : vu_t w181b = NEXT_W( w080b, w0c0f, w1013, w1417 ); FOUR_ROUNDS( vu_add( w080b, vu_ld( fd_sha256_K+ 8UL ) ) ); \
287 : vu_t w1c1f = NEXT_W( w0c0f, w1013, w1417, w181b ); FOUR_ROUNDS( vu_add( w0c0f, vu_ld( fd_sha256_K+12UL ) ) ); \
288 : vu_t w2023 = NEXT_W( w1013, w1417, w181b, w1c1f ); FOUR_ROUNDS( vu_add( w1013, vu_ld( fd_sha256_K+16UL ) ) ); \
289 : vu_t w2427 = NEXT_W( w1417, w181b, w1c1f, w2023 ); FOUR_ROUNDS( vu_add( w1417, vu_ld( fd_sha256_K+20UL ) ) ); \
290 : vu_t w282b = NEXT_W( w181b, w1c1f, w2023, w2427 ); FOUR_ROUNDS( vu_add( w181b, vu_ld( fd_sha256_K+24UL ) ) ); \
291 : vu_t w2c2f = NEXT_W( w1c1f, w2023, w2427, w282b ); FOUR_ROUNDS( vu_add( w1c1f, vu_ld( fd_sha256_K+28UL ) ) ); \
292 : vu_t w3033 = NEXT_W( w2023, w2427, w282b, w2c2f ); FOUR_ROUNDS( vu_add( w2023, vu_ld( fd_sha256_K+32UL ) ) ); \
293 : vu_t w3437 = NEXT_W( w2427, w282b, w2c2f, w3033 ); FOUR_ROUNDS( vu_add( w2427, vu_ld( fd_sha256_K+36UL ) ) ); \
294 : vu_t w383b = NEXT_W( w282b, w2c2f, w3033, w3437 ); FOUR_ROUNDS( vu_add( w282b, vu_ld( fd_sha256_K+40UL ) ) ); \
295 : vu_t w3c3f = NEXT_W( w2c2f, w3033, w3437, w383b ); FOUR_ROUNDS( vu_add( w2c2f, vu_ld( fd_sha256_K+44UL ) ) ); \
296 : /* */ FOUR_ROUNDS( vu_add( w3033, vu_ld( fd_sha256_K+48UL ) ) ); \
297 : /* */ FOUR_ROUNDS( vu_add( w3437, vu_ld( fd_sha256_K+52UL ) ) ); \
298 : /* */ FOUR_ROUNDS( vu_add( w383b, vu_ld( fd_sha256_K+56UL ) ) ); \
299 : /* */ FOUR_ROUNDS( vu_add( w3c3f, vu_ld( fd_sha256_K+60UL ) ) ); \
300 : } while ( 0 )
301 : #else
302 10207064 : #define FULL_ROUNDS() do { \
303 10207064 : /* */ FOUR_ROUNDS( vu_add( w0003, vu_ld( fd_sha256_K+ 0UL ) ) ); \
304 10207064 : /* */ FOUR_ROUNDS( vu_add( w0407, vu_ld( fd_sha256_K+ 4UL ) ) ); \
305 10207064 : /* */ FOUR_ROUNDS( vu_add( w080b, vu_ld( fd_sha256_K+ 8UL ) ) ); \
306 10207064 : /* */ FOUR_ROUNDS( vu_add( w0c0f, vu_ld( fd_sha256_K+12UL ) ) ); \
307 10207064 : vu_t w1013 = NEXT_W( w0003, w0407, w080b, w0c0f ); FOUR_ROUNDS( vu_add( w1013, vu_ld( fd_sha256_K+16UL ) ) ); \
308 10207064 : vu_t w1417 = NEXT_W( w0407, w080b, w0c0f, w1013 ); FOUR_ROUNDS( vu_add( w1417, vu_ld( fd_sha256_K+20UL ) ) ); \
309 10207064 : vu_t w181b = NEXT_W( w080b, w0c0f, w1013, w1417 ); FOUR_ROUNDS( vu_add( w181b, vu_ld( fd_sha256_K+24UL ) ) ); \
310 10207064 : vu_t w1c1f = NEXT_W( w0c0f, w1013, w1417, w181b ); FOUR_ROUNDS( vu_add( w1c1f, vu_ld( fd_sha256_K+28UL ) ) ); \
311 10207064 : vu_t w2023 = NEXT_W( w1013, w1417, w181b, w1c1f ); FOUR_ROUNDS( vu_add( w2023, vu_ld( fd_sha256_K+32UL ) ) ); \
312 10207064 : vu_t w2427 = NEXT_W( w1417, w181b, w1c1f, w2023 ); FOUR_ROUNDS( vu_add( w2427, vu_ld( fd_sha256_K+36UL ) ) ); \
313 10207064 : vu_t w282b = NEXT_W( w181b, w1c1f, w2023, w2427 ); FOUR_ROUNDS( vu_add( w282b, vu_ld( fd_sha256_K+40UL ) ) ); \
314 10207064 : vu_t w2c2f = NEXT_W( w1c1f, w2023, w2427, w282b ); FOUR_ROUNDS( vu_add( w2c2f, vu_ld( fd_sha256_K+44UL ) ) ); \
315 10207064 : vu_t w3033 = NEXT_W( w2023, w2427, w282b, w2c2f ); FOUR_ROUNDS( vu_add( w3033, vu_ld( fd_sha256_K+48UL ) ) ); \
316 10207064 : vu_t w3437 = NEXT_W( w2427, w282b, w2c2f, w3033 ); FOUR_ROUNDS( vu_add( w3437, vu_ld( fd_sha256_K+52UL ) ) ); \
317 10207064 : vu_t w383b = NEXT_W( w282b, w2c2f, w3033, w3437 ); FOUR_ROUNDS( vu_add( w383b, vu_ld( fd_sha256_K+56UL ) ) ); \
318 10207064 : vu_t w3c3f = NEXT_W( w2c2f, w3033, w3437, w383b ); FOUR_ROUNDS( vu_add( w3c3f, vu_ld( fd_sha256_K+60UL ) ) ); \
319 10207064 : } while ( 0 )
320 : #endif
321 :
322 :
323 : void
324 : fd_sha256_core_shaext( uint * state, /* 64-byte aligned, 8 entries */
325 : uchar const * block, /* ideally 128-byte aligned (but not required), 64*block_cnt in size */
326 7920912 : ulong block_cnt ) {/* positive */
327 7920912 : vu_t stateABCD = vu_ld( state );
328 7920912 : vu_t stateEFGH = vu_ld( state+4UL );
329 :
330 7920912 : vu_t baseFEBA = vu_permute2( stateEFGH, stateABCD, 1, 0, 1, 0 );
331 7920912 : vu_t baseHGDC = vu_permute2( stateEFGH, stateABCD, 3, 2, 3, 2 );
332 :
333 18127976 : for( ulong b=0UL; b<block_cnt; b++ ) {
334 10207064 : vu_t stateFEBA = baseFEBA;
335 10207064 : vu_t stateHGDC = baseHGDC;
336 :
337 10207064 : vu_t w0003 = vu_bswap( vu_ldu( block+64UL*b ) );
338 10207064 : vu_t w0407 = vu_bswap( vu_ldu( block+64UL*b+16UL ) );
339 10207064 : vu_t w080b = vu_bswap( vu_ldu( block+64UL*b+32UL ) );
340 10207064 : vu_t w0c0f = vu_bswap( vu_ldu( block+64UL*b+48UL ) );
341 :
342 10207064 : FULL_ROUNDS();
343 :
344 10207064 : baseFEBA = vu_add( baseFEBA, stateFEBA );
345 10207064 : baseHGDC = vu_add( baseHGDC, stateHGDC );
346 :
347 10207064 : }
348 :
349 7920912 : stateABCD = vu_permute2( baseFEBA, baseHGDC, 3, 2, 3, 2 );
350 7920912 : stateEFGH = vu_permute2( baseFEBA, baseHGDC, 1, 0, 1, 0 );
351 7920912 : vu_st( state, stateABCD );
352 7920912 : vu_st( state+4UL, stateEFGH );
353 7920912 : }
354 :
355 : #undef FULL_ROUNDS
356 : #undef NEXT_W
357 : #undef FOUR_ROUNDS
358 :
359 7920912 : #define fd_sha256_core fd_sha256_core_shaext
360 :
361 : #elif FD_SHA256_CORE_IMPL==2
362 :
363 : void
364 : fd_sha256_core_arm( uint * state,
365 : uchar const * block,
366 : ulong block_cnt );
367 :
368 : #define fd_sha256_core fd_sha256_core_arm
369 :
370 : #elif FD_SHA256_CORE_IMPL==3
371 :
372 : void
373 : fd_sha256_core_riscv( uint * state,
374 : uchar const * block,
375 : ulong block_cnt );
376 :
377 : #define fd_sha256_core fd_sha256_core_riscv
378 :
379 : #else
380 : #error "Unsupported FD_SHA256_CORE_IMPL"
381 : #endif
382 :
383 : fd_sha256_t *
384 1252197 : fd_sha256_init( fd_sha256_t * sha ) {
385 1252197 : sha->state[0] = FD_SHA256_INITIAL_A;
386 1252197 : sha->state[1] = FD_SHA256_INITIAL_B;
387 1252197 : sha->state[2] = FD_SHA256_INITIAL_C;
388 1252197 : sha->state[3] = FD_SHA256_INITIAL_D;
389 1252197 : sha->state[4] = FD_SHA256_INITIAL_E;
390 1252197 : sha->state[5] = FD_SHA256_INITIAL_F;
391 1252197 : sha->state[6] = FD_SHA256_INITIAL_G;
392 1252197 : sha->state[7] = FD_SHA256_INITIAL_H;
393 1252197 : sha->buf_used = 0UL;
394 1252197 : sha->bit_cnt = 0UL;
395 1252197 : return sha;
396 1252197 : }
397 :
398 : fd_sha256_t *
399 : fd_sha256_append( fd_sha256_t * sha,
400 : void const * _data,
401 2677614 : ulong sz ) {
402 :
403 : /* If no data to append, we are done */
404 :
405 2677614 : if( FD_UNLIKELY( !sz ) ) return sha; /* optimize for non-trivial append */
406 :
407 : /* Unpack inputs */
408 :
409 2677368 : uint * state = sha->state;
410 2677368 : uchar * buf = sha->buf;
411 2677368 : ulong buf_used = sha->buf_used;
412 2677368 : ulong bit_cnt = sha->bit_cnt;
413 :
414 2677368 : uchar const * data = (uchar const *)_data;
415 :
416 : /* Update bit_cnt */
417 : /* FIXME: could accumulate bytes here and do bit conversion in append */
418 : /* FIXME: Overflow handling if more than 2^64 bits (unlikely) */
419 :
420 2677368 : sha->bit_cnt = bit_cnt + (sz<<3);
421 :
422 : /* Handle buffered bytes from previous appends */
423 :
424 2677368 : if( FD_UNLIKELY( buf_used ) ) { /* optimized for well aligned use of append */
425 :
426 : /* If the append isn't large enough to complete the current block,
427 : buffer these bytes too and return */
428 :
429 309359 : ulong buf_rem = FD_SHA256_PRIVATE_BUF_MAX - buf_used; /* In (0,FD_SHA256_PRIVATE_BUF_MAX) */
430 309359 : if( FD_UNLIKELY( sz < buf_rem ) ) { /* optimize for large append */
431 142494 : fd_memcpy( buf + buf_used, data, sz );
432 142494 : sha->buf_used = buf_used + sz;
433 142494 : return sha;
434 142494 : }
435 :
436 : /* Otherwise, buffer enough leading bytes of data to complete the
437 : block, update the hash and then continue processing any remaining
438 : bytes of data. */
439 :
440 166865 : fd_memcpy( buf + buf_used, data, buf_rem );
441 166865 : data += buf_rem;
442 166865 : sz -= buf_rem;
443 :
444 166865 : fd_sha256_core( state, buf, 1UL );
445 166865 : sha->buf_used = 0UL;
446 166865 : }
447 :
448 : /* Append the bulk of the data */
449 :
450 2534874 : ulong block_cnt = sz >> FD_SHA256_PRIVATE_LG_BUF_MAX;
451 2534874 : if( FD_LIKELY( block_cnt ) ) fd_sha256_core( state, data, block_cnt ); /* optimized for large append */
452 :
453 : /* Buffer any leftover bytes */
454 :
455 2534874 : buf_used = sz & (FD_SHA256_PRIVATE_BUF_MAX-1UL); /* In [0,FD_SHA256_PRIVATE_BUF_MAX) */
456 2534874 : if( FD_UNLIKELY( buf_used ) ) { /* optimized for well aligned use of append */
457 1418752 : fd_memcpy( buf, data + (block_cnt << FD_SHA256_PRIVATE_LG_BUF_MAX), buf_used );
458 1418752 : sha->buf_used = buf_used; /* In (0,FD_SHA256_PRIVATE_BUF_MAX) */
459 1418752 : }
460 :
461 2534874 : return sha;
462 2677368 : }
463 :
464 : void *
465 : fd_sha256_fini( fd_sha256_t * sha,
466 1318026 : void * _hash ) {
467 :
468 : /* Unpack inputs */
469 :
470 1318026 : uint * state = sha->state;
471 1318026 : uchar * buf = sha->buf;
472 1318026 : ulong buf_used = sha->buf_used; /* In [0,FD_SHA256_PRIVATE_BUF_MAX) */
473 1318026 : ulong bit_cnt = sha->bit_cnt;
474 :
475 : /* Append the terminating message byte */
476 :
477 1318026 : buf[ buf_used ] = (uchar)0x80;
478 1318026 : buf_used++;
479 :
480 : /* If there isn't enough room to save the message length in bits at
481 : the end of the in progress block, clear the rest of the in progress
482 : block, update the hash and start a new block. */
483 :
484 1318026 : if( FD_UNLIKELY( buf_used > (FD_SHA256_PRIVATE_BUF_MAX-8UL) ) ) { /* optimize for well aligned use of append */
485 16814 : fd_memset( buf + buf_used, 0, FD_SHA256_PRIVATE_BUF_MAX-buf_used );
486 16814 : fd_sha256_core( state, buf, 1UL );
487 16814 : buf_used = 0UL;
488 16814 : }
489 :
490 : /* Clear in progress block up to last 64-bits, append the message
491 : size in bytes in the last 64-bits of the in progress block and
492 : update the hash to finalize it. */
493 :
494 1318026 : fd_memset( buf + buf_used, 0, FD_SHA256_PRIVATE_BUF_MAX-8UL-buf_used );
495 1318026 : FD_STORE( ulong, buf+FD_SHA256_PRIVATE_BUF_MAX-8UL, fd_ulong_bswap( bit_cnt ) );
496 1318026 : fd_sha256_core( state, buf, 1UL );
497 :
498 : /* Unpack the result into md (annoying bswaps here) */
499 :
500 1318026 : state[0] = fd_uint_bswap( state[0] );
501 1318026 : state[1] = fd_uint_bswap( state[1] );
502 1318026 : state[2] = fd_uint_bswap( state[2] );
503 1318026 : state[3] = fd_uint_bswap( state[3] );
504 1318026 : state[4] = fd_uint_bswap( state[4] );
505 1318026 : state[5] = fd_uint_bswap( state[5] );
506 1318026 : state[6] = fd_uint_bswap( state[6] );
507 1318026 : state[7] = fd_uint_bswap( state[7] );
508 1318026 : return memcpy( _hash, state, 32 );
509 1318026 : }
510 :
511 : void *
512 : fd_sha256_hash( void const * _data,
513 : ulong sz,
514 15328070 : void * _hash ) {
515 15328070 : uchar const * data = (uchar const *)_data;
516 :
517 : /* This is just the above streamlined to eliminate all the overheads
518 : to support incremental hashing. */
519 :
520 15328070 : uchar buf[ FD_SHA256_PRIVATE_BUF_MAX ] __attribute__((aligned(128)));
521 15328070 : uint state[8] __attribute__((aligned(32)));
522 :
523 15328070 : state[0] = FD_SHA256_INITIAL_A;
524 15328070 : state[1] = FD_SHA256_INITIAL_B;
525 15328070 : state[2] = FD_SHA256_INITIAL_C;
526 15328070 : state[3] = FD_SHA256_INITIAL_D;
527 15328070 : state[4] = FD_SHA256_INITIAL_E;
528 15328070 : state[5] = FD_SHA256_INITIAL_F;
529 15328070 : state[6] = FD_SHA256_INITIAL_G;
530 15328070 : state[7] = FD_SHA256_INITIAL_H;
531 :
532 15328070 : ulong block_cnt = sz >> FD_SHA256_PRIVATE_LG_BUF_MAX;
533 15328070 : if( FD_LIKELY( block_cnt ) ) fd_sha256_core( state, data, block_cnt );
534 :
535 15328070 : ulong buf_used = sz & (FD_SHA256_PRIVATE_BUF_MAX-1UL);
536 15328070 : if( FD_UNLIKELY( buf_used ) ) fd_memcpy( buf, data + (block_cnt << FD_SHA256_PRIVATE_LG_BUF_MAX), buf_used );
537 15328070 : buf[ buf_used ] = (uchar)0x80;
538 15328070 : buf_used++;
539 :
540 15328070 : if( FD_UNLIKELY( buf_used > (FD_SHA256_PRIVATE_BUF_MAX-8UL) ) ) {
541 164258 : fd_memset( buf + buf_used, 0, FD_SHA256_PRIVATE_BUF_MAX-buf_used );
542 164258 : fd_sha256_core( state, buf, 1UL );
543 164258 : buf_used = 0UL;
544 164258 : }
545 :
546 15328070 : ulong bit_cnt = sz << 3;
547 15328070 : fd_memset( buf + buf_used, 0, FD_SHA256_PRIVATE_BUF_MAX-8UL-buf_used );
548 15328070 : FD_STORE( ulong, buf+FD_SHA256_PRIVATE_BUF_MAX-8UL, fd_ulong_bswap( bit_cnt ) );
549 15328070 : fd_sha256_core( state, buf, 1UL );
550 :
551 15328070 : state[0] = fd_uint_bswap( state[0] );
552 15328070 : state[1] = fd_uint_bswap( state[1] );
553 15328070 : state[2] = fd_uint_bswap( state[2] );
554 15328070 : state[3] = fd_uint_bswap( state[3] );
555 15328070 : state[4] = fd_uint_bswap( state[4] );
556 15328070 : state[5] = fd_uint_bswap( state[5] );
557 15328070 : state[6] = fd_uint_bswap( state[6] );
558 15328070 : state[7] = fd_uint_bswap( state[7] );
559 15328070 : return memcpy( _hash, state, 32 );
560 15328070 : }
561 :
562 : #undef fd_sha256_core
|
