Line data Source code
1 : #include "fd_sha512.h"
2 :
3 : #if FD_HAS_ARM_SHA512
4 : #include <arm_neon.h>
5 : #endif
6 :
7 : ulong
8 11079 : fd_sha512_align( void ) {
9 11079 : return FD_SHA512_ALIGN;
10 11079 : }
11 :
12 : ulong
13 5589 : fd_sha512_footprint( void ) {
14 5589 : return FD_SHA512_FOOTPRINT;
15 5589 : }
16 :
17 : void *
18 5166 : fd_sha512_new( void * shmem ) {
19 5166 : fd_sha512_t * sha = (fd_sha512_t *)shmem;
20 :
21 5166 : if( FD_UNLIKELY( !shmem ) ) {
22 3 : FD_LOG_WARNING(( "NULL shmem" ));
23 3 : return NULL;
24 3 : }
25 :
26 5163 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)shmem, fd_sha512_align() ) ) ) {
27 3 : FD_LOG_WARNING(( "misaligned shmem" ));
28 3 : return NULL;
29 3 : }
30 :
31 5160 : ulong footprint = fd_sha512_footprint();
32 :
33 5160 : fd_memset( sha, 0, footprint );
34 :
35 5160 : FD_COMPILER_MFENCE();
36 5160 : FD_VOLATILE( sha->magic ) = FD_SHA512_MAGIC;
37 5160 : FD_COMPILER_MFENCE();
38 :
39 5160 : return (void *)sha;
40 5163 : }
41 :
42 : fd_sha512_t *
43 5124 : fd_sha512_join( void * shsha ) {
44 :
45 5124 : if( FD_UNLIKELY( !shsha ) ) {
46 3 : FD_LOG_WARNING(( "NULL shsha" ));
47 3 : return NULL;
48 3 : }
49 :
50 5121 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)shsha, fd_sha512_align() ) ) ) {
51 3 : FD_LOG_WARNING(( "misaligned shsha" ));
52 3 : return NULL;
53 3 : }
54 :
55 5118 : fd_sha512_t * sha = (fd_sha512_t *)shsha;
56 :
57 5118 : if( FD_UNLIKELY( sha->magic!=FD_SHA512_MAGIC ) ) {
58 0 : FD_LOG_WARNING(( "bad magic" ));
59 0 : return NULL;
60 0 : }
61 :
62 5118 : return sha;
63 5118 : }
64 :
65 : void *
66 42 : fd_sha512_leave( fd_sha512_t * sha ) {
67 :
68 42 : if( FD_UNLIKELY( !sha ) ) {
69 3 : FD_LOG_WARNING(( "NULL sha" ));
70 3 : return NULL;
71 3 : }
72 :
73 39 : return (void *)sha;
74 42 : }
75 :
76 : void *
77 45 : fd_sha512_delete( void * shsha ) {
78 :
79 45 : if( FD_UNLIKELY( !shsha ) ) {
80 3 : FD_LOG_WARNING(( "NULL shsha" ));
81 3 : return NULL;
82 3 : }
83 :
84 42 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)shsha, fd_sha512_align() ) ) ) {
85 3 : FD_LOG_WARNING(( "misaligned shsha" ));
86 3 : return NULL;
87 3 : }
88 :
89 39 : fd_sha512_t * sha = (fd_sha512_t *)shsha;
90 :
91 39 : if( FD_UNLIKELY( sha->magic!=FD_SHA512_MAGIC ) ) {
92 0 : FD_LOG_WARNING(( "bad magic" ));
93 0 : return NULL;
94 0 : }
95 :
96 39 : FD_COMPILER_MFENCE();
97 39 : FD_VOLATILE( sha->magic ) = 0UL;
98 39 : FD_COMPILER_MFENCE();
99 :
100 39 : return (void *)sha;
101 39 : }
102 :
103 : #ifndef FD_SHA512_CORE_IMPL
104 : #if FD_HAS_ARM_SHA512
105 : #define FD_SHA512_CORE_IMPL 2
106 : #elif FD_HAS_AVX
107 : #define FD_SHA512_CORE_IMPL 1
108 : #elif FD_HAS_RISCV_SHA512
109 : #define FD_SHA512_CORE_IMPL 3
110 : #else
111 : #define FD_SHA512_CORE_IMPL 0
112 : #endif
113 : #endif
114 :
115 : #if FD_SHA512_CORE_IMPL==0
116 :
117 : /* The implementation below was derived from OpenSSL's sha512
118 : implementation (Apache 2 licensed). See in particular:
119 :
120 : https://github.com/openssl/openssl/blob/master/crypto/sha/sha512.c
121 :
122 : (link valid circa 2022-Oct). It has been made more strict with more
123 : extensive implementation documentation, has been simplified and has
124 : been streamlined specifically for use inside Firedancer base machine
125 : model (no machine specific capabilities required).
126 :
127 : In particular, fd_sha512_core_ref is based on OpenSSL's
128 : OPENSSL_SMALL_FOOTPRINT SHA-512 implementation (Apache licensed).
129 : This should work anywhere but it is not the highest performance
130 : implementation possible.
131 :
132 : It is also straightforward to replace these implementations with HPC
133 : implementations that target specific machine capabilities without
134 : requiring any changes to caller code. */
135 :
136 : static void
137 : fd_sha512_core_ref( ulong * state, /* 64-byte aligned, 8 entries */
138 : uchar const * block, /* ideally 128-byte aligned (but not required), 128*block_cnt in size */
139 : ulong block_cnt ) { /* positive */
140 :
141 : static ulong const K[80] = {
142 : 0x428a2f98d728ae22UL, 0x7137449123ef65cdUL, 0xb5c0fbcfec4d3b2fUL, 0xe9b5dba58189dbbcUL,
143 : 0x3956c25bf348b538UL, 0x59f111f1b605d019UL, 0x923f82a4af194f9bUL, 0xab1c5ed5da6d8118UL,
144 : 0xd807aa98a3030242UL, 0x12835b0145706fbeUL, 0x243185be4ee4b28cUL, 0x550c7dc3d5ffb4e2UL,
145 : 0x72be5d74f27b896fUL, 0x80deb1fe3b1696b1UL, 0x9bdc06a725c71235UL, 0xc19bf174cf692694UL,
146 : 0xe49b69c19ef14ad2UL, 0xefbe4786384f25e3UL, 0x0fc19dc68b8cd5b5UL, 0x240ca1cc77ac9c65UL,
147 : 0x2de92c6f592b0275UL, 0x4a7484aa6ea6e483UL, 0x5cb0a9dcbd41fbd4UL, 0x76f988da831153b5UL,
148 : 0x983e5152ee66dfabUL, 0xa831c66d2db43210UL, 0xb00327c898fb213fUL, 0xbf597fc7beef0ee4UL,
149 : 0xc6e00bf33da88fc2UL, 0xd5a79147930aa725UL, 0x06ca6351e003826fUL, 0x142929670a0e6e70UL,
150 : 0x27b70a8546d22ffcUL, 0x2e1b21385c26c926UL, 0x4d2c6dfc5ac42aedUL, 0x53380d139d95b3dfUL,
151 : 0x650a73548baf63deUL, 0x766a0abb3c77b2a8UL, 0x81c2c92e47edaee6UL, 0x92722c851482353bUL,
152 : 0xa2bfe8a14cf10364UL, 0xa81a664bbc423001UL, 0xc24b8b70d0f89791UL, 0xc76c51a30654be30UL,
153 : 0xd192e819d6ef5218UL, 0xd69906245565a910UL, 0xf40e35855771202aUL, 0x106aa07032bbd1b8UL,
154 : 0x19a4c116b8d2d0c8UL, 0x1e376c085141ab53UL, 0x2748774cdf8eeb99UL, 0x34b0bcb5e19b48a8UL,
155 : 0x391c0cb3c5c95a63UL, 0x4ed8aa4ae3418acbUL, 0x5b9cca4f7763e373UL, 0x682e6ff3d6b2b8a3UL,
156 : 0x748f82ee5defb2fcUL, 0x78a5636f43172f60UL, 0x84c87814a1f0ab72UL, 0x8cc702081a6439ecUL,
157 : 0x90befffa23631e28UL, 0xa4506cebde82bde9UL, 0xbef9a3f7b2c67915UL, 0xc67178f2e372532bUL,
158 : 0xca273eceea26619cUL, 0xd186b8c721c0c207UL, 0xeada7dd6cde0eb1eUL, 0xf57d4f7fee6ed178UL,
159 : 0x06f067aa72176fbaUL, 0x0a637dc5a2c898a6UL, 0x113f9804bef90daeUL, 0x1b710b35131c471bUL,
160 : 0x28db77f523047d84UL, 0x32caab7b40c72493UL, 0x3c9ebe0a15c9bebcUL, 0x431d67c49c100d4cUL,
161 : 0x4cc5d4becb3e42b6UL, 0x597f299cfc657e2aUL, 0x5fcb6fab3ad6faecUL, 0x6c44198c4a475817UL
162 : };
163 :
164 : # define ROTR fd_ulong_rotate_right
165 : # define Sigma0(x) (ROTR((x),28) ^ ROTR((x),34) ^ ROTR((x),39))
166 : # define Sigma1(x) (ROTR((x),14) ^ ROTR((x),18) ^ ROTR((x),41))
167 : # define sigma0(x) (ROTR((x), 1) ^ ROTR((x), 8) ^ ((x)>>7))
168 : # define sigma1(x) (ROTR((x),19) ^ ROTR((x),61) ^ ((x)>>6))
169 : # define Ch(x,y,z) (((x) & (y)) ^ ((~(x)) & (z)))
170 : # define Maj(x,y,z) (((x) & (y)) ^ ((x) & (z)) ^ ((y) & (z)))
171 :
172 : uchar const * W = block;
173 : do {
174 : ulong a = state[0];
175 : ulong b = state[1];
176 : ulong c = state[2];
177 : ulong d = state[3];
178 : ulong e = state[4];
179 : ulong f = state[5];
180 : ulong g = state[6];
181 : ulong h = state[7];
182 :
183 : ulong X[16];
184 :
185 : ulong i;
186 : for( i=0UL; i<16UL; i++ ) {
187 : X[i] = fd_ulong_bswap( FD_LOAD( ulong, W + i*sizeof(ulong) ) );
188 : ulong T1 = X[i] + h + Sigma1(e) + Ch(e, f, g) + K[i];
189 : ulong T2 = Sigma0(a) + Maj(a, b, c);
190 : h = g;
191 : g = f;
192 : f = e;
193 : e = d + T1;
194 : d = c;
195 : c = b;
196 : b = a;
197 : a = T1 + T2;
198 : }
199 : for( ; i<80UL; i++ ) {
200 : ulong s0 = X[(i + 1UL) & 0x0fUL];
201 : ulong s1 = X[(i + 14UL) & 0x0fUL];
202 : s0 = sigma0(s0);
203 : s1 = sigma1(s1);
204 : X[i & 0xfUL] += s0 + s1 + X[(i + 9UL) & 0xfUL];
205 : ulong T1 = X[i & 0xfUL ] + h + Sigma1(e) + Ch(e, f, g) + K[i];
206 : ulong T2 = Sigma0(a) + Maj(a, b, c);
207 : h = g;
208 : g = f;
209 : f = e;
210 : e = d + T1;
211 : d = c;
212 : c = b;
213 : b = a;
214 : a = T1 + T2;
215 : }
216 :
217 : state[0] += a;
218 : state[1] += b;
219 : state[2] += c;
220 : state[3] += d;
221 : state[4] += e;
222 : state[5] += f;
223 : state[6] += g;
224 : state[7] += h;
225 :
226 : W += 16UL*sizeof(ulong);
227 : } while( --block_cnt );
228 :
229 : # undef ROTR
230 : # undef Sigma0
231 : # undef Sigma1
232 : # undef sigma0
233 : # undef sigma1
234 : # undef Ch
235 : # undef Maj
236 :
237 : }
238 :
239 : #define fd_sha512_core fd_sha512_core_ref
240 :
241 : #elif FD_SHA512_CORE_IMPL==1
242 :
243 : __attribute__((sysv_abi))
244 : void
245 : fd_sha512_core_avx2( ulong * state, /* 64-byte aligned, 8 entries */
246 : uchar const * block, /* ideally 128-byte aligned (but not required), 128*block_cnt in size */
247 : ulong block_cnt ); /* positive */
248 :
249 10011588 : #define fd_sha512_core fd_sha512_core_avx2
250 :
251 : #elif FD_SHA512_CORE_IMPL==2
252 :
253 : static void
254 : fd_sha512_core_arm( ulong * state,
255 : uchar const * block,
256 : ulong block_cnt ) {
257 :
258 : static ulong const K[80] = {
259 : 0x428a2f98d728ae22UL, 0x7137449123ef65cdUL, 0xb5c0fbcfec4d3b2fUL, 0xe9b5dba58189dbbcUL,
260 : 0x3956c25bf348b538UL, 0x59f111f1b605d019UL, 0x923f82a4af194f9bUL, 0xab1c5ed5da6d8118UL,
261 : 0xd807aa98a3030242UL, 0x12835b0145706fbeUL, 0x243185be4ee4b28cUL, 0x550c7dc3d5ffb4e2UL,
262 : 0x72be5d74f27b896fUL, 0x80deb1fe3b1696b1UL, 0x9bdc06a725c71235UL, 0xc19bf174cf692694UL,
263 : 0xe49b69c19ef14ad2UL, 0xefbe4786384f25e3UL, 0x0fc19dc68b8cd5b5UL, 0x240ca1cc77ac9c65UL,
264 : 0x2de92c6f592b0275UL, 0x4a7484aa6ea6e483UL, 0x5cb0a9dcbd41fbd4UL, 0x76f988da831153b5UL,
265 : 0x983e5152ee66dfabUL, 0xa831c66d2db43210UL, 0xb00327c898fb213fUL, 0xbf597fc7beef0ee4UL,
266 : 0xc6e00bf33da88fc2UL, 0xd5a79147930aa725UL, 0x06ca6351e003826fUL, 0x142929670a0e6e70UL,
267 : 0x27b70a8546d22ffcUL, 0x2e1b21385c26c926UL, 0x4d2c6dfc5ac42aedUL, 0x53380d139d95b3dfUL,
268 : 0x650a73548baf63deUL, 0x766a0abb3c77b2a8UL, 0x81c2c92e47edaee6UL, 0x92722c851482353bUL,
269 : 0xa2bfe8a14cf10364UL, 0xa81a664bbc423001UL, 0xc24b8b70d0f89791UL, 0xc76c51a30654be30UL,
270 : 0xd192e819d6ef5218UL, 0xd69906245565a910UL, 0xf40e35855771202aUL, 0x106aa07032bbd1b8UL,
271 : 0x19a4c116b8d2d0c8UL, 0x1e376c085141ab53UL, 0x2748774cdf8eeb99UL, 0x34b0bcb5e19b48a8UL,
272 : 0x391c0cb3c5c95a63UL, 0x4ed8aa4ae3418acbUL, 0x5b9cca4f7763e373UL, 0x682e6ff3d6b2b8a3UL,
273 : 0x748f82ee5defb2fcUL, 0x78a5636f43172f60UL, 0x84c87814a1f0ab72UL, 0x8cc702081a6439ecUL,
274 : 0x90befffa23631e28UL, 0xa4506cebde82bde9UL, 0xbef9a3f7b2c67915UL, 0xc67178f2e372532bUL,
275 : 0xca273eceea26619cUL, 0xd186b8c721c0c207UL, 0xeada7dd6cde0eb1eUL, 0xf57d4f7fee6ed178UL,
276 : 0x06f067aa72176fbaUL, 0x0a637dc5a2c898a6UL, 0x113f9804bef90daeUL, 0x1b710b35131c471bUL,
277 : 0x28db77f523047d84UL, 0x32caab7b40c72493UL, 0x3c9ebe0a15c9bebcUL, 0x431d67c49c100d4cUL,
278 : 0x4cc5d4becb3e42b6UL, 0x597f299cfc657e2aUL, 0x5fcb6fab3ad6faecUL, 0x6c44198c4a475817UL
279 : };
280 :
281 : #define SHA512_ROUNDS( MSG, KIDX ) do { \
282 : uint64x2_t ab_prev = ab; \
283 : uint64x2_t ef_prev = ef; \
284 : uint64x2_t wk = vaddq_u64( (MSG), vld1q_u64( K + (KIDX) ) ); \
285 : wk = vextq_u64( wk, wk, 1 ); \
286 : uint64x2_t fg = vextq_u64( ef, gh, 1 ); \
287 : uint64x2_t de = vextq_u64( cd, ef, 1 ); \
288 : gh = vaddq_u64( gh, wk ); \
289 : gh = vsha512hq_u64( gh, fg, de ); \
290 : uint64x2_t new_ef = vaddq_u64( cd, gh ); \
291 : gh = vsha512h2q_u64( gh, cd, ab ); \
292 : ab = gh; \
293 : cd = ab_prev; \
294 : ef = new_ef; \
295 : gh = ef_prev; \
296 : } while( 0 )
297 :
298 : #define SHA512_SCHEDULE( MSG0, MSG1, MSG4, MSG5, MSG7 ) do { \
299 : (MSG0) = vsha512su0q_u64( (MSG0), (MSG1) ); \
300 : uint64x2_t w9_10 = vextq_u64( (MSG4), (MSG5), 1 ); \
301 : (MSG0) = vsha512su1q_u64( (MSG0), (MSG7), w9_10 ); \
302 : } while( 0 )
303 :
304 : do {
305 : uint64x2_t ab = vld1q_u64( state );
306 : uint64x2_t cd = vld1q_u64( state+2 );
307 : uint64x2_t ef = vld1q_u64( state+4 );
308 : uint64x2_t gh = vld1q_u64( state+6 );
309 : uint64x2_t ab_init = ab;
310 : uint64x2_t cd_init = cd;
311 : uint64x2_t ef_init = ef;
312 : uint64x2_t gh_init = gh;
313 :
314 : uint64x2_t msg0 = vreinterpretq_u64_u8( vrev64q_u8( vld1q_u8( block ) ) );
315 : uint64x2_t msg1 = vreinterpretq_u64_u8( vrev64q_u8( vld1q_u8( block+ 16UL ) ) );
316 : uint64x2_t msg2 = vreinterpretq_u64_u8( vrev64q_u8( vld1q_u8( block+ 32UL ) ) );
317 : uint64x2_t msg3 = vreinterpretq_u64_u8( vrev64q_u8( vld1q_u8( block+ 48UL ) ) );
318 : uint64x2_t msg4 = vreinterpretq_u64_u8( vrev64q_u8( vld1q_u8( block+ 64UL ) ) );
319 : uint64x2_t msg5 = vreinterpretq_u64_u8( vrev64q_u8( vld1q_u8( block+ 80UL ) ) );
320 : uint64x2_t msg6 = vreinterpretq_u64_u8( vrev64q_u8( vld1q_u8( block+ 96UL ) ) );
321 : uint64x2_t msg7 = vreinterpretq_u64_u8( vrev64q_u8( vld1q_u8( block+112UL ) ) );
322 :
323 : SHA512_ROUNDS( msg0, 0 ); SHA512_SCHEDULE( msg0, msg1, msg4, msg5, msg7 );
324 : SHA512_ROUNDS( msg1, 2 ); SHA512_SCHEDULE( msg1, msg2, msg5, msg6, msg0 );
325 : SHA512_ROUNDS( msg2, 4 ); SHA512_SCHEDULE( msg2, msg3, msg6, msg7, msg1 );
326 : SHA512_ROUNDS( msg3, 6 ); SHA512_SCHEDULE( msg3, msg4, msg7, msg0, msg2 );
327 : SHA512_ROUNDS( msg4, 8 ); SHA512_SCHEDULE( msg4, msg5, msg0, msg1, msg3 );
328 : SHA512_ROUNDS( msg5, 10 ); SHA512_SCHEDULE( msg5, msg6, msg1, msg2, msg4 );
329 : SHA512_ROUNDS( msg6, 12 ); SHA512_SCHEDULE( msg6, msg7, msg2, msg3, msg5 );
330 : SHA512_ROUNDS( msg7, 14 ); SHA512_SCHEDULE( msg7, msg0, msg3, msg4, msg6 );
331 : SHA512_ROUNDS( msg0, 16 ); SHA512_SCHEDULE( msg0, msg1, msg4, msg5, msg7 );
332 : SHA512_ROUNDS( msg1, 18 ); SHA512_SCHEDULE( msg1, msg2, msg5, msg6, msg0 );
333 : SHA512_ROUNDS( msg2, 20 ); SHA512_SCHEDULE( msg2, msg3, msg6, msg7, msg1 );
334 : SHA512_ROUNDS( msg3, 22 ); SHA512_SCHEDULE( msg3, msg4, msg7, msg0, msg2 );
335 : SHA512_ROUNDS( msg4, 24 ); SHA512_SCHEDULE( msg4, msg5, msg0, msg1, msg3 );
336 : SHA512_ROUNDS( msg5, 26 ); SHA512_SCHEDULE( msg5, msg6, msg1, msg2, msg4 );
337 : SHA512_ROUNDS( msg6, 28 ); SHA512_SCHEDULE( msg6, msg7, msg2, msg3, msg5 );
338 : SHA512_ROUNDS( msg7, 30 ); SHA512_SCHEDULE( msg7, msg0, msg3, msg4, msg6 );
339 : SHA512_ROUNDS( msg0, 32 ); SHA512_SCHEDULE( msg0, msg1, msg4, msg5, msg7 );
340 : SHA512_ROUNDS( msg1, 34 ); SHA512_SCHEDULE( msg1, msg2, msg5, msg6, msg0 );
341 : SHA512_ROUNDS( msg2, 36 ); SHA512_SCHEDULE( msg2, msg3, msg6, msg7, msg1 );
342 : SHA512_ROUNDS( msg3, 38 ); SHA512_SCHEDULE( msg3, msg4, msg7, msg0, msg2 );
343 : SHA512_ROUNDS( msg4, 40 ); SHA512_SCHEDULE( msg4, msg5, msg0, msg1, msg3 );
344 : SHA512_ROUNDS( msg5, 42 ); SHA512_SCHEDULE( msg5, msg6, msg1, msg2, msg4 );
345 : SHA512_ROUNDS( msg6, 44 ); SHA512_SCHEDULE( msg6, msg7, msg2, msg3, msg5 );
346 : SHA512_ROUNDS( msg7, 46 ); SHA512_SCHEDULE( msg7, msg0, msg3, msg4, msg6 );
347 : SHA512_ROUNDS( msg0, 48 ); SHA512_SCHEDULE( msg0, msg1, msg4, msg5, msg7 );
348 : SHA512_ROUNDS( msg1, 50 ); SHA512_SCHEDULE( msg1, msg2, msg5, msg6, msg0 );
349 : SHA512_ROUNDS( msg2, 52 ); SHA512_SCHEDULE( msg2, msg3, msg6, msg7, msg1 );
350 : SHA512_ROUNDS( msg3, 54 ); SHA512_SCHEDULE( msg3, msg4, msg7, msg0, msg2 );
351 : SHA512_ROUNDS( msg4, 56 ); SHA512_SCHEDULE( msg4, msg5, msg0, msg1, msg3 );
352 : SHA512_ROUNDS( msg5, 58 ); SHA512_SCHEDULE( msg5, msg6, msg1, msg2, msg4 );
353 : SHA512_ROUNDS( msg6, 60 ); SHA512_SCHEDULE( msg6, msg7, msg2, msg3, msg5 );
354 : SHA512_ROUNDS( msg7, 62 ); SHA512_SCHEDULE( msg7, msg0, msg3, msg4, msg6 );
355 : SHA512_ROUNDS( msg0, 64 );
356 : SHA512_ROUNDS( msg1, 66 );
357 : SHA512_ROUNDS( msg2, 68 );
358 : SHA512_ROUNDS( msg3, 70 );
359 : SHA512_ROUNDS( msg4, 72 );
360 : SHA512_ROUNDS( msg5, 74 );
361 : SHA512_ROUNDS( msg6, 76 );
362 : SHA512_ROUNDS( msg7, 78 );
363 :
364 : vst1q_u64( state, vaddq_u64( ab, ab_init ) );
365 : vst1q_u64( state+2, vaddq_u64( cd, cd_init ) );
366 : vst1q_u64( state+4, vaddq_u64( ef, ef_init ) );
367 : vst1q_u64( state+6, vaddq_u64( gh, gh_init ) );
368 : block += FD_SHA512_BLOCK_SZ;
369 : } while( --block_cnt );
370 :
371 : #undef SHA512_SCHEDULE
372 : #undef SHA512_ROUNDS
373 : }
374 :
375 : #define fd_sha512_core fd_sha512_core_arm
376 :
377 : #elif FD_SHA512_CORE_IMPL==3
378 :
379 : void
380 : fd_sha512_core_riscv( ulong * state,
381 : uchar const * block,
382 : ulong block_cnt );
383 :
384 : #define fd_sha512_core fd_sha512_core_riscv
385 :
386 : #else
387 : #error "Unsupported FD_SHA512_CORE_IMPL"
388 : #endif
389 :
390 : fd_sha512_t *
391 2118 : fd_sha384_init( fd_sha512_t * sha ) {
392 : /* sha->buf d/c */
393 2118 : sha->state[0] = 0xcbbb9d5dc1059ed8UL;
394 2118 : sha->state[1] = 0x629a292a367cd507UL;
395 2118 : sha->state[2] = 0x9159015a3070dd17UL;
396 2118 : sha->state[3] = 0x152fecd8f70e5939UL;
397 2118 : sha->state[4] = 0x67332667ffc00b31UL;
398 2118 : sha->state[5] = 0x8eb44a8768581511UL;
399 2118 : sha->state[6] = 0xdb0c2e0d64f98fa7UL;
400 2118 : sha->state[7] = 0x47b5481dbefa4fa4UL;
401 2118 : sha->buf_used = 0U;
402 2118 : sha->bit_cnt_lo = 0UL;
403 2118 : sha->bit_cnt_hi = 0UL;
404 2118 : return sha;
405 2118 : }
406 :
407 : fd_sha512_t *
408 1848446 : fd_sha512_init( fd_sha512_t * sha ) {
409 : /* sha->buf d/c */
410 1848446 : sha->state[0] = 0x6a09e667f3bcc908UL;
411 1848446 : sha->state[1] = 0xbb67ae8584caa73bUL;
412 1848446 : sha->state[2] = 0x3c6ef372fe94f82bUL;
413 1848446 : sha->state[3] = 0xa54ff53a5f1d36f1UL;
414 1848446 : sha->state[4] = 0x510e527fade682d1UL;
415 1848446 : sha->state[5] = 0x9b05688c2b3e6c1fUL;
416 1848446 : sha->state[6] = 0x1f83d9abfb41bd6bUL;
417 1848446 : sha->state[7] = 0x5be0cd19137e2179UL;
418 1848446 : sha->buf_used = 0U;
419 1848446 : sha->bit_cnt_lo = 0UL;
420 1848446 : sha->bit_cnt_hi = 0UL;
421 1848446 : return sha;
422 1848446 : }
423 :
424 : fd_sha512_t *
425 : fd_sha512_append( fd_sha512_t * sha,
426 : void const * _data,
427 6496824 : ulong sz ) {
428 :
429 : /* If no data to append, we are done */
430 :
431 6496824 : if( FD_UNLIKELY( !sz ) ) return sha; /* optimize for non-trivial append */
432 :
433 : /* Unpack inputs */
434 :
435 6492213 : ulong * state = sha->state;
436 6492213 : uchar * buf = sha->buf;
437 6492213 : ulong buf_used = sha->buf_used;
438 6492213 : ulong bit_cnt_lo = sha->bit_cnt_lo;
439 6492213 : ulong bit_cnt_hi = sha->bit_cnt_hi;
440 :
441 6492213 : uchar const * data = (uchar const *)_data;
442 :
443 : /* Update bit_cnt */
444 : /* FIXME: could accumulate bytes here and do bit conversion in append */
445 :
446 6492213 : ulong new_bit_cnt_lo = bit_cnt_lo + (sz<< 3);
447 6492213 : ulong new_bit_cnt_hi = bit_cnt_hi + (sz>>61) + (ulong)(new_bit_cnt_lo<bit_cnt_lo);
448 :
449 6492213 : sha->bit_cnt_lo = new_bit_cnt_lo;
450 6492213 : sha->bit_cnt_hi = new_bit_cnt_hi;
451 :
452 : /* Handle buffered bytes from previous appends */
453 :
454 6492213 : if( FD_UNLIKELY( buf_used ) ) { /* optimized for well aligned use of append */
455 :
456 : /* If the append isn't large enough to complete the current block,
457 : buffer these bytes too and return */
458 :
459 1864341 : ulong buf_rem = FD_SHA512_PRIVATE_BUF_MAX - buf_used; /* In (0,FD_SHA512_PRIVATE_BUF_MAX) */
460 1864341 : if( FD_UNLIKELY( sz < buf_rem ) ) { /* optimize for large append */
461 1494669 : fd_memcpy( buf + buf_used, data, sz );
462 1494669 : sha->buf_used = buf_used + sz;
463 1494669 : return sha;
464 1494669 : }
465 :
466 : /* Otherwise, buffer enough leading bytes of data to complete the
467 : block, update the hash and then continue processing any remaining
468 : bytes of data. */
469 :
470 369672 : fd_memcpy( buf + buf_used, data, buf_rem );
471 369672 : data += buf_rem;
472 369672 : sz -= buf_rem;
473 :
474 369672 : fd_sha512_core( state, buf, 1UL );
475 369672 : sha->buf_used = 0UL;
476 369672 : }
477 :
478 : /* Append the bulk of the data */
479 :
480 4997544 : ulong block_cnt = sz >> FD_SHA512_PRIVATE_LG_BUF_MAX;
481 4997544 : if( FD_LIKELY( block_cnt ) ) fd_sha512_core( state, data, block_cnt ); /* optimized for large append */
482 :
483 : /* Buffer any leftover bytes */
484 :
485 4997544 : buf_used = sz & (FD_SHA512_PRIVATE_BUF_MAX-1UL); /* In [0,FD_SHA512_PRIVATE_BUF_MAX) */
486 4997544 : if( FD_UNLIKELY( buf_used ) ) { /* optimized for well aligned use of append */
487 1844922 : fd_memcpy( buf, data + (block_cnt << FD_SHA512_PRIVATE_LG_BUF_MAX), buf_used );
488 1844922 : sha->buf_used = buf_used; /* In (0,FD_SHA512_PRIVATE_BUF_MAX) */
489 1844922 : }
490 :
491 4997544 : return sha;
492 6492213 : }
493 :
494 : void *
495 : fd_sha512_fini( fd_sha512_t * sha,
496 1482501 : void * _hash ) {
497 :
498 : /* Unpack inputs */
499 :
500 1482501 : ulong * state = sha->state;
501 1482501 : uchar * buf = sha->buf;
502 1482501 : ulong buf_used = sha->buf_used; /* In [0,FD_SHA512_PRIVATE_BUF_MAX) */
503 1482501 : ulong bit_cnt_lo = sha->bit_cnt_lo;
504 1482501 : ulong bit_cnt_hi = sha->bit_cnt_hi;
505 :
506 : /* Append the terminating message byte */
507 :
508 1482501 : buf[ buf_used ] = (uchar)0x80;
509 1482501 : buf_used++;
510 :
511 : /* If there isn't enough room to save the message length in bits at
512 : the end of the in progress block, clear the rest of the in progress
513 : block, update the hash and start a new block. */
514 :
515 1482501 : if( FD_UNLIKELY( buf_used > FD_SHA512_PRIVATE_BUF_MAX-16UL ) ) { /* optimize for well aligned use of append */
516 77247 : fd_memset( buf + buf_used, 0, FD_SHA512_PRIVATE_BUF_MAX-buf_used );
517 77247 : fd_sha512_core( state, buf, 1UL );
518 77247 : buf_used = 0UL;
519 77247 : }
520 :
521 : /* Clear in progress block up to last 128-bits, append the message
522 : size in bytes in the last 128-bits of the in progress block and
523 : update the hash to finalize it. */
524 :
525 1482501 : fd_memset( buf + buf_used, 0, FD_SHA512_PRIVATE_BUF_MAX-16UL-buf_used );
526 1482501 : *((ulong *)(buf+FD_SHA512_PRIVATE_BUF_MAX-16UL)) = fd_ulong_bswap( bit_cnt_hi );
527 1482501 : *((ulong *)(buf+FD_SHA512_PRIVATE_BUF_MAX- 8UL)) = fd_ulong_bswap( bit_cnt_lo );
528 1482501 : fd_sha512_core( state, buf, 1UL );
529 :
530 : /* Unpack the result into md (annoying bswaps here) */
531 :
532 1482501 : uchar * hash = (uchar *)_hash;
533 1482501 : FD_STORE( ulong, hash, fd_ulong_bswap( state[0] ) );
534 1482501 : FD_STORE( ulong, hash+ 8UL, fd_ulong_bswap( state[1] ) );
535 1482501 : FD_STORE( ulong, hash+16UL, fd_ulong_bswap( state[2] ) );
536 1482501 : FD_STORE( ulong, hash+24UL, fd_ulong_bswap( state[3] ) );
537 1482501 : FD_STORE( ulong, hash+32UL, fd_ulong_bswap( state[4] ) );
538 1482501 : FD_STORE( ulong, hash+40UL, fd_ulong_bswap( state[5] ) );
539 1482501 : FD_STORE( ulong, hash+48UL, fd_ulong_bswap( state[6] ) );
540 1482501 : FD_STORE( ulong, hash+56UL, fd_ulong_bswap( state[7] ) );
541 1482501 : return _hash;
542 1482501 : }
543 :
544 : void *
545 : fd_sha384_fini( fd_sha512_t * sha,
546 2118 : void * _hash ) {
547 2118 : uchar hash[ FD_SHA512_HASH_SZ ] __attribute__((aligned(64)));
548 2118 : fd_sha512_fini( sha, hash );
549 2118 : memcpy( _hash, hash, FD_SHA384_HASH_SZ );
550 2118 : return _hash;
551 2118 : }
552 :
553 : void *
554 : fd_sha512_hash( void const * _data,
555 : ulong sz,
556 2934156 : void * _hash ) {
557 2934156 : uchar const * data = (uchar const *)_data;
558 :
559 : /* This is just the above streamlined to eliminate all the overheads
560 : to support incremental hashing. */
561 :
562 2934156 : uchar buf[ FD_SHA512_PRIVATE_BUF_MAX ] __attribute__((aligned(128)));
563 2934156 : ulong state[8] __attribute__((aligned(64)));
564 :
565 2934156 : state[0] = 0x6a09e667f3bcc908UL;
566 2934156 : state[1] = 0xbb67ae8584caa73bUL;
567 2934156 : state[2] = 0x3c6ef372fe94f82bUL;
568 2934156 : state[3] = 0xa54ff53a5f1d36f1UL;
569 2934156 : state[4] = 0x510e527fade682d1UL;
570 2934156 : state[5] = 0x9b05688c2b3e6c1fUL;
571 2934156 : state[6] = 0x1f83d9abfb41bd6bUL;
572 2934156 : state[7] = 0x5be0cd19137e2179UL;
573 :
574 2934156 : ulong block_cnt = sz >> FD_SHA512_PRIVATE_LG_BUF_MAX;
575 2934156 : if( FD_LIKELY( block_cnt ) ) fd_sha512_core( state, data, block_cnt );
576 :
577 2934156 : ulong buf_used = sz & (FD_SHA512_PRIVATE_BUF_MAX-1UL);
578 2934156 : if( FD_UNLIKELY( buf_used ) ) fd_memcpy( buf, data + (block_cnt << FD_SHA512_PRIVATE_LG_BUF_MAX), buf_used );
579 2934156 : buf[ buf_used ] = (uchar)0x80;
580 2934156 : buf_used++;
581 :
582 2934156 : if( FD_UNLIKELY( buf_used > (FD_SHA512_PRIVATE_BUF_MAX-16UL) ) ) {
583 288001 : fd_memset( buf + buf_used, 0, FD_SHA512_PRIVATE_BUF_MAX-buf_used );
584 288001 : fd_sha512_core( state, buf, 1UL );
585 288001 : buf_used = 0UL;
586 288001 : }
587 :
588 2934156 : ulong bit_cnt_lo = sz<< 3;
589 2934156 : ulong bit_cnt_hi = sz>>61;
590 2934156 : fd_memset( buf + buf_used, 0, FD_SHA512_PRIVATE_BUF_MAX-16UL-buf_used );
591 2934156 : *((ulong *)(buf+FD_SHA512_PRIVATE_BUF_MAX-16UL)) = fd_ulong_bswap( bit_cnt_hi );
592 2934156 : *((ulong *)(buf+FD_SHA512_PRIVATE_BUF_MAX- 8UL)) = fd_ulong_bswap( bit_cnt_lo );
593 2934156 : fd_sha512_core( state, buf, 1UL );
594 :
595 2934156 : uchar * hash = (uchar *)_hash;
596 2934156 : FD_STORE( ulong, hash, fd_ulong_bswap( state[0] ) );
597 2934156 : FD_STORE( ulong, hash+ 8UL, fd_ulong_bswap( state[1] ) );
598 2934156 : FD_STORE( ulong, hash+16UL, fd_ulong_bswap( state[2] ) );
599 2934156 : FD_STORE( ulong, hash+24UL, fd_ulong_bswap( state[3] ) );
600 2934156 : FD_STORE( ulong, hash+32UL, fd_ulong_bswap( state[4] ) );
601 2934156 : FD_STORE( ulong, hash+40UL, fd_ulong_bswap( state[5] ) );
602 2934156 : FD_STORE( ulong, hash+48UL, fd_ulong_bswap( state[6] ) );
603 2934156 : FD_STORE( ulong, hash+56UL, fd_ulong_bswap( state[7] ) );
604 2934156 : return _hash;
605 2934156 : }
606 :
607 : void *
608 : fd_sha384_hash( void const * _data,
609 : ulong sz,
610 2937 : void * _hash ) {
611 2937 : uchar const * data = (uchar const *)_data;
612 :
613 : /* This is just the above streamlined to eliminate all the overheads
614 : to support incremental hashing. */
615 :
616 2937 : uchar buf[ FD_SHA512_PRIVATE_BUF_MAX ] __attribute__((aligned(128)));
617 2937 : ulong state[8] __attribute__((aligned(64)));
618 :
619 2937 : state[0] = 0xcbbb9d5dc1059ed8UL;
620 2937 : state[1] = 0x629a292a367cd507UL;
621 2937 : state[2] = 0x9159015a3070dd17UL;
622 2937 : state[3] = 0x152fecd8f70e5939UL;
623 2937 : state[4] = 0x67332667ffc00b31UL;
624 2937 : state[5] = 0x8eb44a8768581511UL;
625 2937 : state[6] = 0xdb0c2e0d64f98fa7UL;
626 2937 : state[7] = 0x47b5481dbefa4fa4UL;
627 :
628 2937 : ulong block_cnt = sz >> FD_SHA512_PRIVATE_LG_BUF_MAX;
629 2937 : if( FD_LIKELY( block_cnt ) ) fd_sha512_core( state, data, block_cnt );
630 :
631 2937 : ulong buf_used = sz & (FD_SHA512_PRIVATE_BUF_MAX-1UL);
632 2937 : if( FD_UNLIKELY( buf_used ) ) fd_memcpy( buf, data + (block_cnt << FD_SHA512_PRIVATE_LG_BUF_MAX), buf_used );
633 2937 : buf[ buf_used ] = (uchar)0x80;
634 2937 : buf_used++;
635 :
636 2937 : if( FD_UNLIKELY( buf_used > (FD_SHA512_PRIVATE_BUF_MAX-16UL) ) ) {
637 195 : fd_memset( buf + buf_used, 0, FD_SHA512_PRIVATE_BUF_MAX-buf_used );
638 195 : fd_sha512_core( state, buf, 1UL );
639 195 : buf_used = 0UL;
640 195 : }
641 :
642 2937 : ulong bit_cnt_lo = sz<< 3;
643 2937 : ulong bit_cnt_hi = sz>>61;
644 2937 : fd_memset( buf + buf_used, 0, FD_SHA512_PRIVATE_BUF_MAX-16UL-buf_used );
645 2937 : *((ulong *)(buf+FD_SHA512_PRIVATE_BUF_MAX-16UL)) = fd_ulong_bswap( bit_cnt_hi );
646 2937 : *((ulong *)(buf+FD_SHA512_PRIVATE_BUF_MAX- 8UL)) = fd_ulong_bswap( bit_cnt_lo );
647 2937 : fd_sha512_core( state, buf, 1UL );
648 :
649 2937 : uchar * hash = (uchar *)_hash;
650 2937 : FD_STORE( ulong, hash, fd_ulong_bswap( state[0] ) );
651 2937 : FD_STORE( ulong, hash+ 8UL, fd_ulong_bswap( state[1] ) );
652 2937 : FD_STORE( ulong, hash+16UL, fd_ulong_bswap( state[2] ) );
653 2937 : FD_STORE( ulong, hash+24UL, fd_ulong_bswap( state[3] ) );
654 2937 : FD_STORE( ulong, hash+32UL, fd_ulong_bswap( state[4] ) );
655 2937 : FD_STORE( ulong, hash+40UL, fd_ulong_bswap( state[5] ) );
656 2937 : return _hash;
657 2937 : }
658 :
659 : #undef fd_sha512_core
|
