Line data Source code
1 : /* fd_aes_gcm_x86.c provides wrappers and AES key expansion for
2 : fd_aes_gcm_{aesni,avx10}.S */
3 :
4 : #include "fd_aes_gcm.h"
5 : #include "../../util/sanitize/fd_msan.h"
6 :
7 : #if FD_HAS_AESNI
8 :
9 : #include "../../util/simd/fd_sse.h"
10 :
11 : FD_FN_SENSITIVE static void
12 : expand_aes_key( fd_aes_gcm_aesni_key_t * out,
13 : uchar const * keyp,
14 31545370 : ulong key_sz ) {
15 31545370 : vb_t v0 = vb_ldu( keyp );
16 31545370 : vb_t v1;
17 :
18 315453670 : # define EXPAND_A( gen, shuffle ) do { \
19 315453670 : vb_t v2 = _mm_aeskeygenassist_si128( v1, (gen) ); \
20 315453670 : v2 = _mm_shuffle_epi32( v2, (shuffle) ); \
21 315453670 : vb_t v3 = _mm_slli_si128( v0, 4 ); \
22 315453670 : v0 = vu_xor( v0, v3 ); \
23 315453670 : v3 = _mm_slli_si128( v3, 4 ); \
24 315453670 : v0 = vu_xor( v0, v3 ); \
25 315453670 : v3 = _mm_slli_si128( v3, 4 ); \
26 315453670 : v0 = vu_xor( v0, v3 ); \
27 315453670 : v0 = vu_xor( v0, v2 ); \
28 315453670 : } while(0)
29 :
30 31545370 : ulong nr;
31 31545370 : switch( key_sz ) {
32 31545358 : case 16UL:
33 31545358 : v1 = v0;
34 31545358 : vb_st( out->key_enc, v0 );
35 315453580 : # define EXPAND_128( off, gen ) do { \
36 315453580 : EXPAND_A( (gen), 0xff ); \
37 315453580 : vb_st( out->key_enc + (off), v0 ); \
38 315453580 : v1 = v0; \
39 315453580 : } while(0)
40 31545358 : EXPAND_128( 0x10, 0x01 );
41 31545358 : EXPAND_128( 0x20, 0x02 );
42 31545358 : EXPAND_128( 0x30, 0x04 );
43 31545358 : EXPAND_128( 0x40, 0x08 );
44 31545358 : EXPAND_128( 0x50, 0x10 );
45 31545358 : EXPAND_128( 0x60, 0x20 );
46 31545358 : EXPAND_128( 0x70, 0x40 );
47 31545358 : EXPAND_128( 0x80, 0x80 );
48 31545358 : EXPAND_128( 0x90, 0x1b );
49 31545358 : EXPAND_128( 0xa0, 0x36 );
50 31545358 : # undef EXPAND_128
51 31545358 : nr = 10UL;
52 31545358 : break;
53 6 : case 24UL:
54 6 : v1 = _mm_loadl_epi64( (__m128i const *)(keyp+16) );
55 6 : vb_st ( out->key_enc, v0 );
56 6 : _mm_storel_epi64( (__m128i *)(out->key_enc+16), v1 );
57 42 : # define EXPAND_192( off, gen ) do { \
58 42 : EXPAND_A( (gen), 0x55 ); \
59 42 : vb_t v2 = _mm_shuffle_epi32( v0, 0xff ); \
60 42 : v1 = vu_xor( v1, _mm_slli_si128( v1, 4 ) ); \
61 42 : v1 = vu_xor( v1, v2 ); \
62 42 : vb_stu( out->key_enc + (off), v0 ); \
63 42 : _mm_storel_epi64( (__m128i *)(out->key_enc+(off)+16), v1 ); \
64 42 : } while(0)
65 6 : EXPAND_192( 24, 0x01 );
66 6 : EXPAND_192( 48, 0x02 );
67 6 : EXPAND_192( 72, 0x04 );
68 6 : EXPAND_192( 96, 0x08 );
69 6 : EXPAND_192( 120, 0x10 );
70 6 : EXPAND_192( 144, 0x20 );
71 6 : EXPAND_192( 168, 0x40 );
72 6 : # undef EXPAND_192
73 6 : EXPAND_A( 0x80, 0x55 );
74 6 : vb_st( out->key_enc + 192, v0 );
75 6 : nr = 12UL;
76 6 : break;
77 6 : case 32UL:
78 6 : v1 = vb_ldu( keyp+16 );
79 6 : vb_st( out->key_enc, v0 );
80 6 : vb_st( out->key_enc+0x10, v1 );
81 42 : # define EXPAND_256_A( off, gen ) do { \
82 42 : EXPAND_A( (gen), 0xff ); \
83 42 : vb_st( out->key_enc + (off), v0 ); \
84 42 : } while(0)
85 36 : # define EXPAND_256_B( off ) do { \
86 36 : vb_t v2 = _mm_aeskeygenassist_si128( v0, 0x00 ); \
87 36 : v2 = _mm_shuffle_epi32( v2, 0xaa ); \
88 36 : vb_t v3 = _mm_slli_si128( v1, 4 ); \
89 36 : v1 = vu_xor( v1, v3 ); \
90 36 : v3 = _mm_slli_si128( v3, 4 ); \
91 36 : v1 = vu_xor( v1, v3 ); \
92 36 : v3 = _mm_slli_si128( v3, 4 ); \
93 36 : v1 = vu_xor( v1, v3 ); \
94 36 : v1 = vu_xor( v1, v2 ); \
95 36 : vb_st( out->key_enc + (off), v1 ); \
96 36 : } while(0)
97 6 : EXPAND_256_A( 0x20, 0x01 ); EXPAND_256_B( 0x30 );
98 6 : EXPAND_256_A( 0x40, 0x02 ); EXPAND_256_B( 0x50 );
99 6 : EXPAND_256_A( 0x60, 0x04 ); EXPAND_256_B( 0x70 );
100 6 : EXPAND_256_A( 0x80, 0x08 ); EXPAND_256_B( 0x90 );
101 6 : EXPAND_256_A( 0xa0, 0x10 ); EXPAND_256_B( 0xb0 );
102 6 : EXPAND_256_A( 0xc0, 0x20 ); EXPAND_256_B( 0xd0 );
103 6 : EXPAND_256_A( 0xe0, 0x40 );
104 6 : # undef EXPAND_256_B
105 6 : # undef EXPAND_256_A
106 6 : nr = 14UL;
107 6 : break;
108 0 : default:
109 0 : FD_LOG_CRIT(( "invalid key_sz %lu", key_sz ));
110 31545370 : }
111 31545370 : # undef EXPAND_A
112 :
113 31545370 : vb_st( out->key_dec, vb_ld( out->key_enc + nr*16UL ) );
114 315453736 : for( ulong i=1UL; i<nr; i++ ) {
115 283908366 : vb_t enc_key = vb_ld( out->key_enc + i*16UL );
116 283908366 : vb_st( out->key_dec + (nr-i)*16UL, _mm_aesimc_si128( enc_key ) );
117 283908366 : }
118 31545370 : vb_st( out->key_dec + nr*16UL, vb_ld( out->key_enc ) );
119 31545370 : out->key_sz = (uint)key_sz;
120 31545370 : }
121 :
122 : __attribute__((sysv_abi)) extern void
123 : aes_gcm_precompute_aesni( fd_aes_gcm_aesni_t * key );
124 :
125 : __attribute__((sysv_abi)) extern void
126 : aes_gcm_aad_update_aesni( fd_aes_gcm_aesni_t const * key,
127 : uchar ghash_acc[16],
128 : uchar const * aad,
129 : int aadlen );
130 :
131 : __attribute__((sysv_abi)) extern void
132 : aes_gcm_enc_update_aesni( fd_aes_gcm_aesni_t const * key,
133 : uint const le_ctr[4],
134 : uchar ghash_acc[16],
135 : uchar const * src,
136 : uchar * dst,
137 : int datalen );
138 :
139 : __attribute__((sysv_abi)) extern void
140 : aes_gcm_enc_final_aesni( fd_aes_gcm_aesni_t const * key,
141 : uint const le_ctr[4],
142 : uchar ghash_acc[16],
143 : ulong total_aadlen,
144 : ulong total_datalen );
145 :
146 : __attribute__((sysv_abi)) extern void
147 : aes_gcm_dec_update_aesni( fd_aes_gcm_aesni_t const * key,
148 : uint const le_ctr[4],
149 : uchar ghash_acc[16],
150 : uchar const * src,
151 : uchar * dst,
152 : int datalen );
153 :
154 : __attribute__((sysv_abi,warn_unused_result)) extern int
155 : aes_gcm_dec_final_aesni( fd_aes_gcm_aesni_t const * key,
156 : uint const le_ctr[4],
157 : uchar const ghash_acc[16],
158 : ulong total_aadlen,
159 : ulong total_datalen,
160 : uchar const tag[16],
161 : int taglen );
162 :
163 : void
164 : fd_aes_gcm_set_iv_aesni( fd_aes_gcm_aesni_t * aes_gcm,
165 0 : uchar const iv[ 12 ] ) {
166 0 : memcpy( aes_gcm->iv, iv, 12 );
167 0 : }
168 :
169 : void
170 : fd_aes_gcm_init_aesni( fd_aes_gcm_aesni_t * aes_gcm,
171 : uchar const * key,
172 : ulong key_sz,
173 0 : uchar const iv[ 12 ] ) {
174 0 : expand_aes_key( &aes_gcm->key, key, key_sz );
175 0 : aes_gcm_precompute_aesni( aes_gcm );
176 0 : memcpy( aes_gcm->iv, iv, 12 );
177 0 : }
178 :
179 : static void
180 : load_le_ctr( uint le_ctr[4],
181 31642272 : uchar const iv[12] ) {
182 31642272 : le_ctr[0] = 2;
183 31642272 : le_ctr[1] = fd_uint_bswap( fd_uint_load_4_fast( iv+8 ) );
184 31642272 : le_ctr[2] = fd_uint_bswap( fd_uint_load_4_fast( iv+4 ) );
185 31642272 : le_ctr[3] = fd_uint_bswap( fd_uint_load_4_fast( iv ) );
186 31642272 : }
187 :
188 : void
189 : fd_aes_gcm_encrypt_aesni( fd_aes_gcm_aesni_t * aes_gcm,
190 : uchar * c,
191 : uchar const * p,
192 : ulong sz,
193 : uchar const * aad,
194 : ulong aad_sz,
195 0 : uchar tag[ 16 ] ) {
196 0 : uint le_ctr[4]; load_le_ctr( le_ctr, aes_gcm->iv );
197 0 : uchar ghash_acc[16] = {0};
198 0 : aes_gcm_aad_update_aesni( aes_gcm, ghash_acc, aad, (int)aad_sz );
199 0 : aes_gcm_enc_update_aesni( aes_gcm, le_ctr, ghash_acc, p, c, (int)sz );
200 0 : aes_gcm_enc_final_aesni ( aes_gcm, le_ctr, ghash_acc, aad_sz, sz );
201 0 : memcpy( tag, ghash_acc, 16 );
202 0 : fd_msan_unpoison( c, sz );
203 0 : }
204 :
205 : int
206 : fd_aes_gcm_decrypt_aesni( fd_aes_gcm_aesni_t * aes_gcm,
207 : uchar const * c,
208 : uchar * p,
209 : ulong sz,
210 : uchar const * aad,
211 : ulong aad_sz,
212 0 : uchar const tag[ 16 ] ) {
213 0 : uint le_ctr[4]; load_le_ctr( le_ctr, aes_gcm->iv );
214 0 : uchar ghash_acc[16] = {0};
215 0 : aes_gcm_aad_update_aesni( aes_gcm, ghash_acc, aad, (int)aad_sz );
216 0 : aes_gcm_dec_update_aesni( aes_gcm, le_ctr, ghash_acc, c, p, (int)sz );
217 0 : fd_msan_unpoison( p, sz );
218 0 : return aes_gcm_dec_final_aesni( aes_gcm, le_ctr, ghash_acc, aad_sz, sz, tag, 16 );
219 0 : }
220 :
221 : #endif /* FD_HAS_AESNI */
222 :
223 : #if FD_HAS_AVX && FD_HAS_AESNI
224 :
225 : __attribute__((sysv_abi)) extern void
226 : aes_gcm_precompute_aesni_avx( fd_aes_gcm_aesni_t * key );
227 :
228 : __attribute__((sysv_abi)) extern void
229 : aes_gcm_aad_update_aesni_avx( fd_aes_gcm_aesni_t * key,
230 : uchar ghash_acc[16],
231 : uchar const * aad,
232 : int aadlen );
233 :
234 : __attribute__((sysv_abi)) extern void
235 : aes_gcm_enc_update_aesni_avx( fd_aes_gcm_aesni_t * key,
236 : uint const le_ctr[4],
237 : uchar ghash_acc[16],
238 : uchar const * src,
239 : uchar * dst,
240 : int datalen );
241 :
242 : __attribute__((sysv_abi)) extern void
243 : aes_gcm_enc_final_aesni_avx( fd_aes_gcm_aesni_t * key,
244 : uint const le_ctr[4],
245 : uchar ghash_acc[16],
246 : ulong total_aadlen,
247 : ulong total_datalen );
248 :
249 : __attribute__((sysv_abi)) extern void
250 : aes_gcm_dec_update_aesni_avx( fd_aes_gcm_aesni_t const * key,
251 : uint const le_ctr[4],
252 : uchar ghash_acc[16],
253 : uchar const * src,
254 : uchar * dst,
255 : int datalen );
256 :
257 : __attribute__((sysv_abi,warn_unused_result)) extern int
258 : aes_gcm_dec_final_aesni_avx( fd_aes_gcm_aesni_t const * key,
259 : uint const le_ctr[4],
260 : uchar const ghash_acc[16],
261 : ulong total_aadlen,
262 : ulong total_datalen,
263 : uchar const tag[16],
264 : int taglen );
265 :
266 : void
267 : fd_aes_gcm_set_iv_avx2( fd_aes_gcm_aesni_t * aes_gcm,
268 55825 : uchar const iv[ 12 ] ) {
269 55825 : memcpy( aes_gcm->iv, iv, 12 );
270 55825 : }
271 :
272 : void
273 : fd_aes_gcm_init_avx2( fd_aes_gcm_aesni_t * aes_gcm,
274 : uchar const * key,
275 : ulong key_sz,
276 14818905 : uchar const iv[ 12 ] ) {
277 14818905 : expand_aes_key( &aes_gcm->key, key, key_sz );
278 14818905 : aes_gcm_precompute_aesni_avx( aes_gcm );
279 14818905 : memcpy( aes_gcm->iv, iv, 12 );
280 14818905 : }
281 :
282 : void
283 : fd_aes_gcm_encrypt_avx2( fd_aes_gcm_aesni_t * aes_gcm,
284 : uchar * c,
285 : uchar const * p,
286 : ulong sz,
287 : uchar const * aad,
288 : ulong aad_sz,
289 7408716 : uchar tag[ 16 ] ) {
290 7408716 : uint le_ctr[4]; load_le_ctr( le_ctr, aes_gcm->iv );
291 7408716 : uchar ghash_acc[16] = {0};
292 7408716 : aes_gcm_aad_update_aesni_avx( aes_gcm, ghash_acc, aad, (int)aad_sz );
293 7408716 : aes_gcm_enc_update_aesni_avx( aes_gcm, le_ctr, ghash_acc, p, c, (int)sz );
294 7408716 : aes_gcm_enc_final_aesni_avx ( aes_gcm, le_ctr, ghash_acc, aad_sz, sz );
295 7408716 : memcpy( tag, ghash_acc, 16 );
296 7408716 : fd_msan_unpoison( c, sz );
297 7408716 : }
298 :
299 : int
300 : fd_aes_gcm_decrypt_avx2( fd_aes_gcm_aesni_t * aes_gcm,
301 : uchar const * c,
302 : uchar * p,
303 : ulong sz,
304 : uchar const * aad,
305 : ulong aad_sz,
306 7458640 : uchar const tag[ 16 ] ) {
307 7458640 : uint le_ctr[4]; load_le_ctr( le_ctr, aes_gcm->iv );
308 7458640 : uchar ghash_acc[16] = {0};
309 7458640 : aes_gcm_aad_update_aesni_avx( aes_gcm, ghash_acc, aad, (int)aad_sz );
310 7458640 : aes_gcm_dec_update_aesni_avx( aes_gcm, le_ctr, ghash_acc, c, p, (int)sz );
311 7458640 : fd_msan_unpoison( p, sz );
312 7458640 : return aes_gcm_dec_final_aesni_avx( aes_gcm, le_ctr, ghash_acc, aad_sz, sz, tag, 16 );
313 7458640 : }
314 :
315 : #endif /* FD_HAS_AVX && FD_HAS_AESNI */
316 :
317 : #if FD_HAS_AVX512 && FD_HAS_GFNI && FD_HAS_AESNI
318 :
319 : __attribute__((sysv_abi)) extern void
320 : aes_gcm_precompute_vaes_avx10_512( fd_aes_gcm_avx10_t * key );
321 :
322 : __attribute__((sysv_abi)) extern void
323 : aes_gcm_aad_update_vaes_avx10( fd_aes_gcm_avx10_t const * key,
324 : uchar ghash_acc[16],
325 : uchar const * aad,
326 : int aadlen );
327 :
328 : __attribute__((sysv_abi)) extern void
329 : aes_gcm_enc_update_vaes_avx10_512( fd_aes_gcm_avx10_t * key,
330 : uint const le_ctr[4],
331 : uchar ghash_acc[16],
332 : uchar const * src,
333 : uchar * dst,
334 : int datalen );
335 :
336 : __attribute__((sysv_abi)) extern void
337 : aes_gcm_enc_final_vaes_avx10( fd_aes_gcm_avx10_t * key,
338 : uint const le_ctr[4],
339 : uchar ghash_acc[16],
340 : ulong total_aadlen,
341 : ulong total_datalen );
342 :
343 : __attribute__((sysv_abi)) extern void
344 : aes_gcm_dec_update_vaes_avx10_512( fd_aes_gcm_avx10_t const * key,
345 : uint const le_ctr[4],
346 : uchar ghash_acc[16],
347 : uchar const * src,
348 : uchar * dst,
349 : int datalen );
350 :
351 : __attribute__((sysv_abi,warn_unused_result)) extern int
352 : aes_gcm_dec_final_vaes_avx10( fd_aes_gcm_avx10_t const * key,
353 : uint const le_ctr[4],
354 : uchar const ghash_acc[16],
355 : ulong total_aadlen,
356 : ulong total_datalen,
357 : uchar const tag[16],
358 : int taglen );
359 :
360 : void
361 : fd_aes_gcm_set_iv_avx10_512( fd_aes_gcm_avx10_t * aes_gcm,
362 55825 : uchar const iv[ 12 ] ) {
363 55825 : memcpy( aes_gcm->iv, iv, 12 );
364 55825 : }
365 :
366 : void
367 : fd_aes_gcm_init_avx10_512( fd_aes_gcm_avx10_t * aes_gcm,
368 : uchar const * key,
369 : ulong key_sz,
370 16726465 : uchar const iv[ 12 ] ) {
371 16726465 : expand_aes_key( &aes_gcm->key, key, key_sz );
372 16726465 : aes_gcm_precompute_vaes_avx10_512( aes_gcm );
373 16726465 : memcpy( aes_gcm->iv, iv, 12 );
374 16726465 : }
375 :
376 : void
377 : fd_aes_gcm_encrypt_avx10_512( fd_aes_gcm_avx10_t * aes_gcm,
378 : uchar * c,
379 : uchar const * p,
380 : ulong sz,
381 : uchar const * aad,
382 : ulong aad_sz,
383 8362496 : uchar tag[ 16 ] ) {
384 8362496 : uint le_ctr[4]; load_le_ctr( le_ctr, aes_gcm->iv );
385 8362496 : uchar ghash_acc[16] = {0};
386 8362496 : aes_gcm_aad_update_vaes_avx10 ( aes_gcm, ghash_acc, aad, (int)aad_sz );
387 8362496 : aes_gcm_enc_update_vaes_avx10_512( aes_gcm, le_ctr, ghash_acc, p, c, (int)sz );
388 8362496 : aes_gcm_enc_final_vaes_avx10 ( aes_gcm, le_ctr, ghash_acc, aad_sz, sz );
389 8362496 : memcpy( tag, ghash_acc, 16 );
390 8362496 : fd_msan_unpoison( c, sz );
391 8362496 : }
392 :
393 : int
394 : fd_aes_gcm_decrypt_avx10_512( fd_aes_gcm_avx10_t * aes_gcm,
395 : uchar const * c,
396 : uchar * p,
397 : ulong sz,
398 : uchar const * aad,
399 : ulong aad_sz,
400 8412420 : uchar const tag[ 16 ] ) {
401 8412420 : uint le_ctr[4]; load_le_ctr( le_ctr, aes_gcm->iv );
402 8412420 : uchar ghash_acc[16] = {0};
403 8412420 : aes_gcm_aad_update_vaes_avx10 ( aes_gcm, ghash_acc, aad, (int)aad_sz );
404 8412420 : aes_gcm_dec_update_vaes_avx10_512 ( aes_gcm, le_ctr, ghash_acc, c, p, (int)sz );
405 8412420 : fd_msan_unpoison( p, sz );
406 8412420 : return aes_gcm_dec_final_vaes_avx10( aes_gcm, le_ctr, ghash_acc, aad_sz, sz, tag, 16 );
407 8412420 : }
408 :
409 : #endif /* FD_HAS_AVX512 && FD_HAS_GFNI && FD_HAS_AESNI */
|
