Line data Source code
1 : /* fd_aes_gcm_x86.c provides wrappers and AES key expansion for
2 : fd_aes_gcm_{aesni,avx10}.S */
3 :
4 : #include "fd_aes_gcm.h"
5 :
6 : #if FD_HAS_AESNI
7 :
8 : #include "../../util/simd/fd_sse.h"
9 :
10 : FD_FN_SENSITIVE static void
11 : expand_aes_key( fd_aes_gcm_aesni_key_t * out,
12 64395666 : uchar const * keyp ) {
13 64395666 : vb_t key = vb_ldu( keyp );
14 :
15 : /* Expand encryption key */
16 :
17 64395666 : vb_t v0, v1;
18 64395666 : vb_t enc[11];
19 643956660 : # define ASSIST( out, gen ) do { \
20 643956660 : vb_t v2 = _mm_aeskeygenassist_si128( v0, (gen) ); \
21 643956660 : vb_t v3 = vu_permute2( v1, v0, 0, 0, 1, 0 ); \
22 643956660 : vb_t v4 = vu_xor ( v0, v3 ); \
23 643956660 : v1 = vu_permute2( v3, v4, 0, 3, 0, 2 ); \
24 643956660 : vb_t v5 = vu_xor ( v4, v1 ); \
25 643956660 : vb_t v6 = vu_permute ( v2, 3, 3, 3, 3 ); \
26 643956660 : v0 = vu_xor ( v5, v6 ); \
27 643956660 : (out) = v0; \
28 643956660 : } while(0)
29 64395666 : v0 = key;
30 64395666 : v1 = vu_zero();
31 64395666 : enc[ 0] = v0;
32 64395666 : ASSIST( enc[ 1], 0x01 );
33 64395666 : ASSIST( enc[ 2], 0x02 );
34 64395666 : ASSIST( enc[ 3], 0x04 );
35 64395666 : ASSIST( enc[ 4], 0x08 );
36 64395666 : ASSIST( enc[ 5], 0x10 );
37 64395666 : ASSIST( enc[ 6], 0x20 );
38 64395666 : ASSIST( enc[ 7], 0x40 );
39 64395666 : ASSIST( enc[ 8], 0x80 );
40 64395666 : ASSIST( enc[ 9], 0x1B );
41 64395666 : ASSIST( enc[10], 0x36 );
42 64395666 : # undef ASSIST
43 :
44 64395666 : vb_st( out->key_enc, enc[ 0] );
45 64395666 : vb_st( out->key_enc + 0x10, enc[ 1] );
46 64395666 : vb_st( out->key_enc + 0x20, enc[ 2] );
47 64395666 : vb_st( out->key_enc + 0x30, enc[ 3] );
48 64395666 : vb_st( out->key_enc + 0x40, enc[ 4] );
49 64395666 : vb_st( out->key_enc + 0x50, enc[ 5] );
50 64395666 : vb_st( out->key_enc + 0x60, enc[ 6] );
51 64395666 : vb_st( out->key_enc + 0x70, enc[ 7] );
52 64395666 : vb_st( out->key_enc + 0x80, enc[ 8] );
53 64395666 : vb_st( out->key_enc + 0x90, enc[ 9] );
54 64395666 : vb_st( out->key_enc + 0xa0, enc[10] );
55 :
56 : /* Derive decryption key */
57 :
58 64395666 : vb_st( out->key_dec, enc[10] );
59 64395666 : vb_st( out->key_dec + 0x10, _mm_aesimc_si128( enc[ 9] ) );
60 64395666 : vb_st( out->key_dec + 0x20, _mm_aesimc_si128( enc[ 8] ) );
61 64395666 : vb_st( out->key_dec + 0x30, _mm_aesimc_si128( enc[ 7] ) );
62 64395666 : vb_st( out->key_dec + 0x40, _mm_aesimc_si128( enc[ 6] ) );
63 64395666 : vb_st( out->key_dec + 0x50, _mm_aesimc_si128( enc[ 5] ) );
64 64395666 : vb_st( out->key_dec + 0x60, _mm_aesimc_si128( enc[ 4] ) );
65 64395666 : vb_st( out->key_dec + 0x70, _mm_aesimc_si128( enc[ 3] ) );
66 64395666 : vb_st( out->key_dec + 0x80, _mm_aesimc_si128( enc[ 2] ) );
67 64395666 : vb_st( out->key_dec + 0x90, _mm_aesimc_si128( enc[ 1] ) );
68 64395666 : vb_st( out->key_dec + 0xa0, _mm_aesimc_si128( enc[ 0] ) );
69 :
70 64395666 : out->key_sz = 16;
71 64395666 : }
72 :
73 : __attribute__((sysv_abi)) extern void
74 : aes_gcm_precompute_aesni( fd_aes_gcm_aesni_t * key );
75 :
76 : __attribute__((sysv_abi)) extern void
77 : aes_gcm_aad_update_aesni( fd_aes_gcm_aesni_t const * key,
78 : uchar ghash_acc[16],
79 : uchar const * aad,
80 : int aadlen );
81 :
82 : __attribute__((sysv_abi)) extern void
83 : aes_gcm_enc_update_aesni( fd_aes_gcm_aesni_t const * key,
84 : uint const le_ctr[4],
85 : uchar ghash_acc[16],
86 : uchar const * src,
87 : uchar * dst,
88 : int datalen );
89 :
90 : __attribute__((sysv_abi)) extern void
91 : aes_gcm_enc_final_aesni( fd_aes_gcm_aesni_t const * key,
92 : uint const le_ctr[4],
93 : uchar ghash_acc[16],
94 : ulong total_aadlen,
95 : ulong total_datalen );
96 :
97 : __attribute__((sysv_abi)) extern void
98 : aes_gcm_dec_update_aesni( fd_aes_gcm_aesni_t const * key,
99 : uint const le_ctr[4],
100 : uchar ghash_acc[16],
101 : uchar const * src,
102 : uchar * dst,
103 : int datalen );
104 :
105 : __attribute__((sysv_abi,warn_unused_result)) extern int
106 : aes_gcm_dec_final_aesni( fd_aes_gcm_aesni_t const * key,
107 : uint const le_ctr[4],
108 : uchar const ghash_acc[16],
109 : ulong total_aadlen,
110 : ulong total_datalen,
111 : uchar const tag[16],
112 : int taglen );
113 :
114 : void
115 : fd_aes_128_gcm_init_aesni( fd_aes_gcm_aesni_t * aes_gcm,
116 : uchar const key[ 16 ],
117 0 : uchar const iv [ 12 ] ) {
118 0 : expand_aes_key( &aes_gcm->key, key );
119 0 : aes_gcm_precompute_aesni( aes_gcm );
120 0 : memcpy( aes_gcm->iv, iv, 12 );
121 0 : }
122 :
123 : static void
124 : load_le_ctr( uint le_ctr[4],
125 64395666 : uchar const iv[12] ) {
126 64395666 : le_ctr[0] = 2;
127 64395666 : le_ctr[1] = fd_uint_bswap( fd_uint_load_4_fast( iv+8 ) );
128 64395666 : le_ctr[2] = fd_uint_bswap( fd_uint_load_4_fast( iv+4 ) );
129 64395666 : le_ctr[3] = fd_uint_bswap( fd_uint_load_4_fast( iv ) );
130 64395666 : }
131 :
132 : void
133 : fd_aes_gcm_encrypt_aesni( fd_aes_gcm_aesni_t * aes_gcm,
134 : uchar * c,
135 : uchar const * p,
136 : ulong sz,
137 : uchar const * aad,
138 : ulong aad_sz,
139 0 : uchar tag[ 16 ] ) {
140 0 : uint le_ctr[4]; load_le_ctr( le_ctr, aes_gcm->iv );
141 0 : uchar ghash_acc[16] = {0};
142 0 : aes_gcm_aad_update_aesni( aes_gcm, ghash_acc, aad, (int)aad_sz );
143 0 : aes_gcm_enc_update_aesni( aes_gcm, le_ctr, ghash_acc, p, c, (int)sz );
144 0 : aes_gcm_enc_final_aesni ( aes_gcm, le_ctr, ghash_acc, aad_sz, sz );
145 0 : memcpy( tag, ghash_acc, 16 );
146 0 : }
147 :
148 : int
149 : fd_aes_gcm_decrypt_aesni( fd_aes_gcm_aesni_t * aes_gcm,
150 : uchar const * c,
151 : uchar * p,
152 : ulong sz,
153 : uchar const * aad,
154 : ulong aad_sz,
155 0 : uchar const tag[ 16 ] ) {
156 0 : uint le_ctr[4]; load_le_ctr( le_ctr, aes_gcm->iv );
157 0 : uchar ghash_acc[16] = {0};
158 0 : aes_gcm_aad_update_aesni( aes_gcm, ghash_acc, aad, (int)aad_sz );
159 0 : aes_gcm_dec_update_aesni( aes_gcm, le_ctr, ghash_acc, c, p, (int)sz );
160 0 : return aes_gcm_dec_final_aesni( aes_gcm, le_ctr, ghash_acc, aad_sz, sz, tag, 16 );
161 0 : }
162 :
163 : #endif /* FD_HAS_AESNI */
164 :
165 : #if FD_HAS_AVX && FD_HAS_AESNI
166 :
167 : __attribute__((sysv_abi)) extern void
168 : aes_gcm_precompute_aesni_avx( fd_aes_gcm_aesni_t * key );
169 :
170 : __attribute__((sysv_abi)) extern void
171 : aes_gcm_aad_update_aesni_avx( fd_aes_gcm_aesni_t * key,
172 : uchar ghash_acc[16],
173 : uchar const * aad,
174 : int aadlen );
175 :
176 : __attribute__((sysv_abi)) extern void
177 : aes_gcm_enc_update_aesni_avx( fd_aes_gcm_aesni_t * key,
178 : uint const le_ctr[4],
179 : uchar ghash_acc[16],
180 : uchar const * src,
181 : uchar * dst,
182 : int datalen );
183 :
184 : __attribute__((sysv_abi)) extern void
185 : aes_gcm_enc_final_aesni_avx( fd_aes_gcm_aesni_t * key,
186 : uint const le_ctr[4],
187 : uchar ghash_acc[16],
188 : ulong total_aadlen,
189 : ulong total_datalen );
190 :
191 : __attribute__((sysv_abi)) extern void
192 : aes_gcm_dec_update_aesni_avx( fd_aes_gcm_aesni_t const * key,
193 : uint const le_ctr[4],
194 : uchar ghash_acc[16],
195 : uchar const * src,
196 : uchar * dst,
197 : int datalen );
198 :
199 : __attribute__((sysv_abi,warn_unused_result)) extern int
200 : aes_gcm_dec_final_aesni_avx( fd_aes_gcm_aesni_t const * key,
201 : uint const le_ctr[4],
202 : uchar const ghash_acc[16],
203 : ulong total_aadlen,
204 : ulong total_datalen,
205 : uchar const tag[16],
206 : int taglen );
207 :
208 : void
209 : fd_aes_128_gcm_init_avx2( fd_aes_gcm_aesni_t * aes_gcm,
210 : uchar const key[ 16 ],
211 29678356 : uchar const iv [ 12 ] ) {
212 29678356 : expand_aes_key( fd_type_pun( &aes_gcm->key ), key );
213 29678356 : aes_gcm_precompute_aesni_avx( aes_gcm );
214 29678356 : memcpy( aes_gcm->iv, iv, 12 );
215 29678356 : }
216 :
217 : void
218 : fd_aes_gcm_encrypt_avx2( fd_aes_gcm_aesni_t * aes_gcm,
219 : uchar * c,
220 : uchar const * p,
221 : ulong sz,
222 : uchar const * aad,
223 : ulong aad_sz,
224 14839430 : uchar tag[ 16 ] ) {
225 14839430 : uint le_ctr[4]; load_le_ctr( le_ctr, aes_gcm->iv );
226 14839430 : uchar ghash_acc[16] = {0};
227 14839430 : aes_gcm_aad_update_aesni_avx( aes_gcm, ghash_acc, aad, (int)aad_sz );
228 14839430 : aes_gcm_enc_update_aesni_avx( aes_gcm, le_ctr, ghash_acc, p, c, (int)sz );
229 14839430 : aes_gcm_enc_final_aesni_avx ( aes_gcm, le_ctr, ghash_acc, aad_sz, sz );
230 14839430 : memcpy( tag, ghash_acc, 16 );
231 14839430 : }
232 :
233 : int
234 : fd_aes_gcm_decrypt_avx2( fd_aes_gcm_aesni_t * aes_gcm,
235 : uchar const * c,
236 : uchar * p,
237 : ulong sz,
238 : uchar const * aad,
239 : ulong aad_sz,
240 14838926 : uchar const tag[ 16 ] ) {
241 14838926 : uint le_ctr[4]; load_le_ctr( le_ctr, aes_gcm->iv );
242 14838926 : uchar ghash_acc[16] = {0};
243 14838926 : aes_gcm_aad_update_aesni_avx( aes_gcm, ghash_acc, aad, (int)aad_sz );
244 14838926 : aes_gcm_dec_update_aesni_avx( aes_gcm, le_ctr, ghash_acc, c, p, (int)sz );
245 14838926 : return aes_gcm_dec_final_aesni_avx( aes_gcm, le_ctr, ghash_acc, aad_sz, sz, tag, 16 );
246 14838926 : }
247 :
248 : #endif /* FD_HAS_AVX && FD_HAS_AESNI */
249 :
250 : #if FD_HAS_AVX512 && FD_HAS_GFNI && FD_HAS_AESNI
251 :
252 : __attribute__((sysv_abi)) extern void
253 : aes_gcm_precompute_vaes_avx10_512( fd_aes_gcm_avx10_t * key );
254 :
255 : __attribute__((sysv_abi)) extern void
256 : aes_gcm_aad_update_vaes_avx10( fd_aes_gcm_avx10_t const * key,
257 : uchar ghash_acc[16],
258 : uchar const * aad,
259 : int aadlen );
260 :
261 : __attribute__((sysv_abi)) extern void
262 : aes_gcm_enc_update_vaes_avx10_512( fd_aes_gcm_avx10_t * key,
263 : uint const le_ctr[4],
264 : uchar ghash_acc[16],
265 : uchar const * src,
266 : uchar * dst,
267 : int datalen );
268 :
269 : __attribute__((sysv_abi)) extern void
270 : aes_gcm_enc_final_vaes_avx10( fd_aes_gcm_avx10_t * key,
271 : uint const le_ctr[4],
272 : uchar ghash_acc[16],
273 : ulong total_aadlen,
274 : ulong total_datalen );
275 :
276 : __attribute__((sysv_abi)) extern void
277 : aes_gcm_dec_update_vaes_avx10_512( fd_aes_gcm_avx10_t const * key,
278 : uint const le_ctr[4],
279 : uchar ghash_acc[16],
280 : uchar const * src,
281 : uchar * dst,
282 : int datalen );
283 :
284 : __attribute__((sysv_abi,warn_unused_result)) extern int
285 : aes_gcm_dec_final_vaes_avx10( fd_aes_gcm_avx10_t const * key,
286 : uint const le_ctr[4],
287 : uchar const ghash_acc[16],
288 : ulong total_aadlen,
289 : ulong total_datalen,
290 : uchar const tag[16],
291 : int taglen );
292 :
293 : void
294 : fd_aes_128_gcm_init_avx10( fd_aes_gcm_avx10_t * aes_gcm,
295 : uchar const key[ 16 ],
296 0 : uchar const iv [ 12 ] ) {
297 0 : expand_aes_key( &aes_gcm->key, key );
298 0 : aes_gcm_precompute_vaes_avx10_512( aes_gcm );
299 0 : memcpy( aes_gcm->iv, iv, 12 );
300 0 : }
301 :
302 : void
303 : fd_aes_128_gcm_init_avx10_512( fd_aes_gcm_avx10_t * aes_gcm,
304 : uchar const key[ 16 ],
305 34717310 : uchar const iv [ 12 ] ) {
306 34717310 : expand_aes_key( &aes_gcm->key, key );
307 34717310 : aes_gcm_precompute_vaes_avx10_512( aes_gcm );
308 34717310 : memcpy( aes_gcm->iv, iv, 12 );
309 34717310 : }
310 :
311 : void
312 : fd_aes_gcm_encrypt_avx10_512( fd_aes_gcm_avx10_t * aes_gcm,
313 : uchar * c,
314 : uchar const * p,
315 : ulong sz,
316 : uchar const * aad,
317 : ulong aad_sz,
318 17358907 : uchar tag[ 16 ] ) {
319 17358907 : uint le_ctr[4]; load_le_ctr( le_ctr, aes_gcm->iv );
320 17358907 : uchar ghash_acc[16] = {0};
321 17358907 : aes_gcm_aad_update_vaes_avx10 ( aes_gcm, ghash_acc, aad, (int)aad_sz );
322 17358907 : aes_gcm_enc_update_vaes_avx10_512( aes_gcm, le_ctr, ghash_acc, p, c, (int)sz );
323 17358907 : aes_gcm_enc_final_vaes_avx10 ( aes_gcm, le_ctr, ghash_acc, aad_sz, sz );
324 17358907 : memcpy( tag, ghash_acc, 16 );
325 17358907 : }
326 :
327 : int
328 : fd_aes_gcm_decrypt_avx10_512( fd_aes_gcm_avx10_t * aes_gcm,
329 : uchar const * c,
330 : uchar * p,
331 : ulong sz,
332 : uchar const * aad,
333 : ulong aad_sz,
334 17358403 : uchar const tag[ 16 ] ) {
335 17358403 : uint le_ctr[4]; load_le_ctr( le_ctr, aes_gcm->iv );
336 17358403 : uchar ghash_acc[16] = {0};
337 17358403 : aes_gcm_aad_update_vaes_avx10 ( aes_gcm, ghash_acc, aad, (int)aad_sz );
338 17358403 : aes_gcm_dec_update_vaes_avx10_512 ( aes_gcm, le_ctr, ghash_acc, c, p, (int)sz );
339 17358403 : return aes_gcm_dec_final_vaes_avx10( aes_gcm, le_ctr, ghash_acc, aad_sz, sz, tag, 16 );
340 17358403 : }
341 :
342 : #endif /* FD_HAS_AVX512 && FD_HAS_GFNI && FD_HAS_AESNI */
|
