Line data Source code
1 : /* fd_aes_gcm_x86.c provides wrappers and AES key expansion for
2 : fd_aes_gcm_{aesni,avx10}.S */
3 :
4 : #include "fd_aes_gcm.h"
5 : #include "../../util/simd/fd_sse.h"
6 :
7 : #if FD_HAS_AESNI
8 :
9 : FD_FN_SENSITIVE static void
10 : expand_aes_key( fd_aes_gcm_aesni_key_t * out,
11 53047977 : uchar const * keyp ) {
12 53047977 : vb_t key = vb_ldu( keyp );
13 :
14 : /* Expand encryption key */
15 :
16 53047977 : vb_t v0, v1;
17 53047977 : vb_t enc[11];
18 530479770 : # define ASSIST( out, gen ) do { \
19 530479770 : vb_t v2 = _mm_aeskeygenassist_si128( v0, (gen) ); \
20 530479770 : vb_t v3 = vu_permute2( v1, v0, 0, 0, 1, 0 ); \
21 530479770 : vb_t v4 = vu_xor ( v0, v3 ); \
22 530479770 : v1 = vu_permute2( v3, v4, 0, 3, 0, 2 ); \
23 530479770 : vb_t v5 = vu_xor ( v4, v1 ); \
24 530479770 : vb_t v6 = vu_permute ( v2, 3, 3, 3, 3 ); \
25 530479770 : v0 = vu_xor ( v5, v6 ); \
26 530479770 : (out) = v0; \
27 530479770 : } while(0)
28 53047977 : v0 = key;
29 53047977 : v1 = vu_zero();
30 53047977 : enc[ 0] = v0;
31 53047977 : ASSIST( enc[ 1], 0x01 );
32 53047977 : ASSIST( enc[ 2], 0x02 );
33 53047977 : ASSIST( enc[ 3], 0x04 );
34 53047977 : ASSIST( enc[ 4], 0x08 );
35 53047977 : ASSIST( enc[ 5], 0x10 );
36 53047977 : ASSIST( enc[ 6], 0x20 );
37 53047977 : ASSIST( enc[ 7], 0x40 );
38 53047977 : ASSIST( enc[ 8], 0x80 );
39 53047977 : ASSIST( enc[ 9], 0x1B );
40 53047977 : ASSIST( enc[10], 0x36 );
41 53047977 : # undef ASSIST
42 :
43 53047977 : vb_st( out->key_enc, enc[ 0] );
44 53047977 : vb_st( out->key_enc + 0x10, enc[ 1] );
45 53047977 : vb_st( out->key_enc + 0x20, enc[ 2] );
46 53047977 : vb_st( out->key_enc + 0x30, enc[ 3] );
47 53047977 : vb_st( out->key_enc + 0x40, enc[ 4] );
48 53047977 : vb_st( out->key_enc + 0x50, enc[ 5] );
49 53047977 : vb_st( out->key_enc + 0x60, enc[ 6] );
50 53047977 : vb_st( out->key_enc + 0x70, enc[ 7] );
51 53047977 : vb_st( out->key_enc + 0x80, enc[ 8] );
52 53047977 : vb_st( out->key_enc + 0x90, enc[ 9] );
53 53047977 : vb_st( out->key_enc + 0xa0, enc[10] );
54 :
55 : /* Derive decryption key */
56 :
57 53047977 : vb_st( out->key_dec, enc[10] );
58 53047977 : vb_st( out->key_dec + 0x10, _mm_aesimc_si128( enc[ 9] ) );
59 53047977 : vb_st( out->key_dec + 0x20, _mm_aesimc_si128( enc[ 8] ) );
60 53047977 : vb_st( out->key_dec + 0x30, _mm_aesimc_si128( enc[ 7] ) );
61 53047977 : vb_st( out->key_dec + 0x40, _mm_aesimc_si128( enc[ 6] ) );
62 53047977 : vb_st( out->key_dec + 0x50, _mm_aesimc_si128( enc[ 5] ) );
63 53047977 : vb_st( out->key_dec + 0x60, _mm_aesimc_si128( enc[ 4] ) );
64 53047977 : vb_st( out->key_dec + 0x70, _mm_aesimc_si128( enc[ 3] ) );
65 53047977 : vb_st( out->key_dec + 0x80, _mm_aesimc_si128( enc[ 2] ) );
66 53047977 : vb_st( out->key_dec + 0x90, _mm_aesimc_si128( enc[ 1] ) );
67 53047977 : vb_st( out->key_dec + 0xa0, _mm_aesimc_si128( enc[ 0] ) );
68 :
69 53047977 : out->key_sz = 16;
70 53047977 : }
71 :
72 : __attribute__((sysv_abi)) extern void
73 : aes_gcm_precompute_aesni( fd_aes_gcm_aesni_t * key );
74 :
75 : __attribute__((sysv_abi)) extern void
76 : aes_gcm_aad_update_aesni( fd_aes_gcm_aesni_t const * key,
77 : uchar ghash_acc[16],
78 : uchar const * aad,
79 : int aadlen );
80 :
81 : __attribute__((sysv_abi)) extern void
82 : aes_gcm_enc_update_aesni( fd_aes_gcm_aesni_t const * key,
83 : uint const le_ctr[4],
84 : uchar ghash_acc[16],
85 : uchar const * src,
86 : uchar * dst,
87 : int datalen );
88 :
89 : __attribute__((sysv_abi)) extern void
90 : aes_gcm_enc_final_aesni( fd_aes_gcm_aesni_t const * key,
91 : uint const le_ctr[4],
92 : uchar ghash_acc[16],
93 : ulong total_aadlen,
94 : ulong total_datalen );
95 :
96 : __attribute__((sysv_abi)) extern void
97 : aes_gcm_dec_update_aesni( fd_aes_gcm_aesni_t const * key,
98 : uint const le_ctr[4],
99 : uchar ghash_acc[16],
100 : uchar const * src,
101 : uchar * dst,
102 : int datalen );
103 :
104 : __attribute__((sysv_abi,warn_unused_result)) extern int
105 : aes_gcm_dec_final_aesni( fd_aes_gcm_aesni_t const * key,
106 : uint const le_ctr[4],
107 : uchar const ghash_acc[16],
108 : ulong total_aadlen,
109 : ulong total_datalen,
110 : uchar const tag[16],
111 : int taglen );
112 :
113 : void
114 : fd_aes_128_gcm_init_aesni( fd_aes_gcm_aesni_t * aes_gcm,
115 : uchar const key[ 16 ],
116 0 : uchar const iv [ 12 ] ) {
117 0 : expand_aes_key( &aes_gcm->key, key );
118 0 : aes_gcm_precompute_aesni( aes_gcm );
119 0 : memcpy( aes_gcm->iv, iv, 12 );
120 0 : }
121 :
122 : static void
123 : load_le_ctr( uint le_ctr[4],
124 53047977 : uchar const iv[12] ) {
125 53047977 : le_ctr[0] = 2;
126 53047977 : le_ctr[1] = fd_uint_bswap( fd_uint_load_4_fast( iv+8 ) );
127 53047977 : le_ctr[2] = fd_uint_bswap( fd_uint_load_4_fast( iv+4 ) );
128 53047977 : le_ctr[3] = fd_uint_bswap( fd_uint_load_4_fast( iv ) );
129 53047977 : }
130 :
131 : void
132 : fd_aes_gcm_encrypt_aesni( fd_aes_gcm_aesni_t * aes_gcm,
133 : uchar * c,
134 : uchar const * p,
135 : ulong sz,
136 : uchar const * aad,
137 : ulong aad_sz,
138 0 : uchar tag[ 16 ] ) {
139 0 : uint le_ctr[4]; load_le_ctr( le_ctr, aes_gcm->iv );
140 0 : uchar ghash_acc[16] = {0};
141 0 : aes_gcm_aad_update_aesni( aes_gcm, ghash_acc, aad, (int)aad_sz );
142 0 : aes_gcm_enc_update_aesni( aes_gcm, le_ctr, ghash_acc, p, c, (int)sz );
143 0 : aes_gcm_enc_final_aesni ( aes_gcm, le_ctr, ghash_acc, aad_sz, sz );
144 0 : memcpy( tag, ghash_acc, 16 );
145 0 : }
146 :
147 : int
148 : fd_aes_gcm_decrypt_aesni( fd_aes_gcm_aesni_t * aes_gcm,
149 : uchar const * c,
150 : uchar * p,
151 : ulong sz,
152 : uchar const * aad,
153 : ulong aad_sz,
154 0 : uchar const tag[ 16 ] ) {
155 0 : uint le_ctr[4]; load_le_ctr( le_ctr, aes_gcm->iv );
156 0 : uchar ghash_acc[16] = {0};
157 0 : aes_gcm_aad_update_aesni( aes_gcm, ghash_acc, aad, (int)aad_sz );
158 0 : aes_gcm_dec_update_aesni( aes_gcm, le_ctr, ghash_acc, c, p, (int)sz );
159 0 : return aes_gcm_dec_final_aesni( aes_gcm, le_ctr, ghash_acc, aad_sz, sz, tag, 16 );
160 0 : }
161 :
162 : #endif /* FD_HAS_AESNI */
163 :
164 : #if FD_HAS_AVX && FD_HAS_AESNI
165 :
166 : __attribute__((sysv_abi)) extern void
167 : aes_gcm_precompute_aesni_avx( fd_aes_gcm_aesni_t * key );
168 :
169 : __attribute__((sysv_abi)) extern void
170 : aes_gcm_aad_update_aesni_avx( fd_aes_gcm_aesni_t * key,
171 : uchar ghash_acc[16],
172 : uchar const * aad,
173 : int aadlen );
174 :
175 : __attribute__((sysv_abi)) extern void
176 : aes_gcm_enc_update_aesni_avx( fd_aes_gcm_aesni_t * key,
177 : uint const le_ctr[4],
178 : uchar ghash_acc[16],
179 : uchar const * src,
180 : uchar * dst,
181 : int datalen );
182 :
183 : __attribute__((sysv_abi)) extern void
184 : aes_gcm_enc_final_aesni_avx( fd_aes_gcm_aesni_t * key,
185 : uint const le_ctr[4],
186 : uchar ghash_acc[16],
187 : ulong total_aadlen,
188 : ulong total_datalen );
189 :
190 : __attribute__((sysv_abi)) extern void
191 : aes_gcm_dec_update_aesni_avx( fd_aes_gcm_aesni_t const * key,
192 : uint const le_ctr[4],
193 : uchar ghash_acc[16],
194 : uchar const * src,
195 : uchar * dst,
196 : int datalen );
197 :
198 : __attribute__((sysv_abi,warn_unused_result)) extern int
199 : aes_gcm_dec_final_aesni_avx( fd_aes_gcm_aesni_t const * key,
200 : uint const le_ctr[4],
201 : uchar const ghash_acc[16],
202 : ulong total_aadlen,
203 : ulong total_datalen,
204 : uchar const tag[16],
205 : int taglen );
206 :
207 : void
208 : fd_aes_128_gcm_init_avx2( fd_aes_gcm_aesni_t * aes_gcm,
209 : uchar const key[ 16 ],
210 25719072 : uchar const iv [ 12 ] ) {
211 25719072 : expand_aes_key( fd_type_pun( &aes_gcm->key ), key );
212 25719072 : aes_gcm_precompute_aesni_avx( aes_gcm );
213 25719072 : memcpy( aes_gcm->iv, iv, 12 );
214 25719072 : }
215 :
216 : void
217 : fd_aes_gcm_encrypt_avx2( fd_aes_gcm_aesni_t * aes_gcm,
218 : uchar * c,
219 : uchar const * p,
220 : ulong sz,
221 : uchar const * aad,
222 : ulong aad_sz,
223 13094777 : uchar tag[ 16 ] ) {
224 13094777 : uint le_ctr[4]; load_le_ctr( le_ctr, aes_gcm->iv );
225 13094777 : uchar ghash_acc[16] = {0};
226 13094777 : aes_gcm_aad_update_aesni_avx( aes_gcm, ghash_acc, aad, (int)aad_sz );
227 13094777 : aes_gcm_enc_update_aesni_avx( aes_gcm, le_ctr, ghash_acc, p, c, (int)sz );
228 13094777 : aes_gcm_enc_final_aesni_avx ( aes_gcm, le_ctr, ghash_acc, aad_sz, sz );
229 13094777 : memcpy( tag, ghash_acc, 16 );
230 13094777 : }
231 :
232 : int
233 : fd_aes_gcm_decrypt_avx2( fd_aes_gcm_aesni_t * aes_gcm,
234 : uchar const * c,
235 : uchar * p,
236 : ulong sz,
237 : uchar const * aad,
238 : ulong aad_sz,
239 12624295 : uchar const tag[ 16 ] ) {
240 12624295 : uint le_ctr[4]; load_le_ctr( le_ctr, aes_gcm->iv );
241 12624295 : uchar ghash_acc[16] = {0};
242 12624295 : aes_gcm_aad_update_aesni_avx( aes_gcm, ghash_acc, aad, (int)aad_sz );
243 12624295 : aes_gcm_dec_update_aesni_avx( aes_gcm, le_ctr, ghash_acc, c, p, (int)sz );
244 12624295 : return aes_gcm_dec_final_aesni_avx( aes_gcm, le_ctr, ghash_acc, aad_sz, sz, tag, 16 );
245 12624295 : }
246 :
247 : #endif /* FD_HAS_AVX && FD_HAS_AESNI */
248 :
249 : #if FD_HAS_AVX512 && FD_HAS_GFNI && FD_HAS_AESNI
250 :
251 : __attribute__((sysv_abi)) extern void
252 : aes_gcm_precompute_vaes_avx10_512( fd_aes_gcm_avx10_t * key );
253 :
254 : __attribute__((sysv_abi)) extern void
255 : aes_gcm_aad_update_vaes_avx10( fd_aes_gcm_avx10_t const * key,
256 : uchar ghash_acc[16],
257 : uchar const * aad,
258 : int aadlen );
259 :
260 : __attribute__((sysv_abi)) extern void
261 : aes_gcm_enc_update_vaes_avx10_512( fd_aes_gcm_avx10_t * key,
262 : uint const le_ctr[4],
263 : uchar ghash_acc[16],
264 : uchar const * src,
265 : uchar * dst,
266 : int datalen );
267 :
268 : __attribute__((sysv_abi)) extern void
269 : aes_gcm_enc_final_vaes_avx10( fd_aes_gcm_avx10_t * key,
270 : uint const le_ctr[4],
271 : uchar ghash_acc[16],
272 : ulong total_aadlen,
273 : ulong total_datalen );
274 :
275 : __attribute__((sysv_abi)) extern void
276 : aes_gcm_dec_update_vaes_avx10_512( fd_aes_gcm_avx10_t const * key,
277 : uint const le_ctr[4],
278 : uchar ghash_acc[16],
279 : uchar const * src,
280 : uchar * dst,
281 : int datalen );
282 :
283 : __attribute__((sysv_abi,warn_unused_result)) extern int
284 : aes_gcm_dec_final_vaes_avx10( fd_aes_gcm_avx10_t const * key,
285 : uint const le_ctr[4],
286 : uchar const ghash_acc[16],
287 : ulong total_aadlen,
288 : ulong total_datalen,
289 : uchar const tag[16],
290 : int taglen );
291 :
292 : void
293 : fd_aes_128_gcm_init_avx10( fd_aes_gcm_avx10_t * aes_gcm,
294 : uchar const key[ 16 ],
295 0 : uchar const iv [ 12 ] ) {
296 0 : expand_aes_key( &aes_gcm->key, key );
297 0 : aes_gcm_precompute_vaes_avx10_512( aes_gcm );
298 0 : memcpy( aes_gcm->iv, iv, 12 );
299 0 : }
300 :
301 : void
302 : fd_aes_128_gcm_init_avx10_512( fd_aes_gcm_avx10_t * aes_gcm,
303 : uchar const key[ 16 ],
304 27328905 : uchar const iv [ 12 ] ) {
305 27328905 : expand_aes_key( &aes_gcm->key, key );
306 27328905 : aes_gcm_precompute_vaes_avx10_512( aes_gcm );
307 27328905 : memcpy( aes_gcm->iv, iv, 12 );
308 27328905 : }
309 :
310 : void
311 : fd_aes_gcm_encrypt_avx10_512( fd_aes_gcm_avx10_t * aes_gcm,
312 : uchar * c,
313 : uchar const * p,
314 : ulong sz,
315 : uchar const * aad,
316 : ulong aad_sz,
317 13901092 : uchar tag[ 16 ] ) {
318 13901092 : uint le_ctr[4]; load_le_ctr( le_ctr, aes_gcm->iv );
319 13901092 : uchar ghash_acc[16] = {0};
320 13901092 : aes_gcm_aad_update_vaes_avx10 ( aes_gcm, ghash_acc, aad, (int)aad_sz );
321 13901092 : aes_gcm_enc_update_vaes_avx10_512( aes_gcm, le_ctr, ghash_acc, p, c, (int)sz );
322 13901092 : aes_gcm_enc_final_vaes_avx10 ( aes_gcm, le_ctr, ghash_acc, aad_sz, sz );
323 13901092 : memcpy( tag, ghash_acc, 16 );
324 13901092 : }
325 :
326 : int
327 : fd_aes_gcm_decrypt_avx10_512( fd_aes_gcm_avx10_t * aes_gcm,
328 : uchar const * c,
329 : uchar * p,
330 : ulong sz,
331 : uchar const * aad,
332 : ulong aad_sz,
333 13427813 : uchar const tag[ 16 ] ) {
334 13427813 : uint le_ctr[4]; load_le_ctr( le_ctr, aes_gcm->iv );
335 13427813 : uchar ghash_acc[16] = {0};
336 13427813 : aes_gcm_aad_update_vaes_avx10 ( aes_gcm, ghash_acc, aad, (int)aad_sz );
337 13427813 : aes_gcm_dec_update_vaes_avx10_512 ( aes_gcm, le_ctr, ghash_acc, c, p, (int)sz );
338 13427813 : return aes_gcm_dec_final_vaes_avx10( aes_gcm, le_ctr, ghash_acc, aad_sz, sz, tag, 16 );
339 13427813 : }
340 :
341 : #endif /* FD_HAS_AVX512 && FD_HAS_GFNI && FD_HAS_AESNI */
|
