Line data Source code
1 : #ifndef HEADER_fd_src_util_simd_fd_avx_h
2 : #error "Do not include this directly; use fd_avx.h"
3 : #endif
4 :
5 : /* Vector ulong API ***************************************************/
6 :
7 : /* A wv_t is a vector where each adjacent pair of 32-bit wide lanes
8 : (e.g. 0-1 / 2-3 / 4-5 / 6-7) holds an unsigned 64-bit integer (a
9 : "ulong").
10 :
11 : These mirror the other APIs as much as possible. Macros are
12 : preferred over static inlines when it is possible to do it robustly
13 : to reduce the risk of the compiler mucking it up. */
14 :
15 1591658897 : #define wv_t __m256i
16 :
17 : /* Constructors */
18 :
19 : /* Given the ulong values, return ... */
20 :
21 3827191 : #define wv(v0,v1,v2,v3) _mm256_setr_epi64x( (long)(v0), (long)(v1), (long)(v2), (long)(v3) ) /* [ v0 v1 v2 v3 ] */
22 :
23 266945100 : #define wv_bcast(v0) _mm256_set1_epi64x( (long)(v0) ) /* [ v0 v0 v0 v0 ] */
24 :
25 : static inline wv_t /* [ v0 v1 v0 v1 ] */
26 196608 : wv_bcast_pair( ulong v0, ulong v1 ) {
27 196608 : return _mm256_setr_epi64x( (long)v0, (long)v1, (long)v0, (long)v1 );
28 196608 : }
29 :
30 : static inline wv_t /* [ v0 v0 v1 v1 ] */
31 196608 : wv_bcast_wide( ulong v0, ulong v1 ) {
32 196608 : return _mm256_setr_epi64x( (long)v0, (long)v0, (long)v1, (long)v1 );
33 196608 : }
34 :
35 : /* wv_permute returns [ l(imm_v0) l(imm_i1) l(imm_i2) l(imm_i3) ].
36 : imm_i* should be compile time constants in 0:3. */
37 :
38 : #if FD_USING_CLANG /* Sigh ... clang is sad and can't handle passing compile time const expressions through a static inline */
39 :
40 : static inline wv_t
41 2162688 : wv_permute( wv_t x, int imm_i0, int imm_i1, int imm_i2, int imm_i3 ) {
42 2162688 : union { ulong u[4]; __m256i v[1]; } t, u;
43 2162688 : _mm256_store_si256( t.v, x );
44 2162688 : u.u[0] = t.u[ imm_i0 ];
45 2162688 : u.u[1] = t.u[ imm_i1 ];
46 2162688 : u.u[2] = t.u[ imm_i2 ];
47 2162688 : u.u[3] = t.u[ imm_i3 ];
48 2162688 : return _mm256_load_si256( u.v );
49 2162688 : }
50 :
51 : #else
52 :
53 : #define wv_permute(x,imm_i0,imm_i1,imm_i2,imm_i3) _mm256_permute4x64_epi64( (x), (imm_i0)+4*(imm_i1)+16*(imm_i2)+64*(imm_i3) )
54 :
55 : #endif
56 :
57 : /* Predefined constants */
58 :
59 30608906 : #define wv_zero() _mm256_setzero_si256() /* Return [ 0UL 0UL 0UL 0UL ] */
60 109970179 : #define wv_one() _mm256_set1_epi64x( 1L ) /* Return [ 1UL 1UL 1UL 1UL ] */
61 :
62 : /* Memory operations */
63 :
64 : /* wv_ld return the 4 ulongs at the 32-byte aligned / 32-byte sized
65 : location p as a vector ulong. wv_ldu is the same but p does not have
66 : to be aligned. wv_st writes the vector ulong to the 32-byte aligned
67 : / 32-byte sized location p as 4 ulongs. wv_stu is the same but p
68 : does not have to be aligned. In all these 64-bit lane l wvll be at
69 : p[l]. FIXME: USE ATTRIBUTES ON P PASSED TO THESE?
70 :
71 : Note: gcc knows a __m256i may alias. */
72 :
73 296047005 : static inline wv_t wv_ld( ulong const * p ) { return _mm256_load_si256( (__m256i const *)p ); }
74 664384216 : static inline void wv_st( ulong * p, wv_t i ) { _mm256_store_si256( (__m256i *)p, i ); }
75 :
76 994886951 : static inline wv_t wv_ldu( void const * p ) { return _mm256_loadu_si256( (__m256i const *)p ); }
77 445042291 : static inline void wv_stu( void * p, wv_t i ) { _mm256_storeu_si256( (__m256i *)p, i ); }
78 :
79 : /* wv_ldif is an optimized equivalent to wv_notczero(c,wv_ldu(p)) (may
80 : have different behavior if c is not a proper vector conditional). It
81 : is provided for symmetry with the wv_stif operation. wv_stif stores
82 : x(n) to p[n] if c(n) is true and leaves p[n] unchanged otherwise.
83 : Undefined behavior if c is not a proper vector conditional. */
84 :
85 : #define wv_ldif(c,p) _mm256_maskload_epi64( (p),(c))
86 : #define wv_stif(c,p,x) _mm256_maskstore_epi64((p),(c),(x))
87 :
88 : /* Element operations */
89 :
90 : /* wv_extract extracts the ulong in lane imm from the vector ulong as a
91 : ulong. wv_insert returns the vector ulong formed by replacing the
92 : value in lane imm of a with the provided ulong. imm should be a
93 : compile time known in 0:3. wv_extract_variable and
94 : wv_insert_variable are the slower but the lane n does not have to be
95 : known at compile time (should still be in 0:3).
96 :
97 : Note: C99 TC3 allows type punning through a union. */
98 :
99 1691218668 : #define wv_extract(a,imm) ((ulong)_mm256_extract_epi64( (a), (imm) ))
100 :
101 439880716 : #define wv_insert(a,imm,v) _mm256_insert_epi64( (a), (long)(v), (imm) )
102 :
103 : static inline ulong
104 439880716 : wv_extract_variable( wv_t a, int n ) {
105 439880716 : union { __m256i m[1]; ulong u[4]; } t[1];
106 439880716 : _mm256_store_si256( t->m, a );
107 439880716 : return t->u[n];
108 439880716 : }
109 :
110 : static inline wv_t
111 439880716 : wv_insert_variable( wv_t a, int n, ulong v ) {
112 439880716 : union { __m256i m[1]; ulong u[4]; } t[1];
113 439880716 : _mm256_store_si256( t->m, a );
114 439880716 : t->u[n] = v;
115 439880716 : return _mm256_load_si256( t->m );
116 439880716 : }
117 :
118 : /* Given [a0 a1 a2 a3] and/or [b0 b1 b2 b3], return ... */
119 :
120 : /* Arithmetic operations */
121 :
122 : #define wv_neg(a) _mm256_sub_epi64( _mm256_setzero_si256(), (a) ) /* [ -a0 -a1 ... -a3 ] */
123 : #define wv_abs(a) (a) /* [ |a0| |a1| ... |a3| ] */
124 :
125 : /* Note: _mm256_{min,max}_epu64 are missing pre AVX-512. We emulate
126 : these on pre AVX-512 targets below (and use the AVX-512 versions if
127 : possible). Likewise, there is no _mm256_mullo_epi64 pre AVX-512.
128 : Since this is not cheap to emulate, we only provide wv_mul if it can
129 : run natively. There is a 64L*64L->64 multiply (where the lower 32-bits of
130 : the inputs will be zero extended to 64-bits beforehand) though and
131 : that is very useful. So we do provide that. */
132 :
133 1448657080 : #define wv_add(a,b) _mm256_add_epi64( (a), (b) ) /* [ a0 +b0 a1 +b1 ... a3 +b3 ] */
134 : #define wv_sub(a,b) _mm256_sub_epi64( (a), (b) ) /* [ a0 -b0 a1 -b1 ... a3 -b3 ] */
135 : #if FD_HAS_AVX512 && defined(__AVX512DQ__) && defined(__AVX512VL__)
136 1168048592 : #define wv_mul(a,b) _mm256_mullo_epi64( (a), (b) ) /* [ a0 *b0 a1 *b1 ... a3 *b3 ] */
137 : #endif
138 : #define wv_mul_ll(a,b) _mm256_mul_epu32( (a), (b) ) /* [ a0l*b0l a1l*b1l ... a3l *b3l ] */
139 :
140 : /* Binary operations */
141 :
142 : /* Note: wv_shl/wv_shr is a left/right shift by imm bits; imm should be
143 : a compile time constant in 0:63. The variable variants are slower
144 : but do not require the shift amount to be known at compile time
145 : (should still be in 0:63). */
146 :
147 : #define wv_not(a) _mm256_xor_si256( _mm256_set1_epi64x( -1L ), (a) ) /* [ ~a0 ~a1 ... ~a3 ] */
148 :
149 : #define wv_shl(a,imm) _mm256_slli_epi64( (a), (imm) ) /* [ a0<<imm a1<<imm ... a3<<imm ] */
150 : #define wv_shr(a,imm) _mm256_srli_epi64( (a), (imm) ) /* [ a0>>imm a1>>imm ... a3>>imm ] */
151 :
152 : #define wv_shl_variable(a,n) _mm256_sll_epi64( (a), _mm_insert_epi64( _mm_setzero_si128(), (n), 0 ) )
153 : #define wv_shr_variable(a,n) _mm256_srl_epi64( (a), _mm_insert_epi64( _mm_setzero_si128(), (n), 0 ) )
154 :
155 : #define wv_shl_vector(a,b) _mm256_sllv_epi64( (a), (b) ) /* [ a0<<b0 a1<<b1 ... a3<<b3 ] */
156 : #define wv_shr_vector(a,b) _mm256_srlv_epi64( (a), (b) ) /* [ a0>>b0 a1>>b1 ... a3>>b3 ] */
157 :
158 : #define wv_and(a,b) _mm256_and_si256( (a), (b) ) /* [ a0 &b0 a1& b1 ... a3& b3 ] */
159 408664 : #define wv_andnot(a,b) _mm256_andnot_si256( (a), (b) ) /* [ (~a0)&b0 (~a1)&b1 ... (~a3)&b3 ] */
160 1187526508 : #define wv_or(a,b) _mm256_or_si256( (a), (b) ) /* [ a0 |b0 a1 |b1 ... a3 |b3 ] */
161 : #define wv_xor(a,b) _mm256_xor_si256( (a), (b) ) /* [ a0 ^b0 a1 ^b1 ... a3 ^b3 ] */
162 :
163 : /* wv_rol(x,n) returns wv( rotate_left (x0,n), rotate_left (x1,n), ... )
164 : wv_ror(x,n) returns wv( rotate_right(x0,n), rotate_right(x1,n), ... ) */
165 :
166 : #if FD_HAS_AVX512
167 584024296 : #define wv_rol(a,imm) _mm256_rol_epi64( (a), (imm) )
168 : #define wv_ror(a,imm) _mm256_ror_epi64( (a), (imm) )
169 : #else
170 36486904 : static inline wv_t wv_rol( wv_t a, int imm ) { return wv_or( wv_shl( a, imm & 63 ), wv_shr( a, (-imm) & 63 ) ); }
171 1121344256 : static inline wv_t wv_ror( wv_t a, int imm ) { return wv_or( wv_shr( a, imm & 63 ), wv_shl( a, (-imm) & 63 ) ); }
172 : #endif
173 :
174 12582912 : static inline wv_t wv_rol_variable( wv_t a, int n ) { return wv_or( wv_shl_variable( a, n&63 ), wv_shr_variable( a, (-n)&63 ) ); }
175 12582912 : static inline wv_t wv_ror_variable( wv_t a, int n ) { return wv_or( wv_shr_variable( a, n&63 ), wv_shl_variable( a, (-n)&63 ) ); }
176 :
177 : #if FD_HAS_AVX512 && defined(__AVX512VL__)
178 99954546 : #define wv_rol_vector(a,b) _mm256_rolv_epi64( (a), (b) )
179 : #define wv_ror_vector(a,b) _mm256_rorv_epi64( (a), (b) )
180 : #else
181 0 : static inline wv_t wv_rol_vector( wv_t a, wl_t b ) {
182 0 : wl_t m = wl_bcast( 63L );
183 0 : return wv_or( wv_shl_vector( a, wl_and( b, m ) ), wv_shr_vector( a, wl_and( wl_neg( b ), m ) ) );
184 0 : }
185 :
186 0 : static inline wv_t wv_ror_vector( wv_t a, wl_t b ) {
187 0 : wl_t m = wl_bcast( 63L );
188 0 : return wv_or( wv_shr_vector( a, wl_and( b, m ) ), wv_shl_vector( a, wl_and( wl_neg( b ), m ) ) );
189 0 : }
190 : #endif
191 :
192 4058508 : #define wv_bswap(a) wu_to_wv_raw( wu_bswap( wv_to_wu_raw( wv_rol( (a), 32 ) ) ) )
193 :
194 : /* Logical operations */
195 :
196 : /* Like noted below in the converters, Intel clearly has the hardware to
197 : do a _mm256_cmpgt_epu64 given that _mm256_cmpgt_epi64 exists but
198 : doesn't expose it in the ISA pre AVX-512. Sigh ... twos complement
199 : bit tricks to the rescue for wu_{gt,lt,ge,le}. */
200 :
201 : #define wv_lnot(a) _mm256_cmpeq_epi64( (a), _mm256_setzero_si256() ) /* [ !a0 !a1 ... !a3 ] */
202 : #define wv_lnotnot(a) /* [ !!a0 !!a1 ... !!a3 ] */ \
203 : _mm256_xor_si256( _mm256_set1_epi64x( -1L ), _mm256_cmpeq_epi64( (a), _mm256_setzero_si256() ) )
204 :
205 : #define wv_eq(a,b) _mm256_cmpeq_epi64( (a), (b) ) /* [ a0==b0 a1==b1 ... a3==b3 ] */
206 : #define wv_gt(a,b) /* [ a0> b0 a1> b1 ... a3> b3 ] */ \
207 : _mm256_cmpgt_epi64( _mm256_sub_epi64( (a), _mm256_set1_epi64x( (long)(1UL<<63) ) ), \
208 : _mm256_sub_epi64( (b), _mm256_set1_epi64x( (long)(1UL<<63) ) ) )
209 : #define wv_lt(a,b) wv_gt( (b), (a) ) /* [ a0< b0 a1< b1 ... a3< b3 ] */
210 : #define wv_ne(a,b) _mm256_xor_si256( _mm256_set1_epi64x(-1L), _mm256_cmpeq_epi64( (a), (b) ) ) /* [ a0!=b0 a1!=b1 ... a3!=b3 ] */
211 : #define wv_ge(a,b) _mm256_xor_si256( _mm256_set1_epi64x(-1L), wv_gt( (b), (a) ) ) /* [ a0>=b0 a1>=b1 ... a3>=b3 ] */
212 : #define wv_le(a,b) _mm256_xor_si256( _mm256_set1_epi64x(-1L), wv_gt( (a), (b) ) ) /* [ a0<=b0 a1<=b1 ... a3<=b3 ] */
213 :
214 : /* Conditional operations */
215 :
216 : #define wv_czero(c,f) _mm256_andnot_si256( (c), (f) ) /* [ c0?0UL:f0 c1?0UL:f1 ... c3?0UL:f3 ] */
217 60250600 : #define wv_notczero(c,f) _mm256_and_si256( (c), (f) ) /* [ c0?f0:0UL c1?f1:0UL ... c3?f3:0UL ] */
218 :
219 278824236 : #define wv_if(c,t,f) _mm256_blendv_epi8( (f), (t), (c) ) /* [ c0?t0:f0 c1?t1:f1 ... c3?t3:f3 ] */
220 :
221 : #if defined(__AVX512F__) && defined(__AVX512VL__) /* See note above */
222 131072 : #define wv_min(a,b) _mm256_min_epu64( (a), (b) )
223 131072 : #define wv_max(a,b) _mm256_max_epu64( (a), (b) )
224 : #else
225 393216 : static inline wv_t wv_min( wv_t a, wv_t b ) { return wv_if( wv_lt( a, b ), a, b ); }
226 393216 : static inline wv_t wv_max( wv_t a, wv_t b ) { return wv_if( wv_gt( a, b ), a, b ); }
227 : #endif
228 :
229 : /* Conversion operations */
230 :
231 : /* Summarizing:
232 :
233 : wv_to_wc(d) returns [ !!v0 !!v0 !!v1 !!v1 ... !!v3 !!v3 ]
234 :
235 : wv_to_wf(l,i,0) returns [ (float)v0 (float)v1 (float)v2 (float)v3 f4 f5 f6 f7 ]
236 : wv_to_wf(l,i,1) returns [ f0 f1 f2 f3 (float)v0 (float)v1 (float)v2 (float)v3 ]
237 :
238 : wv_to_wi(l,i,0) returns [ (int)v0 (int)v1 (int)v2 (int)v3 i4 i5 i6 i7 ]
239 : wv_to_wi(l,i,1) returns [ i0 i1 i2 i3 (int)v0 (int)v1 (int)v2 (int)v3 ]
240 :
241 : wv_to_wu(l,u,0) returns [ (uint)v0 (uint)v1 (uint)v2 (uint)v3 u4 u5 u6 u7 ]
242 : wv_to_wu(l,u,1) returns [ v0 v1 v2 v3 (uint)v0 (uint)v1 (uint)v2 (uint)v3 ]
243 :
244 : wv_to_wd(l) returns [ (double)v0 (double)v1 (double)v2 (double)v3 ]
245 :
246 : wv_to_wl(l) returns [ (long)v0 (long)v1 (long)v2 (long)v3 ]
247 :
248 : The raw variants just treat the raw bits as the corresponding vector
249 : type. For wv_to_wc_raw, the user promises wv contains a proper
250 : vector conditional (e.g. 0 or -1 in each lane). The others are
251 : provided to facilitate doing advanced bit tricks on floating point
252 : values. */
253 :
254 157121248 : #define wv_to_wc(a) _mm256_xor_si256( _mm256_set1_epi64x( -1L ), _mm256_cmpeq_epi64( (a), _mm256_setzero_si256() ) )
255 :
256 393216 : static inline wf_t wv_to_wf( wv_t v, wf_t f, int imm_hi ) {
257 393216 : union { ulong u[4]; __m256i v[1]; } t[1];
258 393216 : union { float f[4]; __m128 v[1]; } u[1];
259 393216 : _mm256_store_si256( t->v, v );
260 393216 : u->f[0] = (float)t->u[0];
261 393216 : u->f[1] = (float)t->u[1];
262 393216 : u->f[2] = (float)t->u[2];
263 393216 : u->f[3] = (float)t->u[3];
264 393216 : __m128 w = _mm_load_ps( u->f );
265 393216 : return imm_hi ? _mm256_insertf128_ps( f, w, 1 ) : _mm256_insertf128_ps( f, w, 0 ); /* compile time */
266 393216 : }
267 :
268 393216 : static inline wv_t wv_to_wi( wv_t v, wi_t i, int imm_hi ) {
269 393216 : __m128 v01 = _mm_castsi128_ps( _mm256_extractf128_si256( v, 0 ) ); /* [ v0l v0h v1l v1h ] */
270 393216 : __m128 v23 = _mm_castsi128_ps( _mm256_extractf128_si256( v, 1 ) ); /* [ v2l v2h v3l v3h ] */
271 393216 : __m128i w = _mm_castps_si128( _mm_shuffle_ps( v01, v23, _MM_SHUFFLE(2,0,2,0) ) );
272 393216 : return imm_hi ? _mm256_insertf128_si256( i, w, 1 ) : _mm256_insertf128_si256( i, w, 0 ); /* compile time */
273 393216 : }
274 :
275 393216 : static inline wu_t wv_to_wu( wv_t v, wu_t u, int imm_hi ) {
276 393216 : __m128 v01 = _mm_castsi128_ps( _mm256_extractf128_si256( v, 0 ) ); /* [ v0l v0h v1l v1h ] */
277 393216 : __m128 v23 = _mm_castsi128_ps( _mm256_extractf128_si256( v, 1 ) ); /* [ v2l v2h v3l v3h ] */
278 393216 : __m128i w = _mm_castps_si128( _mm_shuffle_ps( v01, v23, _MM_SHUFFLE(2,0,2,0) ) );
279 393216 : return imm_hi ? _mm256_insertf128_si256( u, w, 1 ) : _mm256_insertf128_si256( u, w, 0 ); /* compile time */
280 393216 : }
281 :
282 : /* FIXME: IS IT FASTER TO USE INSERT / EXTRACT HERE? */
283 196608 : static inline wd_t wv_to_wd( wv_t v ) {
284 196608 : union { ulong u[4]; __m256i v[1]; } t[1];
285 196608 : union { double d[4]; __m256d v[1]; } u[1];
286 196608 : _mm256_store_si256( t->v, v );
287 196608 : u->d[0] = (double)t->u[0];
288 196608 : u->d[1] = (double)t->u[1];
289 196608 : u->d[2] = (double)t->u[2];
290 196608 : u->d[3] = (double)t->u[3];
291 196608 : return _mm256_load_pd( u->d );
292 196608 : }
293 :
294 : #define wv_to_wl(a) (a)
295 :
296 : #define wv_to_wc_raw(a) (a)
297 : #define wv_to_wf_raw(a) _mm256_castsi256_ps( (a) )
298 : #define wv_to_wi_raw(a) (a)
299 : #define wv_to_wu_raw(a) (a)
300 : #define wv_to_wd_raw(a) _mm256_castsi256_pd( (a) )
301 : #define wv_to_wl_raw(a) (a)
302 :
303 : /* Reduction operations */
304 :
305 : static inline wv_t
306 196608 : wv_sum_all( wv_t x ) { /* Returns wv_bcast( sum( x ) ) */
307 196608 : x = _mm256_add_epi64( x, _mm256_permute2f128_si256( x, x, 1 ) );
308 196608 : return _mm256_add_epi64( x, _mm256_castpd_si256( _mm256_permute_pd( _mm256_castsi256_pd( x ), 5 ) ) );
309 196608 : }
310 :
311 : static inline wv_t
312 196608 : wv_min_all( wv_t x ) { /* Returns wv_bcast( min( x ) ) */
313 196608 : x = wv_min( x, _mm256_permute2f128_si256( x, x, 1 ) );
314 196608 : return wv_min( x, _mm256_castpd_si256( _mm256_permute_pd( _mm256_castsi256_pd( x ), 5 ) ) );
315 196608 : }
316 :
317 : static inline wv_t
318 196608 : wv_max_all( wv_t x ) { /* Returns wv_bcast( max( x ) ) */
319 196608 : x = wv_max( x, _mm256_permute2f128_si256( x, x, 1 ) );
320 196608 : return wv_max( x, _mm256_castpd_si256( _mm256_permute_pd( _mm256_castsi256_pd( x ), 5 ) ) );
321 196608 : }
322 :
323 : /* Misc operations */
324 :
325 : /* wv_gather(b,i,imm_hi) returns
326 : [ b[i(0)] b[i(1)] b[i(2)] b[i(3)] ] if imm_hi is 0 and
327 : [ b[i(4)] b[i(5)] b[i(6)] b[i(7)] ] o.w.
328 : where b is a "ulong const*", i is wi_t and imm_hi is a compile time
329 : constant. We use a static inline here instead of a define to keep
330 : strict type checking while working around yet another Intel intrinsic
331 : type mismatch issue. */
332 :
333 219940358 : static inline wv_t wv_gather( ulong const * b, wi_t i, int imm_hi ) {
334 : /* A compile time branch, but older versions of GCC can't handle the
335 : ternary operator with -O0 */
336 219940358 : if( imm_hi ) return _mm256_i32gather_epi64( (long long const *)b, _mm256_extractf128_si256( i, 1 ), 8 );
337 109970179 : else return _mm256_i32gather_epi64( (long long const *)b, _mm256_extractf128_si256( i, 0 ), 8 );
338 219940358 : }
339 :
340 : /* wv_transpose_4x4 transposes the 4x4 matrix stored in wv_t r0,r1,r2,r3
341 : and stores the result in 4x4 matrix wv_t c0,c1,c2,c3. All
342 : c0,c1,c2,c3 should be different for a well defined result.
343 : Otherwise, in-place operation and/or using the same wv_t to specify
344 : multiple rows of r is fine. */
345 :
346 7903680 : #define wv_transpose_4x4( r0,r1,r2,r3, c0,c1,c2,c3 ) do { \
347 7903680 : wv_t _wv_transpose_r0 = (r0); wv_t _wv_transpose_r1 = (r1); wv_t _wv_transpose_r2 = (r2); wv_t _wv_transpose_r3 = (r3); \
348 7903680 : wv_t _wv_transpose_t; \
349 7903680 : /* Transpose 2x2 blocks */ \
350 7903680 : _wv_transpose_t = _wv_transpose_r0; _wv_transpose_r0 = _mm256_permute2f128_si256( _wv_transpose_t, _wv_transpose_r2, 0x20 ); \
351 7903680 : /**/ _wv_transpose_r2 = _mm256_permute2f128_si256( _wv_transpose_t, _wv_transpose_r2, 0x31 ); \
352 7903680 : _wv_transpose_t = _wv_transpose_r1; _wv_transpose_r1 = _mm256_permute2f128_si256( _wv_transpose_t, _wv_transpose_r3, 0x20 ); \
353 7903680 : /**/ _wv_transpose_r3 = _mm256_permute2f128_si256( _wv_transpose_t, _wv_transpose_r3, 0x31 ); \
354 7903680 : /* Transpose 1x1 blocks */ \
355 7903680 : /**/ (c0) = _mm256_unpacklo_epi64( _wv_transpose_r0, _wv_transpose_r1 ); \
356 7903680 : /**/ (c1) = _mm256_unpackhi_epi64( _wv_transpose_r0, _wv_transpose_r1 ); \
357 7903680 : /**/ (c2) = _mm256_unpacklo_epi64( _wv_transpose_r2, _wv_transpose_r3 ); \
358 7903680 : /**/ (c3) = _mm256_unpackhi_epi64( _wv_transpose_r2, _wv_transpose_r3 ); \
359 7903680 : } while(0)
|
