Line data Source code
1 : #ifndef HEADER_fd_src_util_simd_fd_nt_memcpy_h
2 : #define HEADER_fd_src_util_simd_fd_nt_memcpy_h
3 :
4 : #if FD_HAS_SSE || FD_HAS_AVX || FD_HAS_AVX512
5 :
6 : /* Unlike the other headers in this directory, this one does not contain
7 : a vector API. Instead, it includes files for vector accelerated
8 : non-temporal memcpy. In x86, using non-temporal store requires
9 : either assembly or the emmintrin.h header, but really the best way to
10 : use them is via SIMD instructions, which is why the functions are
11 : here.
12 :
13 : Crash course on non-temporal memory hints and write combining memory:
14 :
15 : When we think about normal memory that an application uses, what
16 : we're typically thinking of is memory classified as Write Back (WB)
17 : type memory. WB memory is cacheable and provides total store
18 : ordering. Modern x86 CPUs, however, support other types of memory,
19 : which is normally used for memory-mapped I/O and frame-buffer memory
20 : for a graphics system. (The OS configures these memory types using
21 : the Memory Type Range Registers or the Page Attribute Table.)
22 :
23 : The Write Combining (WC) memory type, unlike WB, on the other hand,
24 : does not allow caching, but it does allow the CPU to delay, reorder,
25 : and combine writes to the same 64 byte cache line. That is, writes
26 : to a cache line of WC memory are initially staged in a special WC
27 : buffer instead of the normal cache. As long as that data remains in
28 : the WC buffer, additional writes to it are "combined" in the WC
29 : buffer; at this point, none of the writes are visible to other
30 : processors, since the WC buffer does not listen to snooping. At some
31 : point, the WC buffer is evicted, and the 64-byte chunk of data is
32 : written back to main memory. AMD's optimization guides mention
33 : that, at least for Zen 4 and Zen 5, writing all 64 bytes is enough to
34 : trigger evicting the WC buffer. Intel's manual makes no such
35 : promise. Executing an SFENCE or MFENCE instruction is always
36 : sufficient to evict all WC buffers, however.
37 :
38 : This is relevant because using a non-temporal write causes the
39 : processor to treat WB memory as if it were WC. In order to do this,
40 : the processor must first make sure the memory is not in the normal
41 : cache (evicting it if so). The written data then goes to the WC
42 : buffer, bypassing the cache. It is eventually written back to main
43 : memory, at which point it becomes visible to other cores. In all of
44 : these steps, it never touches the normal cache, which means it won't
45 : pollute anything there.
46 :
47 : On completely the other hand, the SIMD non-temporal loads are
48 : documented to do nothing special when reading from WB memory. That
49 : means if we want non-temporal-like behavior, we need to implement it
50 : another way. Currently, the least polluting way is to pre-fetch the
51 : data with the non-temporal hint (PREFETCHNTA) and then demote it to
52 : L3 afterwards (CLDEMOTE). It may still evict data on the way in, and
53 : Intel's manual isn't as descriptive about what this actually does,
54 : but it should reduce the amount of cache pollution. */
55 :
56 :
57 :
58 : /* fd_memcpy_{nn,nt,tn,tt} copies `sz` bytes from the source (`_s`) to
59 : the destination (`_d`), where either the source is loaded with the
60 : non-temporal memory hint, the destination is stored with the
61 : non-temporal memory hint, or both. `sz` need not be a multiple of
62 : 64. `s` and `d` must not overlap. Returns `d`.
63 :
64 : The first letter of the function suffix indicates how the destination
65 : is stored, and the second letter of the suffix indicates how the
66 : source is loaded. n means non-temporal while t means temporal
67 : (regular load/store). fd_memcpy_tt is basically just normal memcpy
68 : and is only included for completeness; the compiler may even replace
69 : a call to it with memcpy.
70 :
71 : fd_memcpy_{nn,nt}_nofence are the same without the trailing fence,
72 : for a caller issuing a run of non-temporal copies to disjoint
73 : destinations. The caller MUST execute _mm_sfence() itself after the
74 : last copy and before anything that publishes the destinations. One
75 : fence for the run is much cheaper than one per copy, since each
76 : fence drains the write combining buffers the copies just filled.
77 :
78 : WARNING: the writes to memory that this function issues may become
79 : visible to another core in a surprising order. This is a normal part
80 : of the memcpy contract, but is especially true when using
81 : non-temporal stores. The fencing variants include the appropriate
82 : fencing so that stores issued by the function will become visible
83 : before any stores following the call; with the _nofence variants
84 : that is the caller's job.
85 :
86 : WARNING: on many CPUs, a normal store following a non-temporal store
87 : to the same cache line causes a SEVERE performance degradation
88 : (approx 500-1000 cycles). See the crash course for the background on
89 : why. This function will not trigger these stalls, even when `_d` and
90 : `sz` are not multiples of 64, but it's up to the caller not to touch
91 : the memory pointed to by _d after this function returns. */
92 :
93 : #include "../fd_util_base.h"
94 : #include <immintrin.h>
95 :
96 : #define FD_EMIT_TEMPORAL_MEMCPY( suffix, copy64, needs_sfence ) \
97 : FD_FN_UNUSED static void * \
98 : fd_memcpy_##suffix( void * FD_RESTRICT _d, \
99 : void const * FD_RESTRICT _s, \
100 803992 : ulong sz ) { \
101 803992 : ulong rem = sz; \
102 803992 : uchar * FD_RESTRICT d = (uchar *)_d; \
103 803992 : uchar const * FD_RESTRICT s = (uchar const *)_s; \
104 803992 : ulong align = fd_ulong_min( rem, fd_ulong_align_up( (ulong)d, 64UL ) - (ulong)d ); \
105 803992 : if( FD_UNLIKELY( align ) ) { memcpy( d, s, align ); rem -= align; d += align; s += align; } \
106 1617049940 : for( ; rem>63UL; rem-=64UL, d += 64UL, s += 64UL ) copy64( d, s ); \
107 803992 : if( needs_sfence ) _mm_sfence(); \
108 803992 : if( FD_UNLIKELY( rem ) ) memcpy( d, s, rem ); \
109 803992 : return _d; \
110 803992 : }
111 :
112 :
113 : #if FD_HAS_AVX512
114 :
115 : #if defined(__CLDEMOTE__) && 0
116 : /* cldemote is only supported on Sapphire Rapids and newer. Some
117 : experiments on Emerald Rapids showed that it does reduce cache
118 : pollution, but it dramatically reduces the speed of the copy. For
119 : now, we'll just disable it globally. Maybe it will make sense on a
120 : future CPU. */
121 : #define FD_CLDEMOTE( s ) _cldemote( (void *)(s) )
122 : #else
123 269359938 : #define FD_CLDEMOTE( s ) do { } while( 0 )
124 : #endif
125 :
126 134679969 : # define copy64_nn( d, s ) do { _mm_prefetch( (s)+384UL, _MM_HINT_NTA ); _mm512_stream_si512( (void *)(d), _mm512_loadu_si512( (void const *)(s) ) ); FD_CLDEMOTE( s ); } while( 0 )
127 134766289 : # define copy64_nt( d, s ) do { _mm512_stream_si512( (void *)(d), _mm512_loadu_si512( (void const *)(s) ) ); } while( 0 )
128 134679969 : # define copy64_tn( d, s ) do { _mm_prefetch( (s)+384UL, _MM_HINT_NTA ); _mm512_storeu_si512( (void *)(d), _mm512_loadu_si512( (void const *)(s) ) ); FD_CLDEMOTE( s ); } while( 0 )
129 134679969 : # define copy64_tt( d, s ) do { _mm512_storeu_si512( (void *)(d), _mm512_loadu_si512( (void const *)(s) ) ); } while( 0 )
130 :
131 : #elif FD_HAS_AVX
132 :
133 269359938 : # define copy64_nn( d, s ) do { _mm_prefetch( (s)+384UL, _MM_HINT_NTA ); _mm256_stream_si256( (void *)(d ), _mm256_loadu_si256( (void const *)(s ) ) ); \
134 269359938 : _mm256_stream_si256( (void *)(d+32UL), _mm256_loadu_si256( (void const *)(s+32UL) ) ); } while( 0 )
135 269359938 : # define copy64_nt( d, s ) do { _mm256_stream_si256( (void *)(d ), _mm256_loadu_si256( (void const *)(s ) ) ); \
136 269359938 : _mm256_stream_si256( (void *)(d+32UL), _mm256_loadu_si256( (void const *)(s+32UL) ) ); } while( 0 )
137 269359938 : # define copy64_tn( d, s ) do { _mm_prefetch( (s)+384UL, _MM_HINT_NTA ); _mm256_storeu_si256( (void *)(d ), _mm256_loadu_si256( (void const *)(s ) ) ); \
138 269359938 : _mm256_storeu_si256( (void *)(d+32UL), _mm256_loadu_si256( (void const *)(s+32UL) ) ); } while( 0 )
139 269359938 : # define copy64_tt( d, s ) do { _mm256_storeu_si256( (void *)(d ), _mm256_loadu_si256( (void const *)(s ) ) ); \
140 269359938 : _mm256_storeu_si256( (void *)(d+32UL), _mm256_loadu_si256( (void const *)(s+32UL) ) ); } while( 0 )
141 :
142 : #elif FD_HAS_SSE
143 :
144 : # define copy64_nn( d, s ) do { _mm_prefetch( (s)+384UL, _MM_HINT_NTA ); _mm_stream_si128( (void *)(d ), _mm_loadu_si128( (void const *)(s ) ) ); \
145 : _mm_stream_si128( (void *)(d+16UL), _mm_loadu_si128( (void const *)(s+16UL) ) ); \
146 : _mm_stream_si128( (void *)(d+32UL), _mm_loadu_si128( (void const *)(s+32UL) ) ); \
147 : _mm_stream_si128( (void *)(d+48UL), _mm_loadu_si128( (void const *)(s+48UL) ) ); } while( 0 )
148 : # define copy64_nt( d, s ) do { _mm_stream_si128( (void *)(d ), _mm_loadu_si128( (void const *)(s ) ) ); \
149 : _mm_stream_si128( (void *)(d+16UL), _mm_loadu_si128( (void const *)(s+16UL) ) ); \
150 : _mm_stream_si128( (void *)(d+32UL), _mm_loadu_si128( (void const *)(s+32UL) ) ); \
151 : _mm_stream_si128( (void *)(d+48UL), _mm_loadu_si128( (void const *)(s+48UL) ) ); } while( 0 )
152 : # define copy64_tn( d, s ) do { _mm_prefetch( (s)+384UL, _MM_HINT_NTA ); _mm_storeu_si128( (void *)(d ), _mm_loadu_si128( (void const *)(s ) ) ); \
153 : _mm_storeu_si128( (void *)(d+16UL), _mm_loadu_si128( (void const *)(s+16UL) ) ); \
154 : _mm_storeu_si128( (void *)(d+32UL), _mm_loadu_si128( (void const *)(s+32UL) ) ); \
155 : _mm_storeu_si128( (void *)(d+48UL), _mm_loadu_si128( (void const *)(s+48UL) ) ); } while( 0 )
156 : # define copy64_tt( d, s ) do { _mm_storeu_si128( (void *)(d ), _mm_loadu_si128( (void const *)(s ) ) ); \
157 : _mm_storeu_si128( (void *)(d+16UL), _mm_loadu_si128( (void const *)(s+16UL) ) ); \
158 : _mm_storeu_si128( (void *)(d+32UL), _mm_loadu_si128( (void const *)(s+32UL) ) ); \
159 : _mm_storeu_si128( (void *)(d+48UL), _mm_loadu_si128( (void const *)(s+48UL) ) ); } while( 0 )
160 :
161 : #else
162 : # error "fd_nt_memcpy requires SSE, AVX or AVX512"
163 : #endif
164 :
165 :
166 404039907 : FD_EMIT_TEMPORAL_MEMCPY( nn, copy64_nn, 1 )
167 404039907 : FD_EMIT_TEMPORAL_MEMCPY( nt, copy64_nt, 1 )
168 404039907 : FD_EMIT_TEMPORAL_MEMCPY( tn, copy64_tn, 0 )
169 404039907 : FD_EMIT_TEMPORAL_MEMCPY( tt, copy64_tt, 0 )
170 : FD_EMIT_TEMPORAL_MEMCPY( nn_nofence, copy64_nn, 0 )
171 86320 : FD_EMIT_TEMPORAL_MEMCPY( nt_nofence, copy64_nt, 0 )
172 :
173 : #undef copy64_nn
174 : #undef copy64_nt
175 : #undef copy64_tn
176 : #undef copy64_tt
177 :
178 : #undef FD_EMIT_TEMPORAL_MEMCPY
179 :
180 : #else
181 : #error "Build target does not support non-temporal memcpy"
182 : #endif
183 :
184 : #endif /* HEADER_fd_src_util_simd_fd_nt_memcpy_h */
|
