Line data Source code
1 : #define _GNU_SOURCE /* pipe2 */
2 : #include "fd_waker.h"
3 :
4 : #include "../../util/log/fd_log.h"
5 :
6 : #include <errno.h>
7 : #include <fcntl.h>
8 : #include <sched.h>
9 : #include <stdlib.h>
10 : #include <unistd.h>
11 : #include <sys/epoll.h>
12 : #include <sys/syscall.h>
13 : #include <sys/wait.h>
14 :
15 : /* The wake protocol needs two nested-epoll behaviors: (a) an event
16 : in an inner set wakes a BLOCKED outer epoll_wait (broken on
17 : unpatched v5.5-v5.12), and (b) EPOLL_CTL_MOD re-polls, so a rearm
18 : with readiness pending re-fires (what makes clear/drain/rearm
19 : lost-wake-proof). (a) only shows with a sleeping waiter, so fork
20 : a child that waits until /proc/<parent>/syscall shows us blocked
21 : in epoll_pwait, then makes an inner fd ready, and require the
22 : wake. Runs once pre-fork; logs err on a broken kernel. */
23 :
24 : static void
25 3 : self_test( void ) {
26 3 : int outer = epoll_create1( 0 );
27 3 : int inner = epoll_create1( 0 );
28 3 : if( FD_UNLIKELY( -1==outer || -1==inner ) ) FD_LOG_ERR(( "epoll_create1() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
29 :
30 3 : int pfd[ 2 ];
31 3 : if( FD_UNLIKELY( pipe2( pfd, O_CLOEXEC ) ) ) FD_LOG_ERR(( "pipe2() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
32 :
33 3 : struct epoll_event ev = { .events = EPOLLIN };
34 3 : if( FD_UNLIKELY( -1==epoll_ctl( inner, EPOLL_CTL_ADD, pfd[ 0 ], &ev ) ) ) FD_LOG_ERR(( "epoll_ctl(ADD) failed (%i-%s)", errno, fd_io_strerror( errno ) ));
35 3 : ev = (struct epoll_event){ .events = EPOLLIN|EPOLLONESHOT };
36 3 : if( FD_UNLIKELY( -1==epoll_ctl( outer, EPOLL_CTL_ADD, inner, &ev ) ) ) FD_LOG_ERR(( "epoll_ctl(ADD) failed (%i-%s)", errno, fd_io_strerror( errno ) ));
37 :
38 3 : char path[ 32 ];
39 3 : FD_TEST( fd_cstr_printf_check( path, sizeof(path), NULL, "/proc/%d/syscall", (int)getpid() ) );
40 :
41 3 : pid_t pid = fork();
42 3 : if( FD_UNLIKELY( -1==pid ) ) FD_LOG_ERR(( "fork() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
43 3 : if( FD_UNLIKELY( !pid ) ) {
44 : /* child: spin until the parent is observably blocked in the outer
45 : epoll_pwait (first field of /proc/<pid>/syscall is the syscall
46 : number, or "running"), then make the inner fd ready. Bounded;
47 : on exhaustion write anyway and report the coverage as
48 : unconfirmed (exit 2). */
49 0 : int blocked = 0;
50 0 : for( ulong i=0UL; i<1000000UL; i++ ) {
51 0 : int fd = open( path, O_RDONLY );
52 0 : if( FD_UNLIKELY( -1==fd ) ) break;
53 0 : char buf[ 64 ];
54 0 : long rd = read( fd, buf, sizeof(buf)-1UL );
55 0 : close( fd );
56 0 : if( FD_UNLIKELY( rd<=0L ) ) break;
57 0 : buf[ rd ] = '\0';
58 0 : long nr = strtol( buf, NULL, 10 );
59 0 : int is_epoll_wait = nr==__NR_epoll_pwait;
60 0 : #ifdef __NR_epoll_wait
61 0 : is_epoll_wait |= nr==__NR_epoll_wait;
62 0 : #endif
63 0 : if( FD_LIKELY( is_epoll_wait ) ) { blocked = 1; break; }
64 0 : sched_yield();
65 0 : }
66 0 : long n = write( pfd[ 1 ], "x", 1UL );
67 0 : _exit( 1L!=n ? 1 : ( blocked ? 0 : 2 ) );
68 0 : }
69 :
70 3 : int n;
71 3 : do n = epoll_pwait( outer, &ev, 1, 2000, NULL );
72 3 : while( FD_UNLIKELY( -1==n && errno==EINTR ) );
73 3 : if( FD_UNLIKELY( -1==n ) ) FD_LOG_ERR(( "epoll_pwait() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
74 3 : int wstatus;
75 3 : if( FD_UNLIKELY( pid!=waitpid( pid, &wstatus, 0 ) ) ) FD_LOG_ERR(( "waitpid() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
76 3 : if( FD_UNLIKELY( !WIFEXITED( wstatus ) || WEXITSTATUS( wstatus )==1 ) ) FD_LOG_ERR(( "waker self test child failed" ));
77 3 : if( FD_UNLIKELY( WEXITSTATUS( wstatus )==2 ) )
78 0 : FD_LOG_WARNING(( "waker self test could not observe the parent blocked in epoll_pwait via %s; blocked-waiter coverage unconfirmed", path ));
79 3 : if( FD_UNLIKELY( 1!=n ) )
80 0 : FD_LOG_ERR(( "waker self test failed: an event inside a nested epoll set did not wake a blocked "
81 3 : "epoll_wait on the outer set. This kernel is missing the nested-epoll wakeup fixes "
82 3 : "(present in v5.13+ and in the 5.4.y/5.10.y stable series); the waker tile would "
83 3 : "lose wakes on it." ));
84 :
85 : /* (b) entry now disarmed, inner set still ready (byte unread):
86 : must stay silent until the MOD rearm re-fires it. */
87 3 : if( FD_UNLIKELY( 0!=epoll_wait( outer, &ev, 1, 0 ) ) ) FD_LOG_ERR(( "waker self test failed: EPOLLONESHOT entry not disarmed after delivery" ));
88 3 : ev = (struct epoll_event){ .events = EPOLLIN|EPOLLONESHOT };
89 3 : if( FD_UNLIKELY( -1==epoll_ctl( outer, EPOLL_CTL_MOD, inner, &ev ) ) ) FD_LOG_ERR(( "epoll_ctl(MOD) failed (%i-%s)", errno, fd_io_strerror( errno ) ));
90 3 : if( FD_UNLIKELY( 1!=epoll_wait( outer, &ev, 1, 0 ) ) )
91 0 : FD_LOG_ERR(( "waker self test failed: EPOLL_CTL_MOD rearm did not re-fire pending readiness; "
92 3 : "the waker rearm protocol would lose wakes on this kernel." ));
93 :
94 3 : if( FD_UNLIKELY( close( pfd[ 0 ] ) || close( pfd[ 1 ] ) || close( inner ) || close( outer ) ) )
95 0 : FD_LOG_ERR(( "close() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
96 3 : }
97 :
98 : void
99 3 : fd_waker_install( ulong client_cnt ) {
100 3 : FD_TEST( client_cnt<=FD_WAKER_CLIENT_MAX );
101 :
102 3 : if( FD_LIKELY( client_cnt ) ) self_test();
103 :
104 3 : int outer = epoll_create1( 0 );
105 3 : if( FD_UNLIKELY( -1==outer ) ) FD_LOG_ERR(( "epoll_create1() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
106 3 : if( FD_LIKELY( outer!=FD_WAKER_OUTER_FD ) ) {
107 3 : if( FD_UNLIKELY( -1==dup2( outer, FD_WAKER_OUTER_FD ) ) ) FD_LOG_ERR(( "dup2(%d,%d) failed (%i-%s)", outer, FD_WAKER_OUTER_FD, errno, fd_io_strerror( errno ) ));
108 3 : if( FD_UNLIKELY( close( outer ) ) ) FD_LOG_ERR(( "close() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
109 3 : }
110 :
111 9 : for( ulong i=0UL; i<client_cnt; i++ ) {
112 6 : int inner = epoll_create1( 0 );
113 6 : if( FD_UNLIKELY( -1==inner ) ) FD_LOG_ERR(( "epoll_create1() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
114 6 : if( FD_LIKELY( inner!=FD_WAKER_INNER_FD( i ) ) ) {
115 6 : if( FD_UNLIKELY( -1==dup2( inner, FD_WAKER_INNER_FD( i ) ) ) ) FD_LOG_ERR(( "dup2(%d,%d) failed (%i-%s)", inner, FD_WAKER_INNER_FD( i ), errno, fd_io_strerror( errno ) ));
116 6 : if( FD_UNLIKELY( close( inner ) ) ) FD_LOG_ERR(( "close() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
117 6 : }
118 :
119 6 : struct epoll_event ev = { .events = EPOLLIN|EPOLLONESHOT, .data.u64 = i };
120 6 : if( FD_UNLIKELY( -1==epoll_ctl( FD_WAKER_OUTER_FD, EPOLL_CTL_ADD, FD_WAKER_INNER_FD( i ), &ev ) ) )
121 0 : FD_LOG_ERR(( "epoll_ctl(ADD,inner[%lu]) failed (%i-%s)", i, errno, fd_io_strerror( errno ) ));
122 6 : }
123 3 : }
124 :
125 : void
126 9 : fd_waker_client_rearm( ulong idx ) {
127 9 : struct epoll_event ev = { .events = EPOLLIN|EPOLLONESHOT, .data.u64 = idx };
128 9 : if( FD_UNLIKELY( -1==epoll_ctl( FD_WAKER_OUTER_FD, EPOLL_CTL_MOD, FD_WAKER_INNER_FD( idx ), &ev ) ) )
129 0 : FD_LOG_ERR(( "epoll_ctl(MOD,inner[%lu]) failed (%i-%s)", idx, errno, fd_io_strerror( errno ) ));
130 9 : }
|
