Line data Source code
1 : #include "fd_ssarchive.h"
2 :
3 : #include "../../../util/log/fd_log.h"
4 :
5 : #include <errno.h>
6 : #include <dirent.h>
7 : #include <stdlib.h>
8 : #include <unistd.h>
9 :
10 : struct fd_ssarchive_entry {
11 : ulong slot;
12 : ulong base_slot;
13 : int is_zstd;
14 : char path[ PATH_MAX ];
15 : uchar hash[ FD_HASH_FOOTPRINT ];
16 : };
17 : typedef struct fd_ssarchive_entry fd_ssarchive_entry_t;
18 :
19 : #define SORT_NAME sort_ssarchive_entries
20 24 : #define SORT_KEY_T fd_ssarchive_entry_t
21 12 : #define SORT_BEFORE(a,b) ( (a).slot>(b).slot )
22 : #include "../../../util/tmpl/fd_sort.c"
23 :
24 : #define FD_SSARCHIVE_MAX_ENTRIES (512UL)
25 :
26 : int
27 : fd_ssarchive_parse_filename( char const * _name,
28 : ulong * full_slot,
29 : ulong * incremental_slot,
30 : uchar hash[ static FD_HASH_FOOTPRINT ],
31 33 : int * is_zstd ) {
32 33 : char name[ PATH_MAX ];
33 33 : fd_cstr_ncpy( name, _name, sizeof(name) );
34 :
35 33 : char * ptr = name;
36 33 : int is_incremental;
37 33 : if( !strncmp( ptr, "incremental-snapshot-", 21UL ) ) {
38 15 : is_incremental = 1;
39 15 : ptr += 21UL;
40 18 : } else if( !strncmp( ptr, "snapshot-", 9UL ) ) {
41 18 : is_incremental = 0;
42 18 : ptr += 9UL;
43 18 : } else {
44 0 : return -1;
45 0 : }
46 :
47 33 : char * next = strchr( ptr, '-' );
48 33 : if( FD_UNLIKELY( !next ) ) return -1;
49 33 : *next = '\0';
50 33 : char * endptr;
51 33 : *full_slot = strtoul( ptr, &endptr, 10 );
52 33 : if( FD_UNLIKELY( *endptr!='\0' || endptr==ptr || *full_slot==ULONG_MAX ) ) return -1;
53 :
54 33 : if( is_incremental ) {
55 15 : ptr = next + 1;
56 15 : next = strchr( ptr, '-' );
57 15 : if( FD_UNLIKELY( !next ) ) return -1;
58 15 : *next = '\0';
59 15 : *incremental_slot = strtoul( ptr, &endptr, 10 );
60 15 : if( FD_UNLIKELY( *endptr!='\0' || endptr==ptr || *incremental_slot==ULONG_MAX ) ) return -1;
61 18 : } else {
62 18 : *incremental_slot = ULONG_MAX;
63 18 : }
64 :
65 33 : ptr = next + 1;
66 33 : next = strchr( ptr, '.' );
67 33 : if( FD_UNLIKELY( !next ) ) return -1;
68 33 : *next = '\0';
69 33 : ulong sz = (ulong)(next - ptr);
70 33 : if( FD_UNLIKELY( sz>FD_BASE58_ENCODED_32_LEN ) ) return -1;
71 33 : uchar * result = fd_base58_decode_32( ptr, hash );
72 33 : if( FD_UNLIKELY( result!=hash ) ) return -1;
73 :
74 33 : ptr = next + 1;
75 :
76 33 : if( FD_LIKELY( 0==strcmp( ptr, "tar.zst" ) ) ) *is_zstd = 1;
77 0 : else if ( FD_LIKELY( 0==strcmp( ptr, "tar" ) ) ) *is_zstd = 0;
78 0 : else return -1;
79 :
80 33 : return 0;
81 33 : }
82 :
83 : int
84 : fd_ssarchive_latest_pair( char const * directory,
85 : int incremental_snapshot,
86 : ulong * full_slot,
87 : ulong * incremental_slot,
88 : char full_path[ static PATH_MAX ],
89 : char incremental_path[ static PATH_MAX ],
90 : int * full_is_zstd,
91 : int * incremental_is_zstd,
92 : uchar full_hash[ static FD_HASH_FOOTPRINT ],
93 9 : uchar incremental_hash[ static FD_HASH_FOOTPRINT ] ) {
94 9 : *full_slot = ULONG_MAX;
95 9 : *incremental_slot = ULONG_MAX;
96 :
97 9 : DIR * dir = opendir( directory );
98 9 : if( FD_UNLIKELY( !dir ) ) {
99 0 : if( FD_LIKELY( errno==ENOENT ) ) return -1;
100 0 : FD_LOG_ERR(( "opendir() failed `%s` (%i-%s)", directory, errno, fd_io_strerror( errno ) ));
101 0 : }
102 :
103 9 : fd_ssarchive_entry_t full_snapshots[ FD_SSARCHIVE_MAX_ENTRIES ];
104 9 : fd_ssarchive_entry_t incremental_snapshots[ FD_SSARCHIVE_MAX_ENTRIES ];
105 9 : ulong full_snapshots_cnt = 0UL;
106 9 : ulong incremental_snapshots_cnt = 0UL;
107 :
108 9 : struct dirent * entry;
109 60 : for(;;) {
110 60 : errno = 0;
111 60 : entry = readdir( dir );
112 60 : if( FD_UNLIKELY( !entry ) ) break;
113 51 : if( FD_LIKELY( !strcmp( entry->d_name, "." ) || !strcmp( entry->d_name, ".." ) ) ) continue;
114 :
115 33 : int is_zstd;
116 33 : ulong entry_full_slot, entry_incremental_slot;
117 33 : uchar decoded_hash[ FD_HASH_FOOTPRINT ];
118 33 : if( FD_UNLIKELY( -1==fd_ssarchive_parse_filename( entry->d_name, &entry_full_slot, &entry_incremental_slot, decoded_hash, &is_zstd ) ) ) {
119 0 : FD_LOG_INFO(( "unrecognized snapshot file `%s/%s` in snapshots directory", directory, entry->d_name ));
120 0 : continue;
121 0 : }
122 :
123 33 : if( FD_LIKELY( entry_incremental_slot==ULONG_MAX ) ) {
124 18 : if( FD_UNLIKELY( full_snapshots_cnt>=FD_SSARCHIVE_MAX_ENTRIES ) ) {
125 0 : continue;
126 0 : }
127 :
128 18 : full_snapshots[ full_snapshots_cnt ].slot = entry_full_slot;
129 18 : full_snapshots[ full_snapshots_cnt ].base_slot = ULONG_MAX;
130 18 : full_snapshots[ full_snapshots_cnt ].is_zstd = is_zstd;
131 18 : if( FD_UNLIKELY( !fd_cstr_printf_check( full_snapshots[ full_snapshots_cnt ].path, PATH_MAX, NULL, "%s/%s", directory, entry->d_name ) ) ) {
132 0 : FD_LOG_ERR(( "snapshot path too long `%s/%s`", directory, entry->d_name ));
133 0 : }
134 18 : fd_memcpy( full_snapshots[ full_snapshots_cnt ].hash, decoded_hash, FD_HASH_FOOTPRINT );
135 18 : full_snapshots_cnt++;
136 18 : } else {
137 15 : if( FD_UNLIKELY( incremental_snapshots_cnt>=FD_SSARCHIVE_MAX_ENTRIES ) ) {
138 0 : continue;
139 0 : }
140 :
141 15 : incremental_snapshots[ incremental_snapshots_cnt ].slot = entry_incremental_slot;
142 15 : incremental_snapshots[ incremental_snapshots_cnt ].base_slot = entry_full_slot;
143 15 : incremental_snapshots[ incremental_snapshots_cnt ].is_zstd = is_zstd;
144 15 : if( FD_UNLIKELY( !fd_cstr_printf_check( incremental_snapshots[ incremental_snapshots_cnt ].path, PATH_MAX, NULL, "%s/%s", directory, entry->d_name ) ) ) {
145 0 : FD_LOG_ERR(( "snapshot path too long `%s/%s`", directory, entry->d_name ));
146 0 : }
147 15 : fd_memcpy( incremental_snapshots[ incremental_snapshots_cnt ].hash, decoded_hash, FD_HASH_FOOTPRINT );
148 15 : incremental_snapshots_cnt++;
149 15 : }
150 33 : }
151 :
152 9 : if( FD_UNLIKELY( errno ) ) FD_LOG_ERR(( "readdir() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
153 9 : if( FD_UNLIKELY( -1==closedir( dir ) ) ) FD_LOG_ERR(( "closedir() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
154 :
155 9 : if( FD_LIKELY( incremental_snapshot ) ) {
156 6 : if( FD_UNLIKELY( incremental_snapshots_cnt==0UL && full_snapshots_cnt==0UL ) ) return -1;
157 6 : if( FD_UNLIKELY( full_snapshots_cnt==0UL ) ) return -1;
158 :
159 6 : sort_ssarchive_entries_inplace( incremental_snapshots, incremental_snapshots_cnt );
160 6 : sort_ssarchive_entries_inplace( full_snapshots, full_snapshots_cnt );
161 :
162 6 : if( FD_UNLIKELY( incremental_snapshots_cnt==0UL ) ) {
163 0 : FD_LOG_INFO(("no incremental snapshots found in `%s`, falling back to latest full snapshot", directory ));
164 0 : *full_slot = full_snapshots[ 0UL ].slot;
165 0 : *full_is_zstd = full_snapshots[ 0UL ].is_zstd;
166 0 : *incremental_slot = ULONG_MAX;
167 0 : *incremental_is_zstd = 0;
168 0 : FD_TEST( fd_cstr_printf_check( full_path, PATH_MAX, NULL, "%s", full_snapshots[ 0UL ].path ) );
169 0 : fd_memcpy( full_hash, full_snapshots[ 0UL ].hash, FD_HASH_FOOTPRINT );
170 0 : memset( incremental_hash, 0, FD_HASH_FOOTPRINT );
171 0 : return 0;
172 0 : }
173 :
174 9 : for( ulong i=0UL; i<incremental_snapshots_cnt; i++ ) {
175 6 : ulong base_slot = incremental_snapshots[ i ].base_slot;
176 12 : for( ulong j=0; j<full_snapshots_cnt; j++ ) {
177 12 : if( FD_LIKELY( full_snapshots[ j ].slot==base_slot ) ) {
178 3 : *full_slot = base_slot;
179 3 : *incremental_slot = incremental_snapshots[ i ].slot;
180 3 : *full_is_zstd = full_snapshots[ j ].is_zstd;
181 3 : *incremental_is_zstd = incremental_snapshots[ i ].is_zstd;
182 3 : FD_TEST( fd_cstr_printf_check( full_path, PATH_MAX, NULL, "%s", full_snapshots[ j ].path ) );
183 3 : FD_TEST( fd_cstr_printf_check( incremental_path, PATH_MAX, NULL, "%s", incremental_snapshots[ i ].path ) );
184 3 : fd_memcpy( full_hash, full_snapshots[ j ].hash, FD_HASH_FOOTPRINT );
185 3 : fd_memcpy( incremental_hash, incremental_snapshots[ i ].hash, FD_HASH_FOOTPRINT );
186 3 : return 0;
187 9 : } else if( FD_LIKELY( full_snapshots[ j ].slot<base_slot ) ) {
188 : /* full snapshots are sorted in descending order, so if we reach a
189 : full snapshot with slot smaller than the incremental snapshot's
190 : base slot, we can stop searching. */
191 3 : break;
192 3 : }
193 12 : }
194 6 : }
195 :
196 : /* if we reach here, it means all incrementals are dangling (they
197 : don't build off any full snapshot). fallback to a full
198 : snapshot in that case. */
199 3 : *full_slot = full_snapshots[ 0UL ].slot;
200 3 : *full_is_zstd = full_snapshots[ 0UL ].is_zstd;
201 3 : *incremental_slot = ULONG_MAX;
202 3 : *incremental_is_zstd = 0;
203 3 : FD_TEST( fd_cstr_printf_check( full_path, PATH_MAX, NULL, "%s", full_snapshots[ 0UL ].path ) );
204 3 : incremental_path[ 0UL ] = '\0';
205 3 : fd_memcpy( full_hash, full_snapshots[ 0UL ].hash, FD_HASH_FOOTPRINT );
206 3 : memset( incremental_hash, 0, FD_HASH_FOOTPRINT );
207 3 : return 0;
208 :
209 3 : } else {
210 3 : if( FD_UNLIKELY( full_snapshots_cnt==0UL ) ) return -1;
211 :
212 3 : sort_ssarchive_entries_inplace( full_snapshots, full_snapshots_cnt );
213 :
214 3 : *full_slot = full_snapshots[ 0UL ].slot;
215 3 : *full_is_zstd = full_snapshots[ 0UL ].is_zstd;
216 3 : *incremental_slot = ULONG_MAX;
217 3 : *incremental_is_zstd = 0;
218 3 : FD_TEST( fd_cstr_printf_check( full_path, PATH_MAX, NULL, "%s", full_snapshots[ 0UL ].path ) );
219 3 : incremental_path[ 0UL ] = '\0';
220 3 : fd_memcpy( full_hash, full_snapshots[ 0UL ].hash, FD_HASH_FOOTPRINT );
221 3 : memset( incremental_hash, 0, FD_HASH_FOOTPRINT );
222 3 : return 0;
223 3 : }
224 9 : }
225 :
226 : void
227 : fd_ssarchive_remove_old_snapshots( char const * directory,
228 : uint max_full_snapshots_to_keep,
229 0 : uint max_incremental_snapshots_to_keep ) {
230 0 : ulong full_snapshots_cnt = 0UL;
231 0 : ulong incremental_snapshots_cnt = 0UL;
232 0 : fd_ssarchive_entry_t full_snapshots[ FD_SSARCHIVE_MAX_ENTRIES ];
233 0 : fd_ssarchive_entry_t incremental_snapshots[ FD_SSARCHIVE_MAX_ENTRIES ];
234 :
235 0 : DIR * dir = opendir( directory );
236 0 : if( FD_UNLIKELY( !dir ) ) {
237 0 : if( FD_LIKELY( errno==ENOENT ) ) return;
238 0 : FD_LOG_ERR(( "opendir() failed `%s` (%i-%s)", directory, errno, fd_io_strerror( errno ) ));
239 0 : }
240 :
241 0 : struct dirent * entry;
242 :
243 0 : for(;;) {
244 0 : errno = 0;
245 0 : entry = readdir( dir );
246 0 : if( FD_UNLIKELY( !entry ) ) break;
247 0 : if( FD_LIKELY( !strcmp( entry->d_name, "." ) || !strcmp( entry->d_name, ".." ) ) ) continue;
248 :
249 0 : int is_zstd;
250 0 : ulong entry_full_slot, entry_incremental_slot;
251 0 : uchar decoded_hash[ FD_HASH_FOOTPRINT ];
252 0 : if( FD_UNLIKELY( -1==fd_ssarchive_parse_filename( entry->d_name, &entry_full_slot, &entry_incremental_slot, decoded_hash, &is_zstd ) ) ) {
253 0 : FD_LOG_INFO(( "unrecognized snapshot file `%s/%s` in snapshots directory", directory, entry->d_name ));
254 0 : continue;
255 0 : }
256 :
257 0 : if( FD_LIKELY( entry_incremental_slot==ULONG_MAX ) ) {
258 0 : FD_TEST( entry_full_slot!=ULONG_MAX );
259 :
260 0 : if( FD_UNLIKELY( full_snapshots_cnt>=FD_SSARCHIVE_MAX_ENTRIES ) ) {
261 0 : continue;
262 0 : }
263 :
264 0 : full_snapshots[ full_snapshots_cnt ].slot = entry_full_slot;
265 0 : full_snapshots[ full_snapshots_cnt ].base_slot = ULONG_MAX;
266 0 : full_snapshots[ full_snapshots_cnt ].is_zstd = is_zstd;
267 0 : FD_TEST( fd_cstr_printf_check( full_snapshots[ full_snapshots_cnt ].path, PATH_MAX, NULL, "%s/%s", directory, entry->d_name ) );
268 0 : full_snapshots_cnt++;
269 0 : } else {
270 :
271 0 : if( FD_UNLIKELY( incremental_snapshots_cnt>=FD_SSARCHIVE_MAX_ENTRIES ) ) {
272 0 : continue;
273 0 : }
274 :
275 0 : incremental_snapshots[ incremental_snapshots_cnt ].slot = entry_incremental_slot;
276 0 : incremental_snapshots[ incremental_snapshots_cnt ].base_slot = entry_full_slot;
277 0 : incremental_snapshots[ incremental_snapshots_cnt ].is_zstd = is_zstd;
278 0 : FD_TEST( fd_cstr_printf_check( incremental_snapshots[ incremental_snapshots_cnt ].path, PATH_MAX, NULL, "%s/%s", directory, entry->d_name ) );
279 0 : incremental_snapshots_cnt++;
280 0 : }
281 0 : }
282 :
283 0 : if( FD_UNLIKELY( errno ) ) FD_LOG_ERR(( "readdir() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
284 0 : if( FD_UNLIKELY( -1==closedir( dir ) ) ) FD_LOG_ERR(( "closedir() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
285 :
286 0 : if( FD_LIKELY( full_snapshots_cnt>max_full_snapshots_to_keep ) ) {
287 0 : sort_ssarchive_entries_inplace( full_snapshots, full_snapshots_cnt );
288 0 : for( ulong i=max_full_snapshots_to_keep; i<full_snapshots_cnt; i++ ) {
289 0 : if( FD_UNLIKELY( -1==unlink( full_snapshots[ i ].path ) ) ) {
290 0 : FD_LOG_ERR(( "unlink(%s) failed (%i-%s)", full_snapshots[ i ].path, errno, fd_io_strerror( errno ) ));
291 0 : }
292 0 : }
293 0 : }
294 :
295 0 : if( FD_LIKELY( incremental_snapshots_cnt>max_incremental_snapshots_to_keep ) ) {
296 0 : sort_ssarchive_entries_inplace( incremental_snapshots, incremental_snapshots_cnt );
297 0 : for( ulong i=max_incremental_snapshots_to_keep; i<incremental_snapshots_cnt; i++ ) {
298 0 : if( FD_UNLIKELY( -1==unlink( incremental_snapshots[ i ].path ) ) ) {
299 0 : FD_LOG_ERR(( "unlink(%s) failed (%i-%s)", incremental_snapshots[ i ].path, errno, fd_io_strerror( errno ) ));
300 0 : }
301 0 : }
302 0 : }
303 0 : }
|
