Line data Source code
1 : #ifndef HEADER_fd_src_ballet_blake3_fd_blake3_private_h
2 : #define HEADER_fd_src_ballet_blake3_fd_blake3_private_h
3 :
4 : #include "fd_blake3.h"
5 : #if FD_HAS_AVX
6 : #include "../../util/simd/fd_avx.h"
7 : #endif
8 :
9 : /* Set FD_BLAKE3_TRACING to 1 to dump out a high-level trace of BLAKE3
10 : operations to the debug log. This is useful during debugging or
11 : development. */
12 : #define FD_BLAKE3_TRACING 0
13 :
14 : #if FD_BLAKE3_TRACING
15 : #define FD_BLAKE3_TRACE( ... ) FD_LOG_DEBUG( __VA_ARGS__ )
16 : #else
17 172454327 : #define FD_BLAKE3_TRACE( ... ) (void)0
18 : #endif
19 :
20 : /* Protocol constants *************************************************/
21 :
22 : static const uchar FD_BLAKE3_MSG_SCHEDULE[7][16] = {
23 : { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 },
24 : { 2, 6, 3, 10, 7, 0, 4, 13, 1, 11, 12, 5, 9, 14, 15, 8 },
25 : { 3, 4, 10, 12, 13, 2, 7, 14, 6, 5, 9, 0, 11, 15, 8, 1 },
26 : { 10, 7, 12, 9, 14, 3, 13, 15, 4, 0, 11, 2, 5, 8, 1, 6 },
27 : { 12, 13, 9, 11, 15, 10, 14, 8, 7, 2, 5, 3, 0, 1, 6, 4 },
28 : { 9, 14, 11, 5, 8, 12, 15, 1, 13, 3, 0, 10, 2, 6, 4, 7 },
29 : { 11, 15, 5, 0, 1, 9, 8, 6, 14, 10, 2, 12, 3, 4, 7, 13 },
30 : };
31 :
32 : static const uint FD_BLAKE3_IV[8] = {
33 : 0x6A09E667UL, 0xBB67AE85UL, 0x3C6EF372UL, 0xA54FF53AUL,
34 : 0x510E527FUL, 0x9B05688CUL, 0x1F83D9ABUL, 0x5BE0CD19UL
35 : };
36 :
37 31080024 : #define FD_BLAKE3_FLAG_CHUNK_START (1u<<0) /* 1 */
38 127846241 : #define FD_BLAKE3_FLAG_CHUNK_END (1u<<1) /* 2 */
39 51089896 : #define FD_BLAKE3_FLAG_PARENT (1u<<2) /* 4 */
40 111080061 : #define FD_BLAKE3_FLAG_ROOT (1u<<3) /* 8 */
41 :
42 : /* Possible flag combinations:
43 : 0x1: first block of a chunk with at least 2 blocks
44 : 0x2: last block of a chunk, tree that has at least 1 parent
45 : 0x3: last chunk (<=64 bytes), input >1024 bytes
46 : 0x4: non-root parent node
47 : 0xa: last block of the only chunk, input_sz>64 input_sz<=1024
48 : 0xb: only block, input_sz<=64
49 : 0xc: root parent node */
50 :
51 : /* Scheduler **********************************************************/
52 :
53 : union __attribute__((aligned(32))) fd_blake3_op {
54 :
55 : struct {
56 : uchar const * msg;
57 : uchar * out;
58 :
59 : ulong counter;
60 : union {
61 : struct {
62 : ushort off;
63 : ushort sz;
64 : };
65 : uint off_sz;
66 : };
67 : uchar flags;
68 : };
69 :
70 : };
71 :
72 : typedef union fd_blake3_op fd_blake3_op_t;
73 :
74 : /* Compression function ***********************************************/
75 :
76 : FD_PROTOTYPES_BEGIN
77 :
78 : void
79 : fd_blake3_fini_xof_compress( fd_blake3_t * sha,
80 : uchar * root_msg,
81 : uchar * root_cv_pre );
82 :
83 : void
84 : fd_blake3_ref_compress1( uchar * restrict out, /* align==1 len==32 */
85 : uchar const * restrict msg, /* align==1 len==64 */
86 : uint msg_sz,
87 : ulong counter,
88 : uint flags,
89 : uchar * restrict out_chain, /* optional, 16 byte output chaining value of last block */
90 : uchar const * restrict in_chain ); /* optional, 16 byte input chaining value of first block (default IV) */
91 :
92 : #if FD_HAS_SSE
93 :
94 : void
95 : fd_blake3_sse_compress1( uchar * restrict out, /* align==1 len==32 */
96 : uchar const * restrict msg, /* align==1 len==64 */
97 : uint msg_sz,
98 : ulong counter,
99 : uint flags,
100 : uchar * restrict out_chain,
101 : uchar const * restrict in_chain );
102 :
103 : #endif /* FD_HAS_SSE */
104 :
105 : #if FD_HAS_AVX
106 :
107 : /* BLAKE3 AVX cores
108 :
109 : compress8 compresses one to eight tree nodes. batch_cnt is the
110 : number of nodes to process. For each node in the batch with index i,
111 : - _batch_data[i] points to the input data of the node (message bytes
112 : for leaf nodes, a pair of output chaining values for branch nodes)
113 : - batch_sz[i] is the input byte count of the node, from which the
114 : 'len' value of each of the node's blocks is derived
115 : - ctr_vec[i] is the 'counter' value of the node
116 : - batch_flags[i] is the 'flag' value of the node
117 : - cv is optional. If set, cv[i] is the 'chaining value' of the first
118 : block of the node. This is useful for XOF.
119 :
120 : compress8 has three different output modes:
121 : - "LtHash in-place": If lthash is set, each node's output is expanded
122 : (XOF) to 2048 bytes and interpreted as an 'LtHash' value (i.e.
123 : a vector of 1024 uint16). These vectors are then added together
124 : and the result is written to lthash. The root flag MUST be set for
125 : all batch_flags inputs, otherwise this function will read OOB.
126 : - "Simple": Otherwise, _batch_hash[i] is populated with the 32-byte
127 : output chaining value. (If node i is a root node, this is 'the
128 : BLAKE3 hash', i.e. the first 32 bytes of the XOF stream).
129 :
130 : These modes are all packed into the same function because the
131 : alternatives are worse (either worse code footprint due to duplicated
132 : core, or worse throughput due to high penalty passing vector regs
133 : between functions in SysV ABI).
134 :
135 : compress8_fast does a subset of what compress8 can, but is ~10-20%
136 : faster. */
137 :
138 : void
139 : fd_blake3_avx_compress8( ulong batch_cnt,
140 : void const * restrict _batch_data, /* align==32 len in [1,8) */
141 : uint const * restrict batch_sz, /* len in [1,8] */
142 : ulong const * restrict ctr_vec, /* len==8 */
143 : uint const * restrict batch_flags, /* align==32 len==8 */
144 : void * const * restrict _batch_hash, /* align==32 len in [1,8) */
145 : ushort * restrict lthash, /* align==32 byte_sz=2048 */
146 : uint out_sz, /* 32 or 64 */
147 : void const * restrict batch_cv ); /* align==8 len==8 ele_align==32 optional */
148 :
149 : void
150 : fd_blake3_avx_compress8_fast( uchar const * restrict batch_data, /* align==32 len==8*64 */
151 : uchar * restrict batch_hash, /* align==32 len==8*32 */
152 : ulong counter,
153 : uchar flags );
154 :
155 : #endif /* FD_HAS_AVX */
156 :
157 : #if FD_HAS_AVX512
158 :
159 : /* fd_blake3_avx512_compress16{,fast} are analogous to the avx APIs
160 : above. The only difference is larger alignment assumptions and that
161 : these process up to sixteen elements. */
162 :
163 : void
164 : fd_blake3_avx512_compress16( ulong batch_cnt,
165 : void const * restrict _batch_data, /* align=64 len=16 ele_align=1 */
166 : uint const * restrict batch_sz, /* align= 4 len=16 */
167 : ulong const * restrict ctr_vec, /* align= 8 len=16 */
168 : uint const * restrict batch_flags, /* align= 4 len=16 */
169 : void * const * restrict _batch_hash, /* align=64 len=16 */
170 : ushort * restrict lthash, /* align=32 byte_sz=2048 */
171 : uint out_sz, /* 32 or 64 */
172 : void const * restrict batch_cv ); /* align= 8 len=16 ele_align=16 optional */
173 :
174 : void
175 : fd_blake3_avx512_compress16_fast( uchar const * restrict batch_data, /* align==32 len==16*64 */
176 : uchar * restrict batch_hash, /* align==32 len==16*32 */
177 : ulong counter,
178 : uchar flags );
179 :
180 : #endif /* FD_HAS_AVX512 */
181 :
182 : FD_PROTOTYPES_END
183 :
184 : #endif /* HEADER_fd_src_ballet_blake3_fd_blake3_private_h */
|
