Line data    Source code

       1             : 
       2             : // Source originally from https://github.com/BLAKE3-team/BLAKE3
       3             : // From commit: c0ea395cf91d242f078c23d5f8d87eb9dd5f7b78
       4             : 
       5             : #include "fd_blake3.h"
       6             : #include "fd_blake3_private.h"
       7             : #include "../../util/simd/fd_sse.h"
       8             : 
       9             : #define _mm_shuffle_ps2(a, b, c)                                       \
      10   608064912 :   (_mm_castps_si128(                                                   \
      11   608064912 :       _mm_shuffle_ps(_mm_castsi128_ps(a), _mm_castsi128_ps(b), (c))))
      12             : 
      13   532056798 : #define vu_rot16 vb_exch_adj_pair
      14             : 
      15             : static inline __attribute__((always_inline)) vu_t
      16   532056798 : vu_rot12( vu_t x ) {
      17   532056798 :   return vu_xor( vu_shr( x, 12 ), vu_shl( x, 32-12 ) );
      18   532056798 : }
      19             : 
      20             : static inline __attribute__((always_inline)) vu_t
      21   532056798 : vu_rot8( vu_t x ) {
      22   532056798 :   vb_t const mask = vb( 1,2,3,0, 5,6,7,4, 9,10,11,8, 13,14,15,12 );
      23   532056798 :   return _mm_shuffle_epi8( x, mask );
      24   532056798 : }
      25             : 
      26             : static inline __attribute__((always_inline)) vu_t
      27   532056798 : vu_rot7( vu_t x ) {
      28   532056798 :   return vu_xor( vu_shr( x, 7 ), vu_shl( x, 32-7 ) );
      29   532056798 : }
      30             : 
      31             : static inline __attribute__((always_inline)) void
      32             : g1( vu_t * row0,
      33             :     vu_t * row1,
      34             :     vu_t * row2,
      35             :     vu_t * row3,
      36   532056798 :     vu_t   m ) {
      37   532056798 :   *row0 = vu_add(vu_add(*row0, m), *row1);
      38   532056798 :   *row3 = vu_xor(*row3, *row0);
      39   532056798 :   *row3 = vu_rot16(*row3);
      40   532056798 :   *row2 = vu_add(*row2, *row3);
      41   532056798 :   *row1 = vu_xor(*row1, *row2);
      42   532056798 :   *row1 = vu_rot12(*row1);
      43   532056798 : }
      44             : 
      45             : static inline __attribute__((always_inline)) void
      46             : g2( vu_t * row0,
      47             :     vu_t * row1,
      48             :     vu_t * row2,
      49             :     vu_t * row3,
      50   532056798 :     vu_t   m ) {
      51   532056798 :   *row0 = vu_add(vu_add(*row0, m), *row1);
      52   532056798 :   *row3 = vu_xor(*row3, *row0);
      53   532056798 :   *row3 = vu_rot8(*row3);
      54   532056798 :   *row2 = vu_add(*row2, *row3);
      55   532056798 :   *row1 = vu_xor(*row1, *row2);
      56   532056798 :   *row1 = vu_rot7(*row1);
      57   532056798 : }
      58             : 
      59             : // Note the optimization here of leaving row1 as the unrotated row, rather than
      60             : // row0. All the message loads below are adjusted to compensate for this. See
      61             : // discussion at https://github.com/sneves/blake2-avx2/pull/4
      62             : static inline __attribute__((always_inline)) void
      63   266028399 : diagonalize(vu_t *row0, vu_t *row2, vu_t *row3) {
      64   266028399 :   *row0 = _mm_shuffle_epi32(*row0, _MM_SHUFFLE(2, 1, 0, 3));
      65   266028399 :   *row3 = _mm_shuffle_epi32(*row3, _MM_SHUFFLE(1, 0, 3, 2));
      66   266028399 :   *row2 = _mm_shuffle_epi32(*row2, _MM_SHUFFLE(0, 3, 2, 1));
      67   266028399 : }
      68             : 
      69             : static inline __attribute__((always_inline)) void
      70   266028399 : undiagonalize(vu_t *row0, vu_t *row2, vu_t *row3) {
      71   266028399 :   *row0 = _mm_shuffle_epi32(*row0, _MM_SHUFFLE(0, 3, 2, 1));
      72   266028399 :   *row3 = _mm_shuffle_epi32(*row3, _MM_SHUFFLE(1, 0, 3, 2));
      73   266028399 :   *row2 = _mm_shuffle_epi32(*row2, _MM_SHUFFLE(2, 1, 0, 3));
      74   266028399 : }
      75             : 
      76             : static inline __attribute__((always_inline)) void
      77             : compress_pre( vu_t        rows[4],
      78             :               uint  const cv[ static 8 ],
      79             :               uchar const block[ static FD_BLAKE3_BLOCK_SZ ],
      80             :               uint        block_len,
      81             :               ulong       ctr,
      82    38004057 :               uint        flags ) {
      83    38004057 :   rows[0] = vu_ld( cv   );
      84    38004057 :   rows[1] = vu_ld( cv+4 );
      85    38004057 :   rows[2] = vu( FD_BLAKE3_IV[0], FD_BLAKE3_IV[1], FD_BLAKE3_IV[2], FD_BLAKE3_IV[3] );
      86    38004057 :   rows[3] = vu( (uint)(ctr&UINT_MAX), (uint)(ctr>>32),
      87    38004057 :                 block_len,            flags );
      88             : 
      89    38004057 :   vu_t m0 = vb_ldu( block    ); vu_t m1 = vb_ldu( block+16 );
      90    38004057 :   vu_t m2 = vb_ldu( block+32 ); vu_t m3 = vb_ldu( block+48 );
      91             : 
      92    38004057 :   vu_t t0, t1, t2, t3, tt;
      93             : 
      94             :   // Round 1. The first round permutes the message words from the original
      95             :   // input order, into the groups that get mixed in parallel.
      96    38004057 :   t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(2, 0, 2, 0)); //  6  4  2  0
      97    38004057 :   g1(&rows[0], &rows[1], &rows[2], &rows[3], t0);
      98    38004057 :   t1 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 3, 1)); //  7  5  3  1
      99    38004057 :   g2(&rows[0], &rows[1], &rows[2], &rows[3], t1);
     100    38004057 :   diagonalize(&rows[0], &rows[2], &rows[3]);
     101    38004057 :   t2 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(2, 0, 2, 0)); // 14 12 10  8
     102    38004057 :   t2 = _mm_shuffle_epi32(t2, _MM_SHUFFLE(2, 1, 0, 3));   // 12 10  8 14
     103    38004057 :   g1(&rows[0], &rows[1], &rows[2], &rows[3], t2);
     104    38004057 :   t3 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 1, 3, 1)); // 15 13 11  9
     105    38004057 :   t3 = _mm_shuffle_epi32(t3, _MM_SHUFFLE(2, 1, 0, 3));   // 13 11  9 15
     106    38004057 :   g2(&rows[0], &rows[1], &rows[2], &rows[3], t3);
     107    38004057 :   undiagonalize(&rows[0], &rows[2], &rows[3]);
     108    38004057 :   m0 = t0;
     109    38004057 :   m1 = t1;
     110    38004057 :   m2 = t2;
     111    38004057 :   m3 = t3;
     112             : 
     113             :   // Round 2. This round and all following rounds apply a fixed permutation
     114             :   // to the message words from the round before.
     115    38004057 :   t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2));
     116    38004057 :   t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1));
     117    38004057 :   g1(&rows[0], &rows[1], &rows[2], &rows[3], t0);
     118    38004057 :   t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2));
     119    38004057 :   tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3));
     120    38004057 :   t1 = _mm_blend_epi16(tt, t1, 0xCC);
     121    38004057 :   g2(&rows[0], &rows[1], &rows[2], &rows[3], t1);
     122    38004057 :   diagonalize(&rows[0], &rows[2], &rows[3]);
     123    38004057 :   t2 = _mm_unpacklo_epi64(m3, m1);
     124    38004057 :   tt = _mm_blend_epi16(t2, m2, 0xC0);
     125    38004057 :   t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0));
     126    38004057 :   g1(&rows[0], &rows[1], &rows[2], &rows[3], t2);
     127    38004057 :   t3 = _mm_unpackhi_epi32(m1, m3);
     128    38004057 :   tt = _mm_unpacklo_epi32(m2, t3);
     129    38004057 :   t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2));
     130    38004057 :   g2(&rows[0], &rows[1], &rows[2], &rows[3], t3);
     131    38004057 :   undiagonalize(&rows[0], &rows[2], &rows[3]);
     132    38004057 :   m0 = t0;
     133    38004057 :   m1 = t1;
     134    38004057 :   m2 = t2;
     135    38004057 :   m3 = t3;
     136             : 
     137             :   // Round 3
     138    38004057 :   t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2));
     139    38004057 :   t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1));
     140    38004057 :   g1(&rows[0], &rows[1], &rows[2], &rows[3], t0);
     141    38004057 :   t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2));
     142    38004057 :   tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3));
     143    38004057 :   t1 = _mm_blend_epi16(tt, t1, 0xCC);
     144    38004057 :   g2(&rows[0], &rows[1], &rows[2], &rows[3], t1);
     145    38004057 :   diagonalize(&rows[0], &rows[2], &rows[3]);
     146    38004057 :   t2 = _mm_unpacklo_epi64(m3, m1);
     147    38004057 :   tt = _mm_blend_epi16(t2, m2, 0xC0);
     148    38004057 :   t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0));
     149    38004057 :   g1(&rows[0], &rows[1], &rows[2], &rows[3], t2);
     150    38004057 :   t3 = _mm_unpackhi_epi32(m1, m3);
     151    38004057 :   tt = _mm_unpacklo_epi32(m2, t3);
     152    38004057 :   t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2));
     153    38004057 :   g2(&rows[0], &rows[1], &rows[2], &rows[3], t3);
     154    38004057 :   undiagonalize(&rows[0], &rows[2], &rows[3]);
     155    38004057 :   m0 = t0;
     156    38004057 :   m1 = t1;
     157    38004057 :   m2 = t2;
     158    38004057 :   m3 = t3;
     159             : 
     160             :   // Round 4
     161    38004057 :   t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2));
     162    38004057 :   t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1));
     163    38004057 :   g1(&rows[0], &rows[1], &rows[2], &rows[3], t0);
     164    38004057 :   t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2));
     165    38004057 :   tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3));
     166    38004057 :   t1 = _mm_blend_epi16(tt, t1, 0xCC);
     167    38004057 :   g2(&rows[0], &rows[1], &rows[2], &rows[3], t1);
     168    38004057 :   diagonalize(&rows[0], &rows[2], &rows[3]);
     169    38004057 :   t2 = _mm_unpacklo_epi64(m3, m1);
     170    38004057 :   tt = _mm_blend_epi16(t2, m2, 0xC0);
     171    38004057 :   t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0));
     172    38004057 :   g1(&rows[0], &rows[1], &rows[2], &rows[3], t2);
     173    38004057 :   t3 = _mm_unpackhi_epi32(m1, m3);
     174    38004057 :   tt = _mm_unpacklo_epi32(m2, t3);
     175    38004057 :   t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2));
     176    38004057 :   g2(&rows[0], &rows[1], &rows[2], &rows[3], t3);
     177    38004057 :   undiagonalize(&rows[0], &rows[2], &rows[3]);
     178    38004057 :   m0 = t0;
     179    38004057 :   m1 = t1;
     180    38004057 :   m2 = t2;
     181    38004057 :   m3 = t3;
     182             : 
     183             :   // Round 5
     184    38004057 :   t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2));
     185    38004057 :   t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1));
     186    38004057 :   g1(&rows[0], &rows[1], &rows[2], &rows[3], t0);
     187    38004057 :   t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2));
     188    38004057 :   tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3));
     189    38004057 :   t1 = _mm_blend_epi16(tt, t1, 0xCC);
     190    38004057 :   g2(&rows[0], &rows[1], &rows[2], &rows[3], t1);
     191    38004057 :   diagonalize(&rows[0], &rows[2], &rows[3]);
     192    38004057 :   t2 = _mm_unpacklo_epi64(m3, m1);
     193    38004057 :   tt = _mm_blend_epi16(t2, m2, 0xC0);
     194    38004057 :   t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0));
     195    38004057 :   g1(&rows[0], &rows[1], &rows[2], &rows[3], t2);
     196    38004057 :   t3 = _mm_unpackhi_epi32(m1, m3);
     197    38004057 :   tt = _mm_unpacklo_epi32(m2, t3);
     198    38004057 :   t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2));
     199    38004057 :   g2(&rows[0], &rows[1], &rows[2], &rows[3], t3);
     200    38004057 :   undiagonalize(&rows[0], &rows[2], &rows[3]);
     201    38004057 :   m0 = t0;
     202    38004057 :   m1 = t1;
     203    38004057 :   m2 = t2;
     204    38004057 :   m3 = t3;
     205             : 
     206             :   // Round 6
     207    38004057 :   t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2));
     208    38004057 :   t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1));
     209    38004057 :   g1(&rows[0], &rows[1], &rows[2], &rows[3], t0);
     210    38004057 :   t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2));
     211    38004057 :   tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3));
     212    38004057 :   t1 = _mm_blend_epi16(tt, t1, 0xCC);
     213    38004057 :   g2(&rows[0], &rows[1], &rows[2], &rows[3], t1);
     214    38004057 :   diagonalize(&rows[0], &rows[2], &rows[3]);
     215    38004057 :   t2 = _mm_unpacklo_epi64(m3, m1);
     216    38004057 :   tt = _mm_blend_epi16(t2, m2, 0xC0);
     217    38004057 :   t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0));
     218    38004057 :   g1(&rows[0], &rows[1], &rows[2], &rows[3], t2);
     219    38004057 :   t3 = _mm_unpackhi_epi32(m1, m3);
     220    38004057 :   tt = _mm_unpacklo_epi32(m2, t3);
     221    38004057 :   t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2));
     222    38004057 :   g2(&rows[0], &rows[1], &rows[2], &rows[3], t3);
     223    38004057 :   undiagonalize(&rows[0], &rows[2], &rows[3]);
     224    38004057 :   m0 = t0;
     225    38004057 :   m1 = t1;
     226    38004057 :   m2 = t2;
     227    38004057 :   m3 = t3;
     228             : 
     229             :   // Round 7
     230    38004057 :   t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2));
     231    38004057 :   t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1));
     232    38004057 :   g1(&rows[0], &rows[1], &rows[2], &rows[3], t0);
     233    38004057 :   t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2));
     234    38004057 :   tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3));
     235    38004057 :   t1 = _mm_blend_epi16(tt, t1, 0xCC);
     236    38004057 :   g2(&rows[0], &rows[1], &rows[2], &rows[3], t1);
     237    38004057 :   diagonalize(&rows[0], &rows[2], &rows[3]);
     238    38004057 :   t2 = _mm_unpacklo_epi64(m3, m1);
     239    38004057 :   tt = _mm_blend_epi16(t2, m2, 0xC0);
     240    38004057 :   t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0));
     241    38004057 :   g1(&rows[0], &rows[1], &rows[2], &rows[3], t2);
     242    38004057 :   t3 = _mm_unpackhi_epi32(m1, m3);
     243    38004057 :   tt = _mm_unpacklo_epi32(m2, t3);
     244    38004057 :   t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2));
     245    38004057 :   g2(&rows[0], &rows[1], &rows[2], &rows[3], t3);
     246    38004057 :   undiagonalize(&rows[0], &rows[2], &rows[3]);
     247    38004057 : }
     248             : 
     249             : void
     250             : fd_blake3_sse_compress1( uchar * restrict       out,
     251             :                          uchar const * restrict msg,
     252             :                          uint                   msg_sz,
     253             :                          ulong                  counter,
     254             :                          uint const             flags,
     255             :                          uchar * restrict       out_chain,
     256    11187266 :                          uchar const * restrict in_chain ) {
     257    11187266 :   FD_BLAKE3_TRACE(( "fd_blake3_sse_compress1(out=%p,msg=%p,sz=%u,counter=%lu,flags=%02x)",
     258    11187266 :                     (void *)out, (void *)msg, msg_sz, counter, flags ));
     259    11187266 :   FD_DCHECK_CRIT( msg_sz<=FD_BLAKE3_CHUNK_SZ, "internal error" );
     260             : 
     261    11187266 :   uint cv[8] = { FD_BLAKE3_IV[0], FD_BLAKE3_IV[1], FD_BLAKE3_IV[2], FD_BLAKE3_IV[3],
     262    11187266 :                  FD_BLAKE3_IV[4], FD_BLAKE3_IV[5], FD_BLAKE3_IV[6], FD_BLAKE3_IV[7] };
     263    11187266 :   if( FD_UNLIKELY( in_chain ) ) {
     264           0 :     memcpy( cv, in_chain, FD_BLAKE3_OUTCHAIN_SZ );
     265           0 :   }
     266    11187266 :   vu_t rows[4];
     267             : 
     268    11187266 :   uint flag_mask = ~fd_uint_if( flags&FD_BLAKE3_FLAG_PARENT,
     269    11187266 :                                 FD_BLAKE3_FLAG_CHUNK_START|FD_BLAKE3_FLAG_CHUNK_END,
     270    11187266 :                                 0U );
     271             : 
     272    11187266 :   uint block_flags = flags | (flag_mask & FD_BLAKE3_FLAG_CHUNK_START);
     273    11187266 :   if( FD_UNLIKELY( in_chain && !(flags&FD_BLAKE3_FLAG_CHUNK_START) ) ) {
     274           0 :     block_flags &= ~FD_BLAKE3_FLAG_CHUNK_START;
     275           0 :   }
     276    48657752 :   do {
     277    48657752 :     uint block_sz = fd_uint_min( msg_sz, FD_BLAKE3_BLOCK_SZ );
     278    48657752 :     block_flags |= FD_BLAKE3_FLAG_CHUNK_END;
     279    48657752 :     block_flags &= (flag_mask & ~fd_uint_if( msg_sz<=FD_BLAKE3_BLOCK_SZ, 0, (FD_BLAKE3_FLAG_CHUNK_END|FD_BLAKE3_FLAG_ROOT) ) );
     280             : 
     281    48657752 :     uchar tail[ FD_BLAKE3_BLOCK_SZ ] __attribute__((aligned(16)));
     282    48657752 :     uchar const * restrict block;
     283    48657752 :     if( FD_LIKELY( msg_sz>=FD_BLAKE3_BLOCK_SZ ) ) {
     284    44674166 :       block = msg;
     285    44674166 :     } else {
     286     3983586 :       vb_st( tail,    vu_zero() );
     287     3983586 :       vb_st( tail+16, vu_zero() );
     288     3983586 :       vb_st( tail+32, vu_zero() );
     289     3983586 :       vb_st( tail+48, vu_zero() );
     290     3983586 :       fd_memcpy( tail, msg, msg_sz );
     291     3983586 :       block = tail;
     292     3983586 :     }
     293             : 
     294    48657752 :     if( FD_UNLIKELY( out_chain && (block_flags & FD_BLAKE3_FLAG_CHUNK_END) ) ) {
     295             :       /* FIXME better document and polish the transition from the compress
     296             :                part to the expand part. */
     297    10653695 :       fd_memcpy( out,       block, FD_BLAKE3_BLOCK_SZ    ); /* FIXME DOCUMENT OVERLOADING OF OUT ARGUMENT */
     298    10653695 :       fd_memcpy( out_chain, cv,    FD_BLAKE3_OUTCHAIN_SZ );
     299    10653695 :       FD_BLAKE3_TRACE(( "fd_blake3_sse_compress1: done (XOF mode)" ));
     300    10653695 :       return;
     301    10653695 :     }
     302             : 
     303    38004057 :     FD_BLAKE3_TRACE(( "fd_blake3_sse_compress1: sz=%u counter=%u flags=%x", block_sz, (uint)counter, block_flags ));
     304    38004057 :     compress_pre( rows, cv, block, block_sz, counter, block_flags );
     305    38004057 :     if( FD_UNLIKELY( in_chain ) ) {
     306             :       /* FIXME UGLY */
     307           0 :       vu_stu( out+32, vu_xor( vu_ld( cv   ), rows[2] ) );
     308           0 :       vu_stu( out+48, vu_xor( vu_ld( cv+4 ), rows[3] ) );
     309           0 :     }
     310    38004057 :     vu_st( cv,   vu_xor( rows[0], rows[2] ) );
     311    38004057 :     vu_st( cv+4, vu_xor( rows[1], rows[3] ) );
     312    38004057 :     msg    += FD_BLAKE3_BLOCK_SZ;
     313    38004057 :     msg_sz -= block_sz;
     314    38004057 :     block_flags = flags;
     315    38004057 :   } while( (int)msg_sz>0 );
     316             : 
     317      533571 :   vu_stu( out,    vu_ld( cv   ) );
     318      533571 :   vu_stu( out+16, vu_ld( cv+4 ) );
     319             : 
     320      533571 :   FD_BLAKE3_TRACE(( "fd_blake3_sse_compress1: done" ));
     321      533571 : }