Line data    Source code

       1             : 
       2             : // Source originally from https://github.com/BLAKE3-team/BLAKE3
       3             : // From commit: c0ea395cf91d242f078c23d5f8d87eb9dd5f7b78
       4             : 
       5             : #include "fd_blake3.h"
       6             : #include "fd_blake3_private.h"
       7             : #include "../../util/simd/fd_sse.h"
       8             : 
       9             : #define _mm_shuffle_ps2(a, b, c)                                       \
      10   610069104 :   (_mm_castps_si128(                                                   \
      11   610069104 :       _mm_shuffle_ps(_mm_castsi128_ps(a), _mm_castsi128_ps(b), (c))))
      12             : 
      13   533810466 : #define vu_rot16 vb_exch_adj_pair
      14             : 
      15             : static inline __attribute__((always_inline)) vu_t
      16   533810466 : vu_rot12( vu_t x ) {
      17   533810466 :   return vu_xor( vu_shr( x, 12 ), vu_shl( x, 32-12 ) );
      18   533810466 : }
      19             : 
      20             : static inline __attribute__((always_inline)) vu_t
      21   533810466 : vu_rot8( vu_t x ) {
      22   533810466 :   vb_t const mask = vb( 1,2,3,0, 5,6,7,4, 9,10,11,8, 13,14,15,12 );
      23   533810466 :   return _mm_shuffle_epi8( x, mask );
      24   533810466 : }
      25             : 
      26             : static inline __attribute__((always_inline)) vu_t
      27   533810466 : vu_rot7( vu_t x ) {
      28   533810466 :   return vu_xor( vu_shr( x, 7 ), vu_shl( x, 32-7 ) );
      29   533810466 : }
      30             : 
      31             : static inline __attribute__((always_inline)) void
      32             : g1( vu_t * row0,
      33             :     vu_t * row1,
      34             :     vu_t * row2,
      35             :     vu_t * row3,
      36   533810466 :     vu_t   m ) {
      37   533810466 :   *row0 = vu_add(vu_add(*row0, m), *row1);
      38   533810466 :   *row3 = vu_xor(*row3, *row0);
      39   533810466 :   *row3 = vu_rot16(*row3);
      40   533810466 :   *row2 = vu_add(*row2, *row3);
      41   533810466 :   *row1 = vu_xor(*row1, *row2);
      42   533810466 :   *row1 = vu_rot12(*row1);
      43   533810466 : }
      44             : 
      45             : static inline __attribute__((always_inline)) void
      46             : g2( vu_t * row0,
      47             :     vu_t * row1,
      48             :     vu_t * row2,
      49             :     vu_t * row3,
      50   533810466 :     vu_t   m ) {
      51   533810466 :   *row0 = vu_add(vu_add(*row0, m), *row1);
      52   533810466 :   *row3 = vu_xor(*row3, *row0);
      53   533810466 :   *row3 = vu_rot8(*row3);
      54   533810466 :   *row2 = vu_add(*row2, *row3);
      55   533810466 :   *row1 = vu_xor(*row1, *row2);
      56   533810466 :   *row1 = vu_rot7(*row1);
      57   533810466 : }
      58             : 
      59             : // Note the optimization here of leaving row1 as the unrotated row, rather than
      60             : // row0. All the message loads below are adjusted to compensate for this. See
      61             : // discussion at https://github.com/sneves/blake2-avx2/pull/4
      62             : static inline __attribute__((always_inline)) void
      63   266905233 : diagonalize(vu_t *row0, vu_t *row2, vu_t *row3) {
      64   266905233 :   *row0 = _mm_shuffle_epi32(*row0, _MM_SHUFFLE(2, 1, 0, 3));
      65   266905233 :   *row3 = _mm_shuffle_epi32(*row3, _MM_SHUFFLE(1, 0, 3, 2));
      66   266905233 :   *row2 = _mm_shuffle_epi32(*row2, _MM_SHUFFLE(0, 3, 2, 1));
      67   266905233 : }
      68             : 
      69             : static inline __attribute__((always_inline)) void
      70   266905233 : undiagonalize(vu_t *row0, vu_t *row2, vu_t *row3) {
      71   266905233 :   *row0 = _mm_shuffle_epi32(*row0, _MM_SHUFFLE(0, 3, 2, 1));
      72   266905233 :   *row3 = _mm_shuffle_epi32(*row3, _MM_SHUFFLE(1, 0, 3, 2));
      73   266905233 :   *row2 = _mm_shuffle_epi32(*row2, _MM_SHUFFLE(2, 1, 0, 3));
      74   266905233 : }
      75             : 
      76             : static inline __attribute__((always_inline)) void
      77             : compress_pre( vu_t        rows[4],
      78             :               uint  const cv[ static 8 ],
      79             :               uchar const block[ static FD_BLAKE3_BLOCK_SZ ],
      80             :               uint        block_len,
      81             :               ulong       ctr,
      82    38129319 :               uint        flags ) {
      83    38129319 :   rows[0] = vu_ld( cv   );
      84    38129319 :   rows[1] = vu_ld( cv+4 );
      85    38129319 :   rows[2] = vu( FD_BLAKE3_IV[0], FD_BLAKE3_IV[1], FD_BLAKE3_IV[2], FD_BLAKE3_IV[3] );
      86    38129319 :   rows[3] = vu( (uint)(ctr&UINT_MAX), (uint)(ctr>>32),
      87    38129319 :                 block_len,            flags );
      88             : 
      89    38129319 :   vu_t m0 = vb_ldu( block    ); vu_t m1 = vb_ldu( block+16 );
      90    38129319 :   vu_t m2 = vb_ldu( block+32 ); vu_t m3 = vb_ldu( block+48 );
      91             : 
      92    38129319 :   vu_t t0, t1, t2, t3, tt;
      93             : 
      94             :   // Round 1. The first round permutes the message words from the original
      95             :   // input order, into the groups that get mixed in parallel.
      96    38129319 :   t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(2, 0, 2, 0)); //  6  4  2  0
      97    38129319 :   g1(&rows[0], &rows[1], &rows[2], &rows[3], t0);
      98    38129319 :   t1 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 3, 1)); //  7  5  3  1
      99    38129319 :   g2(&rows[0], &rows[1], &rows[2], &rows[3], t1);
     100    38129319 :   diagonalize(&rows[0], &rows[2], &rows[3]);
     101    38129319 :   t2 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(2, 0, 2, 0)); // 14 12 10  8
     102    38129319 :   t2 = _mm_shuffle_epi32(t2, _MM_SHUFFLE(2, 1, 0, 3));   // 12 10  8 14
     103    38129319 :   g1(&rows[0], &rows[1], &rows[2], &rows[3], t2);
     104    38129319 :   t3 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 1, 3, 1)); // 15 13 11  9
     105    38129319 :   t3 = _mm_shuffle_epi32(t3, _MM_SHUFFLE(2, 1, 0, 3));   // 13 11  9 15
     106    38129319 :   g2(&rows[0], &rows[1], &rows[2], &rows[3], t3);
     107    38129319 :   undiagonalize(&rows[0], &rows[2], &rows[3]);
     108    38129319 :   m0 = t0;
     109    38129319 :   m1 = t1;
     110    38129319 :   m2 = t2;
     111    38129319 :   m3 = t3;
     112             : 
     113             :   // Round 2. This round and all following rounds apply a fixed permutation
     114             :   // to the message words from the round before.
     115    38129319 :   t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2));
     116    38129319 :   t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1));
     117    38129319 :   g1(&rows[0], &rows[1], &rows[2], &rows[3], t0);
     118    38129319 :   t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2));
     119    38129319 :   tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3));
     120    38129319 :   t1 = _mm_blend_epi16(tt, t1, 0xCC);
     121    38129319 :   g2(&rows[0], &rows[1], &rows[2], &rows[3], t1);
     122    38129319 :   diagonalize(&rows[0], &rows[2], &rows[3]);
     123    38129319 :   t2 = _mm_unpacklo_epi64(m3, m1);
     124    38129319 :   tt = _mm_blend_epi16(t2, m2, 0xC0);
     125    38129319 :   t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0));
     126    38129319 :   g1(&rows[0], &rows[1], &rows[2], &rows[3], t2);
     127    38129319 :   t3 = _mm_unpackhi_epi32(m1, m3);
     128    38129319 :   tt = _mm_unpacklo_epi32(m2, t3);
     129    38129319 :   t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2));
     130    38129319 :   g2(&rows[0], &rows[1], &rows[2], &rows[3], t3);
     131    38129319 :   undiagonalize(&rows[0], &rows[2], &rows[3]);
     132    38129319 :   m0 = t0;
     133    38129319 :   m1 = t1;
     134    38129319 :   m2 = t2;
     135    38129319 :   m3 = t3;
     136             : 
     137             :   // Round 3
     138    38129319 :   t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2));
     139    38129319 :   t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1));
     140    38129319 :   g1(&rows[0], &rows[1], &rows[2], &rows[3], t0);
     141    38129319 :   t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2));
     142    38129319 :   tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3));
     143    38129319 :   t1 = _mm_blend_epi16(tt, t1, 0xCC);
     144    38129319 :   g2(&rows[0], &rows[1], &rows[2], &rows[3], t1);
     145    38129319 :   diagonalize(&rows[0], &rows[2], &rows[3]);
     146    38129319 :   t2 = _mm_unpacklo_epi64(m3, m1);
     147    38129319 :   tt = _mm_blend_epi16(t2, m2, 0xC0);
     148    38129319 :   t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0));
     149    38129319 :   g1(&rows[0], &rows[1], &rows[2], &rows[3], t2);
     150    38129319 :   t3 = _mm_unpackhi_epi32(m1, m3);
     151    38129319 :   tt = _mm_unpacklo_epi32(m2, t3);
     152    38129319 :   t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2));
     153    38129319 :   g2(&rows[0], &rows[1], &rows[2], &rows[3], t3);
     154    38129319 :   undiagonalize(&rows[0], &rows[2], &rows[3]);
     155    38129319 :   m0 = t0;
     156    38129319 :   m1 = t1;
     157    38129319 :   m2 = t2;
     158    38129319 :   m3 = t3;
     159             : 
     160             :   // Round 4
     161    38129319 :   t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2));
     162    38129319 :   t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1));
     163    38129319 :   g1(&rows[0], &rows[1], &rows[2], &rows[3], t0);
     164    38129319 :   t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2));
     165    38129319 :   tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3));
     166    38129319 :   t1 = _mm_blend_epi16(tt, t1, 0xCC);
     167    38129319 :   g2(&rows[0], &rows[1], &rows[2], &rows[3], t1);
     168    38129319 :   diagonalize(&rows[0], &rows[2], &rows[3]);
     169    38129319 :   t2 = _mm_unpacklo_epi64(m3, m1);
     170    38129319 :   tt = _mm_blend_epi16(t2, m2, 0xC0);
     171    38129319 :   t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0));
     172    38129319 :   g1(&rows[0], &rows[1], &rows[2], &rows[3], t2);
     173    38129319 :   t3 = _mm_unpackhi_epi32(m1, m3);
     174    38129319 :   tt = _mm_unpacklo_epi32(m2, t3);
     175    38129319 :   t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2));
     176    38129319 :   g2(&rows[0], &rows[1], &rows[2], &rows[3], t3);
     177    38129319 :   undiagonalize(&rows[0], &rows[2], &rows[3]);
     178    38129319 :   m0 = t0;
     179    38129319 :   m1 = t1;
     180    38129319 :   m2 = t2;
     181    38129319 :   m3 = t3;
     182             : 
     183             :   // Round 5
     184    38129319 :   t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2));
     185    38129319 :   t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1));
     186    38129319 :   g1(&rows[0], &rows[1], &rows[2], &rows[3], t0);
     187    38129319 :   t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2));
     188    38129319 :   tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3));
     189    38129319 :   t1 = _mm_blend_epi16(tt, t1, 0xCC);
     190    38129319 :   g2(&rows[0], &rows[1], &rows[2], &rows[3], t1);
     191    38129319 :   diagonalize(&rows[0], &rows[2], &rows[3]);
     192    38129319 :   t2 = _mm_unpacklo_epi64(m3, m1);
     193    38129319 :   tt = _mm_blend_epi16(t2, m2, 0xC0);
     194    38129319 :   t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0));
     195    38129319 :   g1(&rows[0], &rows[1], &rows[2], &rows[3], t2);
     196    38129319 :   t3 = _mm_unpackhi_epi32(m1, m3);
     197    38129319 :   tt = _mm_unpacklo_epi32(m2, t3);
     198    38129319 :   t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2));
     199    38129319 :   g2(&rows[0], &rows[1], &rows[2], &rows[3], t3);
     200    38129319 :   undiagonalize(&rows[0], &rows[2], &rows[3]);
     201    38129319 :   m0 = t0;
     202    38129319 :   m1 = t1;
     203    38129319 :   m2 = t2;
     204    38129319 :   m3 = t3;
     205             : 
     206             :   // Round 6
     207    38129319 :   t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2));
     208    38129319 :   t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1));
     209    38129319 :   g1(&rows[0], &rows[1], &rows[2], &rows[3], t0);
     210    38129319 :   t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2));
     211    38129319 :   tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3));
     212    38129319 :   t1 = _mm_blend_epi16(tt, t1, 0xCC);
     213    38129319 :   g2(&rows[0], &rows[1], &rows[2], &rows[3], t1);
     214    38129319 :   diagonalize(&rows[0], &rows[2], &rows[3]);
     215    38129319 :   t2 = _mm_unpacklo_epi64(m3, m1);
     216    38129319 :   tt = _mm_blend_epi16(t2, m2, 0xC0);
     217    38129319 :   t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0));
     218    38129319 :   g1(&rows[0], &rows[1], &rows[2], &rows[3], t2);
     219    38129319 :   t3 = _mm_unpackhi_epi32(m1, m3);
     220    38129319 :   tt = _mm_unpacklo_epi32(m2, t3);
     221    38129319 :   t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2));
     222    38129319 :   g2(&rows[0], &rows[1], &rows[2], &rows[3], t3);
     223    38129319 :   undiagonalize(&rows[0], &rows[2], &rows[3]);
     224    38129319 :   m0 = t0;
     225    38129319 :   m1 = t1;
     226    38129319 :   m2 = t2;
     227    38129319 :   m3 = t3;
     228             : 
     229             :   // Round 7
     230    38129319 :   t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2));
     231    38129319 :   t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1));
     232    38129319 :   g1(&rows[0], &rows[1], &rows[2], &rows[3], t0);
     233    38129319 :   t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2));
     234    38129319 :   tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3));
     235    38129319 :   t1 = _mm_blend_epi16(tt, t1, 0xCC);
     236    38129319 :   g2(&rows[0], &rows[1], &rows[2], &rows[3], t1);
     237    38129319 :   diagonalize(&rows[0], &rows[2], &rows[3]);
     238    38129319 :   t2 = _mm_unpacklo_epi64(m3, m1);
     239    38129319 :   tt = _mm_blend_epi16(t2, m2, 0xC0);
     240    38129319 :   t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0));
     241    38129319 :   g1(&rows[0], &rows[1], &rows[2], &rows[3], t2);
     242    38129319 :   t3 = _mm_unpackhi_epi32(m1, m3);
     243    38129319 :   tt = _mm_unpacklo_epi32(m2, t3);
     244    38129319 :   t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2));
     245    38129319 :   g2(&rows[0], &rows[1], &rows[2], &rows[3], t3);
     246    38129319 :   undiagonalize(&rows[0], &rows[2], &rows[3]);
     247    38129319 : }
     248             : 
     249             : void
     250             : fd_blake3_sse_compress1( uchar * restrict       out,
     251             :                          uchar const * restrict msg,
     252             :                          uint                   msg_sz,
     253             :                          ulong                  counter,
     254             :                          uint const             flags,
     255             :                          uchar * restrict       out_chain,
     256    11200418 :                          uchar const * restrict in_chain ) {
     257    11200418 :   FD_BLAKE3_TRACE(( "fd_blake3_sse_compress1(out=%p,msg=%p,sz=%u,counter=%lu,flags=%02x)",
     258    11200418 :                     (void *)out, (void *)msg, msg_sz, counter, flags ));
     259    11200418 :   FD_DCHECK_CRIT( msg_sz<=FD_BLAKE3_CHUNK_SZ, "internal error" );
     260             : 
     261    11200418 :   uint cv[8] = { FD_BLAKE3_IV[0], FD_BLAKE3_IV[1], FD_BLAKE3_IV[2], FD_BLAKE3_IV[3],
     262    11200418 :                  FD_BLAKE3_IV[4], FD_BLAKE3_IV[5], FD_BLAKE3_IV[6], FD_BLAKE3_IV[7] };
     263    11200418 :   if( FD_UNLIKELY( in_chain ) ) {
     264           0 :     memcpy( cv, in_chain, FD_BLAKE3_OUTCHAIN_SZ );
     265           0 :   }
     266    11200418 :   vu_t rows[4];
     267             : 
     268    11200418 :   uint flag_mask = ~fd_uint_if( flags&FD_BLAKE3_FLAG_PARENT,
     269    11200418 :                                 FD_BLAKE3_FLAG_CHUNK_START|FD_BLAKE3_FLAG_CHUNK_END,
     270    11200418 :                                 0U );
     271             : 
     272    11200418 :   uint block_flags = flags | (flag_mask & FD_BLAKE3_FLAG_CHUNK_START);
     273    11200418 :   if( FD_UNLIKELY( in_chain && !(flags&FD_BLAKE3_FLAG_CHUNK_START) ) ) {
     274           0 :     block_flags &= ~FD_BLAKE3_FLAG_CHUNK_START;
     275           0 :   }
     276    48782816 :   do {
     277    48782816 :     uint block_sz = fd_uint_min( msg_sz, FD_BLAKE3_BLOCK_SZ );
     278    48782816 :     block_flags |= FD_BLAKE3_FLAG_CHUNK_END;
     279    48782816 :     block_flags &= (flag_mask & ~fd_uint_if( msg_sz<=FD_BLAKE3_BLOCK_SZ, 0, (FD_BLAKE3_FLAG_CHUNK_END|FD_BLAKE3_FLAG_ROOT) ) );
     280             : 
     281    48782816 :     uchar tail[ FD_BLAKE3_BLOCK_SZ ] __attribute__((aligned(16)));
     282    48782816 :     uchar const * restrict block;
     283    48782816 :     if( FD_LIKELY( msg_sz>=FD_BLAKE3_BLOCK_SZ ) ) {
     284    44785291 :       block = msg;
     285    44785291 :     } else {
     286     3997525 :       vb_st( tail,    vu_zero() );
     287     3997525 :       vb_st( tail+16, vu_zero() );
     288     3997525 :       vb_st( tail+32, vu_zero() );
     289     3997525 :       vb_st( tail+48, vu_zero() );
     290     3997525 :       fd_memcpy( tail, msg, msg_sz );
     291     3997525 :       block = tail;
     292     3997525 :     }
     293             : 
     294    48782816 :     if( FD_UNLIKELY( out_chain && (block_flags & FD_BLAKE3_FLAG_CHUNK_END) ) ) {
     295             :       /* FIXME better document and polish the transition from the compress
     296             :                part to the expand part. */
     297    10653497 :       fd_memcpy( out,       block, FD_BLAKE3_BLOCK_SZ    ); /* FIXME DOCUMENT OVERLOADING OF OUT ARGUMENT */
     298    10653497 :       fd_memcpy( out_chain, cv,    FD_BLAKE3_OUTCHAIN_SZ );
     299    10653497 :       FD_BLAKE3_TRACE(( "fd_blake3_sse_compress1: done (XOF mode)" ));
     300    10653497 :       return;
     301    10653497 :     }
     302             : 
     303    38129319 :     FD_BLAKE3_TRACE(( "fd_blake3_sse_compress1: sz=%u counter=%u flags=%x", block_sz, (uint)counter, block_flags ));
     304    38129319 :     compress_pre( rows, cv, block, block_sz, counter, block_flags );
     305    38129319 :     if( FD_UNLIKELY( in_chain ) ) {
     306             :       /* FIXME UGLY */
     307           0 :       vu_stu( out+32, vu_xor( vu_ld( cv   ), rows[2] ) );
     308           0 :       vu_stu( out+48, vu_xor( vu_ld( cv+4 ), rows[3] ) );
     309           0 :     }
     310    38129319 :     vu_st( cv,   vu_xor( rows[0], rows[2] ) );
     311    38129319 :     vu_st( cv+4, vu_xor( rows[1], rows[3] ) );
     312    38129319 :     msg    += FD_BLAKE3_BLOCK_SZ;
     313    38129319 :     msg_sz -= block_sz;
     314    38129319 :     block_flags = flags;
     315    38129319 :   } while( (int)msg_sz>0 );
     316             : 
     317      546921 :   vu_stu( out,    vu_ld( cv   ) );
     318      546921 :   vu_stu( out+16, vu_ld( cv+4 ) );
     319             : 
     320      546921 :   FD_BLAKE3_TRACE(( "fd_blake3_sse_compress1: done" ));
     321      546921 : }