15 #define BLAKE3_VERSION_STRING LLVM_BLAKE3_VERSION_STRING
16 #define BLAKE3_KEY_LEN LLVM_BLAKE3_KEY_LEN
17 #define BLAKE3_OUT_LEN LLVM_BLAKE3_OUT_LEN
18 #define BLAKE3_BLOCK_LEN LLVM_BLAKE3_BLOCK_LEN
19 #define BLAKE3_CHUNK_LEN LLVM_BLAKE3_CHUNK_LEN
20 #define BLAKE3_MAX_DEPTH LLVM_BLAKE3_MAX_DEPTH
21 #define blake3_hasher llvm_blake3_hasher
22 #define blake3_chunk_state llvm_blake3_chunk_state
38 #define INLINE static __forceinline
40 #define INLINE static inline __attribute__((always_inline))
43 #if defined(__x86_64__) || defined(_M_X64)
48 #if defined(__i386__) || defined(_M_IX86)
53 #if defined(__aarch64__) || defined(_M_ARM64)
61 #include <immintrin.h>
64 #if !defined(BLAKE3_USE_NEON)
66 #if defined(IS_AARCH64)
67 #define BLAKE3_USE_NEON 1
69 #define BLAKE3_USE_NEON 0
74 #define MAX_SIMD_DEGREE 16
75 #elif BLAKE3_USE_NEON == 1
76 #define MAX_SIMD_DEGREE 4
78 #define MAX_SIMD_DEGREE 1
83 #define MAX_SIMD_DEGREE_OR_2 (MAX_SIMD_DEGREE > 2 ? MAX_SIMD_DEGREE : 2)
85 static const uint32_t IV[8] = {0x6A09E667UL, 0xBB67AE85UL, 0x3C6EF372UL,
86 0xA54FF53AUL, 0x510E527FUL, 0x9B05688CUL,
87 0x1F83D9ABUL, 0x5BE0CD19UL};
90 {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15},
91 {2, 6, 3, 10, 7, 0, 4, 13, 1, 11, 12, 5, 9, 14, 15, 8},
92 {3, 4, 10, 12, 13, 2, 7, 14, 6, 5, 9, 0, 11, 15, 8, 1},
93 {10, 7, 12, 9, 14, 3, 13, 15, 4, 0, 11, 2, 5, 8, 1, 6},
94 {12, 13, 9, 11, 15, 10, 14, 8, 7, 2, 5, 3, 0, 1, 6, 4},
95 {9, 14, 11, 5, 8, 12, 15, 1, 13, 3, 0, 10, 2, 6, 4, 7},
96 {11, 15, 5, 0, 1, 9, 8, 6, 14, 10, 2, 12, 3, 4, 7, 13},
102 #if defined(__GNUC__) || defined(__clang__)
103 return 63 ^ __builtin_clzll(
x);
104 #elif defined(_MSC_VER) && defined(IS_X86_64)
106 _BitScanReverse64(&
index,
x);
108 #elif defined(_MSC_VER) && defined(IS_X86_32)
111 _BitScanReverse(&
index, (
unsigned long)(
x >> 32));
115 _BitScanReverse(&
index, (
unsigned long)
x);
120 if(
x & 0xffffffff00000000ULL) {
x >>= 32;
c += 32; }
121 if(
x & 0x00000000ffff0000ULL) {
x >>= 16;
c += 16; }
122 if(
x & 0x000000000000ff00ULL) {
x >>= 8;
c += 8; }
123 if(
x & 0x00000000000000f0ULL) {
x >>= 4;
c += 4; }
124 if(
x & 0x000000000000000cULL) {
x >>= 2;
c += 2; }
125 if(
x & 0x0000000000000002ULL) {
c += 1; }
132 #if defined(__GNUC__) || defined(__clang__)
133 return __builtin_popcountll(
x);
135 unsigned int count = 0;
157 const uint8_t *
p = (
const uint8_t *)src;
164 key_words[0] =
load32(&key[0 * 4]);
165 key_words[1] =
load32(&key[1 * 4]);
166 key_words[2] =
load32(&key[2 * 4]);
167 key_words[3] =
load32(&key[3 * 4]);
168 key_words[4] =
load32(&key[4 * 4]);
169 key_words[5] =
load32(&key[5 * 4]);
170 key_words[6] =
load32(&key[6 * 4]);
171 key_words[7] =
load32(&key[7 * 4]);
175 uint8_t *
p = (uint8_t *)dst;
176 p[0] = (uint8_t)(w >> 0);
177 p[1] = (uint8_t)(w >> 8);
178 p[2] = (uint8_t)(w >> 16);
179 p[3] = (uint8_t)(w >> 24);
183 store32(&bytes_out[0 * 4], cv_words[0]);
184 store32(&bytes_out[1 * 4], cv_words[1]);
185 store32(&bytes_out[2 * 4], cv_words[2]);
186 store32(&bytes_out[3 * 4], cv_words[3]);
187 store32(&bytes_out[4 * 4], cv_words[4]);
188 store32(&bytes_out[5 * 4], cv_words[5]);
189 store32(&bytes_out[6 * 4], cv_words[6]);
190 store32(&bytes_out[7 * 4], cv_words[7]);
196 uint8_t block_len,
uint64_t counter,
202 uint8_t block_len,
uint64_t counter, uint8_t flags,
208 bool increment_counter, uint8_t flags,
209 uint8_t flags_start, uint8_t flags_end, uint8_t *out);
219 uint8_t block_len,
uint64_t counter,
225 uint8_t block_len,
uint64_t counter,
226 uint8_t flags, uint8_t out[64]);
230 size_t blocks,
const uint32_t key[8],
231 uint64_t counter,
bool increment_counter,
232 uint8_t flags, uint8_t flags_start,
233 uint8_t flags_end, uint8_t *out);
236 #if !defined(BLAKE3_NO_SSE2)
240 uint8_t block_len,
uint64_t counter,
245 uint8_t block_len,
uint64_t counter,
246 uint8_t flags, uint8_t out[64]);
249 size_t blocks,
const uint32_t key[8],
250 uint64_t counter,
bool increment_counter,
251 uint8_t flags, uint8_t flags_start,
252 uint8_t flags_end, uint8_t *out);
254 #if !defined(BLAKE3_NO_SSE41)
258 uint8_t block_len,
uint64_t counter,
263 uint8_t block_len,
uint64_t counter,
264 uint8_t flags, uint8_t out[64]);
267 size_t blocks,
const uint32_t key[8],
268 uint64_t counter,
bool increment_counter,
269 uint8_t flags, uint8_t flags_start,
270 uint8_t flags_end, uint8_t *out);
272 #if !defined(BLAKE3_NO_AVX2)
275 size_t blocks,
const uint32_t key[8],
276 uint64_t counter,
bool increment_counter,
277 uint8_t flags, uint8_t flags_start,
278 uint8_t flags_end, uint8_t *out);
280 #if !defined(BLAKE3_NO_AVX512)
284 uint8_t block_len,
uint64_t counter,
290 uint8_t block_len,
uint64_t counter,
291 uint8_t flags, uint8_t out[64]);
295 size_t blocks,
const uint32_t key[8],
296 uint64_t counter,
bool increment_counter,
297 uint8_t flags, uint8_t flags_start,
298 uint8_t flags_end, uint8_t *out);
302 #if BLAKE3_USE_NEON == 1
304 void blake3_hash_many_neon(
const uint8_t *
const *inputs,
size_t num_inputs,
305 size_t blocks,
const uint32_t key[8],
306 uint64_t counter,
bool increment_counter,
307 uint8_t flags, uint8_t flags_start,
308 uint8_t flags_end, uint8_t *out);