Add: Float6 vector-vector x86 kernels

On Skylake: 5-6 GB/s
On Genoa: 9-10 GB/s

Author: ashvardanian

include/numkong/numkong.h

@@ -2534,6 +2534,14 @@ NK_INTERNAL void nk_find_kernel_punned_e5m2_(nk_capability_t v, nk_kernel_kind_t
 NK_INTERNAL void nk_find_kernel_punned_e2m3_(nk_capability_t v, nk_kernel_kind_t k, nk_kernel_punned_t *m,
                                              nk_capability_t *c) {
     typedef nk_kernel_punned_t m_t;
+#if NK_TARGET_GENOA
+    if (v & nk_cap_genoa_k) switch (k) {
+        case nk_kernel_euclidean_k: *m = (m_t)&nk_euclidean_e2m3_genoa, *c = nk_cap_genoa_k; return;
+        case nk_kernel_sqeuclidean_k: *m = (m_t)&nk_sqeuclidean_e2m3_genoa, *c = nk_cap_genoa_k; return;
+        case nk_kernel_angular_k: *m = (m_t)&nk_angular_e2m3_genoa, *c = nk_cap_genoa_k; return;
+        default: break;
+        }
+#endif
 #if NK_TARGET_NEONFHM
     if (v & nk_cap_neonfhm_k) switch (k) {
         case nk_kernel_dot_k: *m = (m_t)&nk_dot_e2m3_neonfhm, *c = nk_cap_neonfhm_k; return;
@@ -2543,6 +2551,14 @@ NK_INTERNAL void nk_find_kernel_punned_e2m3_(nk_capability_t v, nk_kernel_kind_t
         case nk_kernel_dots_symmetric_k: *m = (m_t)&nk_dots_symmetric_e2m3_neonfhm, *c = nk_cap_neonfhm_k; return;
         default: break;
         }
+#endif
+#if NK_TARGET_SKYLAKE
+    if (v & nk_cap_skylake_k) switch (k) {
+        case nk_kernel_euclidean_k: *m = (m_t)&nk_euclidean_e2m3_skylake, *c = nk_cap_skylake_k; return;
+        case nk_kernel_sqeuclidean_k: *m = (m_t)&nk_sqeuclidean_e2m3_skylake, *c = nk_cap_skylake_k; return;
+        case nk_kernel_angular_k: *m = (m_t)&nk_angular_e2m3_skylake, *c = nk_cap_skylake_k; return;
+        default: break;
+        }
 #endif
     if (v & nk_cap_serial_k) switch (k) {
         case nk_kernel_dot_k: *m = (m_t)&nk_dot_e2m3_serial, *c = nk_cap_serial_k; return;
@@ -2560,6 +2576,14 @@ NK_INTERNAL void nk_find_kernel_punned_e2m3_(nk_capability_t v, nk_kernel_kind_t
 NK_INTERNAL void nk_find_kernel_punned_e3m2_(nk_capability_t v, nk_kernel_kind_t k, nk_kernel_punned_t *m,
                                              nk_capability_t *c) {
     typedef nk_kernel_punned_t m_t;
+#if NK_TARGET_GENOA
+    if (v & nk_cap_genoa_k) switch (k) {
+        case nk_kernel_euclidean_k: *m = (m_t)&nk_euclidean_e3m2_genoa, *c = nk_cap_genoa_k; return;
+        case nk_kernel_sqeuclidean_k: *m = (m_t)&nk_sqeuclidean_e3m2_genoa, *c = nk_cap_genoa_k; return;
+        case nk_kernel_angular_k: *m = (m_t)&nk_angular_e3m2_genoa, *c = nk_cap_genoa_k; return;
+        default: break;
+        }
+#endif
 #if NK_TARGET_NEONFHM
     if (v & nk_cap_neonfhm_k) switch (k) {
         case nk_kernel_dot_k: *m = (m_t)&nk_dot_e3m2_neonfhm, *c = nk_cap_neonfhm_k; return;
@@ -2569,6 +2593,14 @@ NK_INTERNAL void nk_find_kernel_punned_e3m2_(nk_capability_t v, nk_kernel_kind_t
         case nk_kernel_dots_symmetric_k: *m = (m_t)&nk_dots_symmetric_e3m2_neonfhm, *c = nk_cap_neonfhm_k; return;
         default: break;
         }
+#endif
+#if NK_TARGET_SKYLAKE
+    if (v & nk_cap_skylake_k) switch (k) {
+        case nk_kernel_euclidean_k: *m = (m_t)&nk_euclidean_e3m2_skylake, *c = nk_cap_skylake_k; return;
+        case nk_kernel_sqeuclidean_k: *m = (m_t)&nk_sqeuclidean_e3m2_skylake, *c = nk_cap_skylake_k; return;
+        case nk_kernel_angular_k: *m = (m_t)&nk_angular_e3m2_skylake, *c = nk_cap_skylake_k; return;
+        default: break;
+        }
 #endif
     if (v & nk_cap_serial_k) switch (k) {
         case nk_kernel_dot_k: *m = (m_t)&nk_dot_e3m2_serial, *c = nk_cap_serial_k; return;

include/numkong/spatial/genoa.h

@@ -293,6 +293,128 @@ NK_PUBLIC void nk_angular_e5m2_genoa(nk_e5m2_t const *a, nk_e5m2_t const *b, nk_
     *result = nk_angular_normalize_f32_haswell_(dot_f32, a_norm_sq_f32, b_norm_sq_f32);
 }
 
+NK_PUBLIC void nk_sqeuclidean_e2m3_genoa(nk_e2m3_t const *a, nk_e2m3_t const *b, nk_size_t n, nk_f32_t *result) {
+    __m512 distance_sq_f32x16 = _mm512_setzero_ps();
+    __m256i a_e2m3x32, b_e2m3x32;
+
+nk_sqeuclidean_e2m3_genoa_cycle:
+    if (n < 32) {
+        __mmask32 mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, n);
+        a_e2m3x32 = _mm256_maskz_loadu_epi8(mask, a);
+        b_e2m3x32 = _mm256_maskz_loadu_epi8(mask, b);
+        n = 0;
+    }
+    else {
+        a_e2m3x32 = _mm256_loadu_epi8(a);
+        b_e2m3x32 = _mm256_loadu_epi8(b);
+        a += 32, b += 32, n -= 32;
+    }
+    __m512i a_bf16x32 = nk_e2m3x32_to_bf16x32_ice_(a_e2m3x32);
+    __m512i b_bf16x32 = nk_e2m3x32_to_bf16x32_ice_(b_e2m3x32);
+    __m512i diff_bf16x32 = nk_substract_bf16x32_genoa_(a_bf16x32, b_bf16x32);
+    distance_sq_f32x16 = _mm512_dpbf16_ps(distance_sq_f32x16, (__m512bh)(diff_bf16x32), (__m512bh)(diff_bf16x32));
+    if (n) goto nk_sqeuclidean_e2m3_genoa_cycle;
+
+    *result = nk_reduce_add_f32x16_skylake_(distance_sq_f32x16);
+}
+
+NK_PUBLIC void nk_euclidean_e2m3_genoa(nk_e2m3_t const *a, nk_e2m3_t const *b, nk_size_t n, nk_f32_t *result) {
+    nk_sqeuclidean_e2m3_genoa(a, b, n, result);
+    *result = nk_sqrt_f32_haswell_(*result);
+}
+
+NK_PUBLIC void nk_angular_e2m3_genoa(nk_e2m3_t const *a, nk_e2m3_t const *b, nk_size_t n, nk_f32_t *result) {
+    __m512 dot_f32x16 = _mm512_setzero_ps();
+    __m512 a_norm_sq_f32x16 = _mm512_setzero_ps();
+    __m512 b_norm_sq_f32x16 = _mm512_setzero_ps();
+    __m256i a_e2m3x32, b_e2m3x32;
+
+nk_angular_e2m3_genoa_cycle:
+    if (n < 32) {
+        __mmask32 mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, n);
+        a_e2m3x32 = _mm256_maskz_loadu_epi8(mask, a);
+        b_e2m3x32 = _mm256_maskz_loadu_epi8(mask, b);
+        n = 0;
+    }
+    else {
+        a_e2m3x32 = _mm256_loadu_epi8(a);
+        b_e2m3x32 = _mm256_loadu_epi8(b);
+        a += 32, b += 32, n -= 32;
+    }
+    __m512i a_bf16x32 = nk_e2m3x32_to_bf16x32_ice_(a_e2m3x32);
+    __m512i b_bf16x32 = nk_e2m3x32_to_bf16x32_ice_(b_e2m3x32);
+    dot_f32x16 = _mm512_dpbf16_ps(dot_f32x16, (__m512bh)(a_bf16x32), (__m512bh)(b_bf16x32));
+    a_norm_sq_f32x16 = _mm512_dpbf16_ps(a_norm_sq_f32x16, (__m512bh)(a_bf16x32), (__m512bh)(a_bf16x32));
+    b_norm_sq_f32x16 = _mm512_dpbf16_ps(b_norm_sq_f32x16, (__m512bh)(b_bf16x32), (__m512bh)(b_bf16x32));
+    if (n) goto nk_angular_e2m3_genoa_cycle;
+
+    nk_f32_t dot_f32 = nk_reduce_add_f32x16_skylake_(dot_f32x16);
+    nk_f32_t a_norm_sq_f32 = nk_reduce_add_f32x16_skylake_(a_norm_sq_f32x16);
+    nk_f32_t b_norm_sq_f32 = nk_reduce_add_f32x16_skylake_(b_norm_sq_f32x16);
+    *result = nk_angular_normalize_f32_haswell_(dot_f32, a_norm_sq_f32, b_norm_sq_f32);
+}
+
+NK_PUBLIC void nk_sqeuclidean_e3m2_genoa(nk_e3m2_t const *a, nk_e3m2_t const *b, nk_size_t n, nk_f32_t *result) {
+    __m512 distance_sq_f32x16 = _mm512_setzero_ps();
+    __m256i a_e3m2x32, b_e3m2x32;
+
+nk_sqeuclidean_e3m2_genoa_cycle:
+    if (n < 32) {
+        __mmask32 mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, n);
+        a_e3m2x32 = _mm256_maskz_loadu_epi8(mask, a);
+        b_e3m2x32 = _mm256_maskz_loadu_epi8(mask, b);
+        n = 0;
+    }
+    else {
+        a_e3m2x32 = _mm256_loadu_epi8(a);
+        b_e3m2x32 = _mm256_loadu_epi8(b);
+        a += 32, b += 32, n -= 32;
+    }
+    __m512i a_bf16x32 = nk_e3m2x32_to_bf16x32_ice_(a_e3m2x32);
+    __m512i b_bf16x32 = nk_e3m2x32_to_bf16x32_ice_(b_e3m2x32);
+    __m512i diff_bf16x32 = nk_substract_bf16x32_genoa_(a_bf16x32, b_bf16x32);
+    distance_sq_f32x16 = _mm512_dpbf16_ps(distance_sq_f32x16, (__m512bh)(diff_bf16x32), (__m512bh)(diff_bf16x32));
+    if (n) goto nk_sqeuclidean_e3m2_genoa_cycle;
+
+    *result = nk_reduce_add_f32x16_skylake_(distance_sq_f32x16);
+}
+
+NK_PUBLIC void nk_euclidean_e3m2_genoa(nk_e3m2_t const *a, nk_e3m2_t const *b, nk_size_t n, nk_f32_t *result) {
+    nk_sqeuclidean_e3m2_genoa(a, b, n, result);
+    *result = nk_sqrt_f32_haswell_(*result);
+}
+
+NK_PUBLIC void nk_angular_e3m2_genoa(nk_e3m2_t const *a, nk_e3m2_t const *b, nk_size_t n, nk_f32_t *result) {
+    __m512 dot_f32x16 = _mm512_setzero_ps();
+    __m512 a_norm_sq_f32x16 = _mm512_setzero_ps();
+    __m512 b_norm_sq_f32x16 = _mm512_setzero_ps();
+    __m256i a_e3m2x32, b_e3m2x32;
+
+nk_angular_e3m2_genoa_cycle:
+    if (n < 32) {
+        __mmask32 mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, n);
+        a_e3m2x32 = _mm256_maskz_loadu_epi8(mask, a);
+        b_e3m2x32 = _mm256_maskz_loadu_epi8(mask, b);
+        n = 0;
+    }
+    else {
+        a_e3m2x32 = _mm256_loadu_epi8(a);
+        b_e3m2x32 = _mm256_loadu_epi8(b);
+        a += 32, b += 32, n -= 32;
+    }
+    __m512i a_bf16x32 = nk_e3m2x32_to_bf16x32_ice_(a_e3m2x32);
+    __m512i b_bf16x32 = nk_e3m2x32_to_bf16x32_ice_(b_e3m2x32);
+    dot_f32x16 = _mm512_dpbf16_ps(dot_f32x16, (__m512bh)(a_bf16x32), (__m512bh)(b_bf16x32));
+    a_norm_sq_f32x16 = _mm512_dpbf16_ps(a_norm_sq_f32x16, (__m512bh)(a_bf16x32), (__m512bh)(a_bf16x32));
+    b_norm_sq_f32x16 = _mm512_dpbf16_ps(b_norm_sq_f32x16, (__m512bh)(b_bf16x32), (__m512bh)(b_bf16x32));
+    if (n) goto nk_angular_e3m2_genoa_cycle;
+
+    nk_f32_t dot_f32 = nk_reduce_add_f32x16_skylake_(dot_f32x16);
+    nk_f32_t a_norm_sq_f32 = nk_reduce_add_f32x16_skylake_(a_norm_sq_f32x16);
+    nk_f32_t b_norm_sq_f32 = nk_reduce_add_f32x16_skylake_(b_norm_sq_f32x16);
+    *result = nk_angular_normalize_f32_haswell_(dot_f32, a_norm_sq_f32, b_norm_sq_f32);
+}
+
 #if defined(__cplusplus)
 } // extern "C"
 #endif

include/numkong/spatial/skylake.h

@@ -577,6 +577,128 @@ NK_PUBLIC void nk_angular_e5m2_skylake(nk_e5m2_t const *a, nk_e5m2_t const *b, n
     *result = nk_angular_normalize_f32_haswell_(dot_f32, a_norm_sq_f32, b_norm_sq_f32);
 }
 
+NK_PUBLIC void nk_sqeuclidean_e2m3_skylake(nk_e2m3_t const *a, nk_e2m3_t const *b, nk_size_t n, nk_f32_t *result) {
+    __m512 sum_f32x16 = _mm512_setzero_ps();
+    __m128i a_e2m3x16, b_e2m3x16;
+
+nk_sqeuclidean_e2m3_skylake_cycle:
+    if (n < 16) {
+        __mmask16 mask = (__mmask16)_bzhi_u32(0xFFFF, n);
+        a_e2m3x16 = _mm_maskz_loadu_epi8(mask, a);
+        b_e2m3x16 = _mm_maskz_loadu_epi8(mask, b);
+        n = 0;
+    }
+    else {
+        a_e2m3x16 = _mm_loadu_si128((__m128i const *)a);
+        b_e2m3x16 = _mm_loadu_si128((__m128i const *)b);
+        a += 16, b += 16, n -= 16;
+    }
+    __m512 a_f32x16 = nk_e2m3x16_to_f32x16_skylake_(a_e2m3x16);
+    __m512 b_f32x16 = nk_e2m3x16_to_f32x16_skylake_(b_e2m3x16);
+    __m512 diff_f32x16 = _mm512_sub_ps(a_f32x16, b_f32x16);
+    sum_f32x16 = _mm512_fmadd_ps(diff_f32x16, diff_f32x16, sum_f32x16);
+    if (n) goto nk_sqeuclidean_e2m3_skylake_cycle;
+
+    *result = nk_reduce_add_f32x16_skylake_(sum_f32x16);
+}
+
+NK_PUBLIC void nk_euclidean_e2m3_skylake(nk_e2m3_t const *a, nk_e2m3_t const *b, nk_size_t n, nk_f32_t *result) {
+    nk_sqeuclidean_e2m3_skylake(a, b, n, result);
+    *result = nk_sqrt_f32_haswell_(*result);
+}
+
+NK_PUBLIC void nk_angular_e2m3_skylake(nk_e2m3_t const *a, nk_e2m3_t const *b, nk_size_t n, nk_f32_t *result) {
+    __m512 dot_f32x16 = _mm512_setzero_ps();
+    __m512 a_norm_sq_f32x16 = _mm512_setzero_ps();
+    __m512 b_norm_sq_f32x16 = _mm512_setzero_ps();
+    __m128i a_e2m3x16, b_e2m3x16;
+
+nk_angular_e2m3_skylake_cycle:
+    if (n < 16) {
+        __mmask16 mask = (__mmask16)_bzhi_u32(0xFFFF, n);
+        a_e2m3x16 = _mm_maskz_loadu_epi8(mask, a);
+        b_e2m3x16 = _mm_maskz_loadu_epi8(mask, b);
+        n = 0;
+    }
+    else {
+        a_e2m3x16 = _mm_loadu_si128((__m128i const *)a);
+        b_e2m3x16 = _mm_loadu_si128((__m128i const *)b);
+        a += 16, b += 16, n -= 16;
+    }
+    __m512 a_f32x16 = nk_e2m3x16_to_f32x16_skylake_(a_e2m3x16);
+    __m512 b_f32x16 = nk_e2m3x16_to_f32x16_skylake_(b_e2m3x16);
+    dot_f32x16 = _mm512_fmadd_ps(a_f32x16, b_f32x16, dot_f32x16);
+    a_norm_sq_f32x16 = _mm512_fmadd_ps(a_f32x16, a_f32x16, a_norm_sq_f32x16);
+    b_norm_sq_f32x16 = _mm512_fmadd_ps(b_f32x16, b_f32x16, b_norm_sq_f32x16);
+    if (n) goto nk_angular_e2m3_skylake_cycle;
+
+    nk_f32_t dot_f32 = nk_reduce_add_f32x16_skylake_(dot_f32x16);
+    nk_f32_t a_norm_sq_f32 = nk_reduce_add_f32x16_skylake_(a_norm_sq_f32x16);
+    nk_f32_t b_norm_sq_f32 = nk_reduce_add_f32x16_skylake_(b_norm_sq_f32x16);
+    *result = nk_angular_normalize_f32_haswell_(dot_f32, a_norm_sq_f32, b_norm_sq_f32);
+}
+
+NK_PUBLIC void nk_sqeuclidean_e3m2_skylake(nk_e3m2_t const *a, nk_e3m2_t const *b, nk_size_t n, nk_f32_t *result) {
+    __m512 sum_f32x16 = _mm512_setzero_ps();
+    __m128i a_e3m2x16, b_e3m2x16;
+
+nk_sqeuclidean_e3m2_skylake_cycle:
+    if (n < 16) {
+        __mmask16 mask = (__mmask16)_bzhi_u32(0xFFFF, n);
+        a_e3m2x16 = _mm_maskz_loadu_epi8(mask, a);
+        b_e3m2x16 = _mm_maskz_loadu_epi8(mask, b);
+        n = 0;
+    }
+    else {
+        a_e3m2x16 = _mm_loadu_si128((__m128i const *)a);
+        b_e3m2x16 = _mm_loadu_si128((__m128i const *)b);
+        a += 16, b += 16, n -= 16;
+    }
+    __m512 a_f32x16 = nk_e3m2x16_to_f32x16_skylake_(a_e3m2x16);
+    __m512 b_f32x16 = nk_e3m2x16_to_f32x16_skylake_(b_e3m2x16);
+    __m512 diff_f32x16 = _mm512_sub_ps(a_f32x16, b_f32x16);
+    sum_f32x16 = _mm512_fmadd_ps(diff_f32x16, diff_f32x16, sum_f32x16);
+    if (n) goto nk_sqeuclidean_e3m2_skylake_cycle;
+
+    *result = nk_reduce_add_f32x16_skylake_(sum_f32x16);
+}
+
+NK_PUBLIC void nk_euclidean_e3m2_skylake(nk_e3m2_t const *a, nk_e3m2_t const *b, nk_size_t n, nk_f32_t *result) {
+    nk_sqeuclidean_e3m2_skylake(a, b, n, result);
+    *result = nk_sqrt_f32_haswell_(*result);
+}
+
+NK_PUBLIC void nk_angular_e3m2_skylake(nk_e3m2_t const *a, nk_e3m2_t const *b, nk_size_t n, nk_f32_t *result) {
+    __m512 dot_f32x16 = _mm512_setzero_ps();
+    __m512 a_norm_sq_f32x16 = _mm512_setzero_ps();
+    __m512 b_norm_sq_f32x16 = _mm512_setzero_ps();
+    __m128i a_e3m2x16, b_e3m2x16;
+
+nk_angular_e3m2_skylake_cycle:
+    if (n < 16) {
+        __mmask16 mask = (__mmask16)_bzhi_u32(0xFFFF, n);
+        a_e3m2x16 = _mm_maskz_loadu_epi8(mask, a);
+        b_e3m2x16 = _mm_maskz_loadu_epi8(mask, b);
+        n = 0;
+    }
+    else {
+        a_e3m2x16 = _mm_loadu_si128((__m128i const *)a);
+        b_e3m2x16 = _mm_loadu_si128((__m128i const *)b);
+        a += 16, b += 16, n -= 16;
+    }
+    __m512 a_f32x16 = nk_e3m2x16_to_f32x16_skylake_(a_e3m2x16);
+    __m512 b_f32x16 = nk_e3m2x16_to_f32x16_skylake_(b_e3m2x16);
+    dot_f32x16 = _mm512_fmadd_ps(a_f32x16, b_f32x16, dot_f32x16);
+    a_norm_sq_f32x16 = _mm512_fmadd_ps(a_f32x16, a_f32x16, a_norm_sq_f32x16);
+    b_norm_sq_f32x16 = _mm512_fmadd_ps(b_f32x16, b_f32x16, b_norm_sq_f32x16);
+    if (n) goto nk_angular_e3m2_skylake_cycle;
+
+    nk_f32_t dot_f32 = nk_reduce_add_f32x16_skylake_(dot_f32x16);
+    nk_f32_t a_norm_sq_f32 = nk_reduce_add_f32x16_skylake_(a_norm_sq_f32x16);
+    nk_f32_t b_norm_sq_f32 = nk_reduce_add_f32x16_skylake_(b_norm_sq_f32x16);
+    *result = nk_angular_normalize_f32_haswell_(dot_f32, a_norm_sq_f32, b_norm_sq_f32);
+}
+
 #if defined(__cplusplus)
 } // extern "C"
 #endif

scripts/bench.cpp

@@ -1562,6 +1562,18 @@ int main(int argc, char **argv) {
 
     dense_<e4m3_k, f32_k>("dot_e4m3_skylake", nk_dot_e4m3_skylake);
     dense_<e5m2_k, f32_k>("dot_e5m2_skylake", nk_dot_e5m2_skylake);
+    dense_<e4m3_k, f32_k>("angular_e4m3_skylake", nk_angular_e4m3_skylake);
+    dense_<e4m3_k, f32_k>("sqeuclidean_e4m3_skylake", nk_sqeuclidean_e4m3_skylake);
+    dense_<e4m3_k, f32_k>("euclidean_e4m3_skylake", nk_euclidean_e4m3_skylake);
+    dense_<e5m2_k, f32_k>("angular_e5m2_skylake", nk_angular_e5m2_skylake);
+    dense_<e5m2_k, f32_k>("sqeuclidean_e5m2_skylake", nk_sqeuclidean_e5m2_skylake);
+    dense_<e5m2_k, f32_k>("euclidean_e5m2_skylake", nk_euclidean_e5m2_skylake);
+    dense_<e2m3_k, f32_k>("angular_e2m3_skylake", nk_angular_e2m3_skylake);
+    dense_<e2m3_k, f32_k>("sqeuclidean_e2m3_skylake", nk_sqeuclidean_e2m3_skylake);
+    dense_<e2m3_k, f32_k>("euclidean_e2m3_skylake", nk_euclidean_e2m3_skylake);
+    dense_<e3m2_k, f32_k>("angular_e3m2_skylake", nk_angular_e3m2_skylake);
+    dense_<e3m2_k, f32_k>("sqeuclidean_e3m2_skylake", nk_sqeuclidean_e3m2_skylake);
+    dense_<e3m2_k, f32_k>("euclidean_e3m2_skylake", nk_euclidean_e3m2_skylake);
 
     dense_<i8_k, i32_k>("dot_i8_skylake", nk_dot_i8_skylake);
     dense_<u8_k, u32_k>("dot_u8_skylake", nk_dot_u8_skylake);
@@ -1657,6 +1669,18 @@ int main(int argc, char **argv) {
 
     dense_<e4m3_k, f32_k>("dot_e4m3_genoa", nk_dot_e4m3_genoa);
     dense_<e5m2_k, f32_k>("dot_e5m2_genoa", nk_dot_e5m2_genoa);
+    dense_<e4m3_k, f32_k>("angular_e4m3_genoa", nk_angular_e4m3_genoa);
+    dense_<e4m3_k, f32_k>("sqeuclidean_e4m3_genoa", nk_sqeuclidean_e4m3_genoa);
+    dense_<e4m3_k, f32_k>("euclidean_e4m3_genoa", nk_euclidean_e4m3_genoa);
+    dense_<e5m2_k, f32_k>("angular_e5m2_genoa", nk_angular_e5m2_genoa);
+    dense_<e5m2_k, f32_k>("sqeuclidean_e5m2_genoa", nk_sqeuclidean_e5m2_genoa);
+    dense_<e5m2_k, f32_k>("euclidean_e5m2_genoa", nk_euclidean_e5m2_genoa);
+    dense_<e2m3_k, f32_k>("angular_e2m3_genoa", nk_angular_e2m3_genoa);
+    dense_<e2m3_k, f32_k>("sqeuclidean_e2m3_genoa", nk_sqeuclidean_e2m3_genoa);
+    dense_<e2m3_k, f32_k>("euclidean_e2m3_genoa", nk_euclidean_e2m3_genoa);
+    dense_<e3m2_k, f32_k>("angular_e3m2_genoa", nk_angular_e3m2_genoa);
+    dense_<e3m2_k, f32_k>("sqeuclidean_e3m2_genoa", nk_sqeuclidean_e3m2_genoa);
+    dense_<e3m2_k, f32_k>("euclidean_e3m2_genoa", nk_euclidean_e3m2_genoa);
 
     curved_<bf16_k, f32_k>("bilinear_bf16_genoa", nk_bilinear_bf16_genoa);
     curved_<bf16_k, f32_k>("mahalanobis_bf16_genoa", nk_mahalanobis_bf16_genoa);
@@ -1762,8 +1786,20 @@ int main(int argc, char **argv) {
 
     dense_<e4m3_k, f32_k>("dot_e4m3_serial", nk_dot_e4m3_serial);
     dense_<e5m2_k, f32_k>("dot_e5m2_serial", nk_dot_e5m2_serial);
+    dense_<e4m3_k, f32_k>("angular_e4m3_serial", nk_angular_e4m3_serial);
+    dense_<e4m3_k, f32_k>("sqeuclidean_e4m3_serial", nk_sqeuclidean_e4m3_serial);
+    dense_<e4m3_k, f32_k>("euclidean_e4m3_serial", nk_euclidean_e4m3_serial);
+    dense_<e5m2_k, f32_k>("angular_e5m2_serial", nk_angular_e5m2_serial);
+    dense_<e5m2_k, f32_k>("sqeuclidean_e5m2_serial", nk_sqeuclidean_e5m2_serial);
+    dense_<e5m2_k, f32_k>("euclidean_e5m2_serial", nk_euclidean_e5m2_serial);
     dense_<e2m3_k, f32_k>("dot_e2m3_serial", nk_dot_e2m3_serial);
+    dense_<e2m3_k, f32_k>("angular_e2m3_serial", nk_angular_e2m3_serial);
+    dense_<e2m3_k, f32_k>("sqeuclidean_e2m3_serial", nk_sqeuclidean_e2m3_serial);
+    dense_<e2m3_k, f32_k>("euclidean_e2m3_serial", nk_euclidean_e2m3_serial);
     dense_<e3m2_k, f32_k>("dot_e3m2_serial", nk_dot_e3m2_serial);
+    dense_<e3m2_k, f32_k>("angular_e3m2_serial", nk_angular_e3m2_serial);
+    dense_<e3m2_k, f32_k>("sqeuclidean_e3m2_serial", nk_sqeuclidean_e3m2_serial);
+    dense_<e3m2_k, f32_k>("euclidean_e3m2_serial", nk_euclidean_e3m2_serial);
 
     dense_<f16_k, f32_k>("dot_f16_serial", nk_dot_f16_serial);
     dense_<f16_k, f32_k>("angular_f16_serial", nk_angular_f16_serial);