Catcrafts/Crafter.Math
Watch
2
0
Fork
You've already forked Crafter.Math
 0
Code Issues 1 Pull requests Projects Releases Packages Wiki Activity Actions

ARM NEON

Browse source 
NEON backend for VectorF32 on AArch64: one float32x4_t per vector
(Len * Packing <= 4), with a vectorised sin/cos port of the SSE helpers.

VectorF16 gets a native float16x8_t path when the target has ARMv8.2-FP16
arithmetic. Without it VectorF16 aliases VectorF32, which converts
fp16 <-> fp32 on Load/Store.

The scalar fallback gains the batched Dot that MatrixRowMajor relies on.
Test tiers added for armv8-a and armv8.2-a+fp16.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Jorijn van der Graaf 2026-10-06 17:13:54 +02:00
commit 0c55e76873
Signed by: jorijnvdgraaf
GPG key ID: 2937E59CDCC1BCFB
6 changed files with 825 additions and 17 deletions
Download patch file Download diff file Expand all files Collapse all files

94
interfaces/Crafter.Math-Common.cppm
View file

@ -8,6 +8,9 @@ module;
#ifdef __wasm_simd128__
#include <wasm_simd128.h>
#endif
#if defined(__aarch64__) && defined(__ARM_NEON)
#include <arm_neon.h>
#endif
#ifdef __riscv_vector
#include <riscv_vector.h>
// Compile-time VLEN selection. RVV is VLA at the ISA level, but storage in
@ -37,9 +40,11 @@ import std;
// VectorF16 exists as a real struct when _Float16 is available AND we are not
// on x86_64 without AVX512FP16 (that path aliases VectorF16 to VectorF32 in
// Crafter.Math:Basic for performance). The same alias kicks in on RISC-V until
// a Zvfh path lands. Each translation unit that needs this distinction
// redefines the same condition since macros do not cross module boundaries.
#if defined(__FLT16_MAX__) && (!defined(__x86_64) || defined(__AVX512FP16__)) && !defined(__riscv_vector)
// a Zvfh path lands, and on AArch64 NEON without ARMv8.2-FP16 arithmetic,
// where VectorF32 converts fp16 <-> fp32 on Load/Store. Each translation unit
// that needs this distinction redefines the same condition since macros do not
// cross module boundaries.
#if defined(__FLT16_MAX__) && (!defined(__x86_64) || defined(__AVX512FP16__)) && !defined(__riscv_vector) && !(defined(__aarch64__) && defined(__ARM_NEON) && !defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC))
namespace Crafter {
export template <std::uint8_t Len, std::uint8_t Packing>
struct VectorF16;
@ -52,7 +57,7 @@ namespace Crafter {
template <std::uint8_t Len, std::uint8_t Packing, typename T>
struct VectorBase {
#if defined(__FLT16_MAX__) && (!defined(__x86_64) || defined(__AVX512FP16__)) && !defined(__riscv_vector)
#if defined(__FLT16_MAX__) && (!defined(__x86_64) || defined(__AVX512FP16__)) && !defined(__riscv_vector) && !(defined(__aarch64__) && defined(__ARM_NEON) && !defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC))
template <std::uint8_t L, std::uint8_t P>
friend struct VectorF16;
#endif
@ -89,6 +94,11 @@ namespace Crafter {
>;
#elif defined(__wasm_simd128__)
using VectorType = v128_t;
#elif defined(__aarch64__) && defined(__ARM_NEON)
// NEON registers are 128 bits wide, so every instantiation is one
// register: four floats, or eight halves for the native VectorF16
// that exists when the target has ARMv8.2-FP16 arithmetic.
using VectorType = std::conditional_t<std::is_same_v<T, float>, float32x4_t, float16x8_t>;
#elif defined(__riscv_vector)
// RVV tier mirrors the x86 selector: pick the widest register the
// toolchain guarantees, then size each instantiation down to the
@ -118,6 +128,10 @@ namespace Crafter {
// WASM SIMD only has 128-bit vectors; cap at 16 bytes so the entire
// VectorType always fits in a single v128_t.
static constexpr std::uint8_t Max = 16;
#elif defined(__aarch64__) && defined(__ARM_NEON)
// NEON only has 128-bit vectors; cap at 16 bytes so the entire
// VectorType always fits in a single register.
static constexpr std::uint8_t Max = 16;
#elif defined(__riscv_vector)
// RVV tier selected at compile time from the guaranteed VLEN. ZVL128B
// is the RVA23 baseline; ZVL256B / ZVL512B unlock wider registers
@ -698,5 +712,77 @@ namespace Crafter {
out_sin = _mm512_xor_ps(out_sin, x_sign);
}
#endif
#if defined(__aarch64__) && defined(__ARM_NEON)
// --- 128-bit (NEON) helpers, same algorithm as the SSE ones above ---
// Reduce |x| into [-pi/4, pi/4], return reduced value and quadrant
static constexpr void range_reduce_f32x4(float32x4_t ax, float32x4_t& r, float32x4_t& r2, int32x4_t& q) {
float32x4_t fq = vrndnq_f32(vmulq_f32(ax, vdupq_n_f32(two_over_pi)));
q = vcvtq_s32_f32(fq);
// ax - fq * pi/2 as an fmla with the constant negated, not as
// vfmsq_f32: that one is a wrapper function in clang's arm_neon.h,
// and calling it from two module units (here and :VectorF32) fails
// some importers with "definition with same mangled name as
// another definition".
r = vfmaq_f32(ax, fq, vdupq_n_f32(-pi_over_2_hi));
r = vfmaq_f32(r, fq, vdupq_n_f32(-pi_over_2_lo));
r2 = vmulq_f32(r, r);
}
// vfmaq_f32(a, b, c) is a + b*c, so each Horner step reads
// "next coefficient + accumulator * r2".
static constexpr void sincos_poly_f32x4(float32x4_t r, float32x4_t r2, float32x4_t& cos_r, float32x4_t& sin_r) {
cos_r = vfmaq_f32(vdupq_n_f32(c8), vdupq_n_f32(c10), r2);
cos_r = vfmaq_f32(vdupq_n_f32(c6), cos_r, r2);
cos_r = vfmaq_f32(vdupq_n_f32(c4), cos_r, r2);
cos_r = vfmaq_f32(vdupq_n_f32(c2), cos_r, r2);
cos_r = vfmaq_f32(vdupq_n_f32(c0), cos_r, r2);
sin_r = vfmaq_f32(vdupq_n_f32(s7), vdupq_n_f32(s9), r2);
sin_r = vfmaq_f32(vdupq_n_f32(s5), sin_r, r2);
sin_r = vfmaq_f32(vdupq_n_f32(s3), sin_r, r2);
sin_r = vfmaq_f32(vdupq_n_f32(s1), sin_r, r2);
sin_r = vfmaq_f32(vdupq_n_f32(1.0f), sin_r, r2);
sin_r = vmulq_f32(sin_r, r);
}
static constexpr void sincos_f32x4(float32x4_t x, float32x4_t& out_sin, float32x4_t& out_cos) {
uint32x4_t x_sign = vandq_u32(vreinterpretq_u32_f32(x), vdupq_n_u32(0x80000000u));
float32x4_t ax = vabsq_f32(x);
float32x4_t r, r2; int32x4_t q;
range_reduce_f32x4(ax, r, r2, q);
float32x4_t cos_r, sin_r;
sincos_poly_f32x4(r, r2, cos_r, sin_r);
uint32x4_t is_odd = vtstq_s32(q, vdupq_n_s32(1));
// cos: swap on odd, negate if (q+1)&2
out_cos = vbslq_f32(is_odd, sin_r, cos_r);
int32x4_t cos_neg = vandq_s32(vaddq_s32(q, vdupq_n_s32(1)), vdupq_n_s32(2));
out_cos = vreinterpretq_f32_s32(veorq_s32(vreinterpretq_s32_f32(out_cos), vshlq_n_s32(cos_neg, 30)));
// sin: swap on odd, negate if q&2, apply input sign
out_sin = vbslq_f32(is_odd, cos_r, sin_r);
int32x4_t sin_neg = vandq_s32(q, vdupq_n_s32(2));
out_sin = vreinterpretq_f32_s32(veorq_s32(vreinterpretq_s32_f32(out_sin), vshlq_n_s32(sin_neg, 30)));
out_sin = vreinterpretq_f32_u32(veorq_u32(vreinterpretq_u32_f32(out_sin), x_sign));
}
// The single-result variants share sincos_f32x4; the half that is not
// returned is dead code the compiler drops after inlining.
static constexpr float32x4_t cos_f32x4(float32x4_t x) {
float32x4_t s, c;
sincos_f32x4(x, s, c);
return c;
}
static constexpr float32x4_t sin_f32x4(float32x4_t x) {
float32x4_t s, c;
sincos_f32x4(x, s, c);
return s;
}
#endif
};
}